diff --git a/.gitignore b/.gitignore index 33f95c7902c..aa503c4d337 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,7 @@ awsapi/modules/* .settings/ db.properties.override awsapi/overlays/ +tools/marvin/marvin/cloudstackAPI/* *.egg-info/ docs/tmp docs/publish diff --git a/api/src/com/cloud/agent/api/BackupSnapshotCommand.java b/api/src/com/cloud/agent/api/BackupSnapshotCommand.java index 9476d7d7065..04a0bb4a44d 100644 --- a/api/src/com/cloud/agent/api/BackupSnapshotCommand.java +++ b/api/src/com/cloud/agent/api/BackupSnapshotCommand.java @@ -17,6 +17,7 @@ package com.cloud.agent.api; import com.cloud.agent.api.LogLevel.Log4jLevel; +import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.storage.StoragePool; @@ -32,6 +33,7 @@ public class BackupSnapshotCommand extends SnapshotCommand { private Long snapshotId; @LogLevel(Log4jLevel.Off) private SwiftTO swift; + private S3TO s3; StorageFilerTO pool; protected BackupSnapshotCommand() { @@ -88,7 +90,7 @@ public class BackupSnapshotCommand extends SnapshotCommand { } public String getVmName() { - return vmName; + return vmName; } public SwiftTO getSwift() { @@ -99,6 +101,14 @@ public class BackupSnapshotCommand extends SnapshotCommand { this.swift = swift; } + public S3TO getS3() { + return s3; + } + + public void setS3(S3TO s3) { + this.s3 = s3; + } + public Long getSnapshotId() { return snapshotId; } diff --git a/api/src/com/cloud/agent/api/DeleteSnapshotBackupCommand.java b/api/src/com/cloud/agent/api/DeleteSnapshotBackupCommand.java index 3fa8c2b7eb4..5e5557002e6 100644 --- a/api/src/com/cloud/agent/api/DeleteSnapshotBackupCommand.java +++ b/api/src/com/cloud/agent/api/DeleteSnapshotBackupCommand.java @@ -17,6 +17,7 @@ package com.cloud.agent.api; import com.cloud.agent.api.LogLevel.Log4jLevel; +import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; /** @@ -26,6 +27,7 @@ import com.cloud.agent.api.to.SwiftTO; public class DeleteSnapshotBackupCommand extends SnapshotCommand { @LogLevel(Log4jLevel.Off) private SwiftTO swift; + private S3TO s3; private Boolean all; public SwiftTO getSwift() { @@ -44,6 +46,10 @@ public class DeleteSnapshotBackupCommand extends SnapshotCommand { this.swift = swift; } + public S3TO getS3() { + return s3; + } + protected DeleteSnapshotBackupCommand() { } @@ -73,6 +79,7 @@ public class DeleteSnapshotBackupCommand extends SnapshotCommand { * @param childUUID The child VHD file of the backup whose parent is reset to its grandparent. */ public DeleteSnapshotBackupCommand(SwiftTO swift, + S3TO s3, String secondaryStoragePoolURL, Long dcId, Long accountId, @@ -81,6 +88,7 @@ public class DeleteSnapshotBackupCommand extends SnapshotCommand { { super(null, secondaryStoragePoolURL, backupUUID, null, dcId, accountId, volumeId); setSwift(swift); + this.s3 = s3; setAll(all); } } \ No newline at end of file diff --git a/api/src/com/cloud/agent/api/DeleteTemplateFromS3Command.java b/api/src/com/cloud/agent/api/DeleteTemplateFromS3Command.java new file mode 100644 index 00000000000..278669b2c97 --- /dev/null +++ b/api/src/com/cloud/agent/api/DeleteTemplateFromS3Command.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.agent.api; + +import com.cloud.agent.api.to.S3TO; + +public class DeleteTemplateFromS3Command extends Command { + + private S3TO s3; + private Long templateId; + private Long accountId; + + protected DeleteTemplateFromS3Command() { + super(); + } + + public DeleteTemplateFromS3Command(final S3TO s3, final Long accountId, + final Long templateId) { + + super(); + + this.s3 = s3; + this.accountId = accountId; + this.templateId = templateId; + + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((accountId == null) ? 0 : accountId.hashCode()); + result = prime * result + ((s3 == null) ? 0 : s3.hashCode()); + result = prime * result + + ((templateId == null) ? 0 : templateId.hashCode()); + return result; + } + + @Override + public boolean equals(Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null) { + return false; + } + + if (getClass() != thatObject.getClass()) { + return false; + } + + final DeleteTemplateFromS3Command thatCommand = (DeleteTemplateFromS3Command) thatObject; + + if (!(accountId == thatCommand.accountId) + || (this.accountId != null && this.accountId + .equals(thatCommand.accountId))) { + return false; + } + + if (!(templateId == thatCommand.templateId) + || (this.templateId != null && this.templateId + .equals(thatCommand.templateId))) { + return false; + } + + return true; + + } + + public S3TO getS3() { + return s3; + } + + public Long getTemplateId() { + return templateId; + } + + public Long getAccountId() { + return accountId; + } + + @Override + public boolean executeInSequence() { + return true; + } + +} diff --git a/api/src/com/cloud/agent/api/DownloadSnapshotFromS3Command.java b/api/src/com/cloud/agent/api/DownloadSnapshotFromS3Command.java new file mode 100644 index 00000000000..1c44dcd6846 --- /dev/null +++ b/api/src/com/cloud/agent/api/DownloadSnapshotFromS3Command.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.agent.api; + +import com.cloud.agent.api.to.S3TO; + +public class DownloadSnapshotFromS3Command extends SnapshotCommand { + + private S3TO s3; + private String parent; + + protected DownloadSnapshotFromS3Command() { + super(); + } + + public DownloadSnapshotFromS3Command(S3TO s3, String parent, + String secondaryStorageUrl, Long dcId, Long accountId, + Long volumeId, String backupUuid, int wait) { + + super("", secondaryStorageUrl, backupUuid, "", dcId, accountId, + volumeId); + + this.s3 = s3; + this.parent = parent; + setWait(wait); + + } + + public S3TO getS3() { + return s3; + } + + public void setS3(S3TO s3) { + this.s3 = s3; + } + + public String getParent() { + return parent; + } + + public void setParent(String parent) { + this.parent = parent; + } + +} diff --git a/api/src/com/cloud/agent/api/DownloadTemplateFromS3ToSecondaryStorageCommand.java b/api/src/com/cloud/agent/api/DownloadTemplateFromS3ToSecondaryStorageCommand.java new file mode 100644 index 00000000000..af61228c020 --- /dev/null +++ b/api/src/com/cloud/agent/api/DownloadTemplateFromS3ToSecondaryStorageCommand.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.agent.api; + +import com.cloud.agent.api.to.S3TO; + +public final class DownloadTemplateFromS3ToSecondaryStorageCommand extends Command { + + private final S3TO s3; + private final Long accountId; + private final Long templateId; + private final String storagePath; + + public DownloadTemplateFromS3ToSecondaryStorageCommand(final S3TO s3, + final Long accountId, final Long templateId, + final String storagePath, final int wait) { + + super(); + + this.s3 = s3; + this.accountId = accountId; + this.templateId = templateId; + this.storagePath = storagePath; + + setWait(wait); + + } + + public S3TO getS3() { + return this.s3; + } + + public Long getAccountId() { + return this.accountId; + } + + public Long getTemplateId() { + return this.templateId; + } + + public String getStoragePath() { + return this.storagePath; + } + + @Override + public boolean executeInSequence() { + return true; + } + +} diff --git a/api/src/com/cloud/agent/api/UploadTemplateToS3FromSecondaryStorageCommand.java b/api/src/com/cloud/agent/api/UploadTemplateToS3FromSecondaryStorageCommand.java new file mode 100644 index 00000000000..1807cd56315 --- /dev/null +++ b/api/src/com/cloud/agent/api/UploadTemplateToS3FromSecondaryStorageCommand.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.agent.api; + +import com.cloud.agent.api.to.S3TO; + +public class UploadTemplateToS3FromSecondaryStorageCommand extends Command { + + private final S3TO s3; + private final String storagePath; + private final Long dataCenterId; + private final Long accountId; + private final Long templateId; + + public UploadTemplateToS3FromSecondaryStorageCommand(final S3TO s3, + final String storagePath, final Long dataCenterId, final Long accountId, + final Long templateId) { + + super(); + + this.s3 = s3; + this.storagePath = storagePath; + this.dataCenterId = dataCenterId; + this.accountId = accountId; + this.templateId = templateId; + + } + + @Override + public boolean executeInSequence() { + return false; + } + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null || getClass() != thatObject.getClass()) { + return false; + } + + final UploadTemplateToS3FromSecondaryStorageCommand thatCommand = + (UploadTemplateToS3FromSecondaryStorageCommand) thatObject; + + if (this.accountId != null ? !this.accountId.equals(thatCommand + .accountId) : thatCommand.accountId != null) { + return false; + } + + if (this.dataCenterId != null ? !this.dataCenterId.equals(thatCommand + .dataCenterId) : thatCommand.dataCenterId != null) { + return false; + } + + if (this.s3 != null ? !this.s3.equals(thatCommand.s3) : thatCommand.s3 != null) { + return false; + } + + if (this.storagePath != null ? !this.storagePath.equals(thatCommand + .storagePath) : thatCommand.storagePath != null) { + return false; + } + + if (this.templateId != null ? !this.templateId.equals(thatCommand.templateId) : + thatCommand.templateId != null) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + int result = this.s3 != null ? this.s3.hashCode() : 0; + result = 31 * result + (this.storagePath != null ? this.storagePath.hashCode() : 0); + result = 31 * result + (this.dataCenterId != null ? this.dataCenterId.hashCode() : 0); + result = 31 * result + (this.accountId != null ? this.accountId.hashCode() : 0); + result = 31 * result + (this.templateId != null ? this.templateId.hashCode() : 0); + return result; + } + + public S3TO getS3() { + return this.s3; + } + + public String getStoragePath() { + return this.storagePath; + } + + public Long getDataCenterId() { + return this.dataCenterId; + } + + public Long getAccountId() { + return this.accountId; + } + + public Long getTemplateId() { + return this.templateId; + } + +} diff --git a/api/src/com/cloud/agent/api/to/S3TO.java b/api/src/com/cloud/agent/api/to/S3TO.java new file mode 100644 index 00000000000..879df229c31 --- /dev/null +++ b/api/src/com/cloud/agent/api/to/S3TO.java @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.to; + +import com.cloud.utils.S3Utils; + +import java.util.Date; + +public final class S3TO implements S3Utils.ClientOptions { + + private Long id; + private String uuid; + private String accessKey; + private String secretKey; + private String endPoint; + private String bucketName; + private Boolean httpsFlag; + private Integer connectionTimeout; + private Integer maxErrorRetry; + private Integer socketTimeout; + private Date created; + + public S3TO() { + + super(); + + } + + public S3TO(final Long id, final String uuid, final String accessKey, + final String secretKey, final String endPoint, + final String bucketName, final Boolean httpsFlag, + final Integer connectionTimeout, final Integer maxErrorRetry, + final Integer socketTimeout, final Date created) { + + super(); + + this.id = id; + this.uuid = uuid; + this.accessKey = accessKey; + this.secretKey = secretKey; + this.endPoint = endPoint; + this.bucketName = bucketName; + this.httpsFlag = httpsFlag; + this.connectionTimeout = connectionTimeout; + this.maxErrorRetry = maxErrorRetry; + this.socketTimeout = socketTimeout; + this.created = created; + + } + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) + return true; + if (thatObject == null || getClass() != thatObject.getClass()) + return false; + + final S3TO thatS3TO = (S3TO) thatObject; + + if (httpsFlag != null ? !httpsFlag.equals(thatS3TO.httpsFlag) + : thatS3TO.httpsFlag != null) { + return false; + } + + if (accessKey != null ? !accessKey.equals(thatS3TO.accessKey) + : thatS3TO.accessKey != null) { + return false; + } + + if (connectionTimeout != null ? !connectionTimeout + .equals(thatS3TO.connectionTimeout) + : thatS3TO.connectionTimeout != null) { + return false; + } + + if (endPoint != null ? !endPoint.equals(thatS3TO.endPoint) + : thatS3TO.endPoint != null) { + return false; + } + + if (id != null ? !id.equals(thatS3TO.id) : thatS3TO.id != null) { + return false; + } + + if (uuid != null ? !uuid.equals(thatS3TO.uuid) : thatS3TO.uuid != null) { + return false; + } + + if (maxErrorRetry != null ? !maxErrorRetry + .equals(thatS3TO.maxErrorRetry) + : thatS3TO.maxErrorRetry != null) { + return false; + } + + if (secretKey != null ? !secretKey.equals(thatS3TO.secretKey) + : thatS3TO.secretKey != null) { + return false; + } + + if (socketTimeout != null ? !socketTimeout + .equals(thatS3TO.socketTimeout) + : thatS3TO.socketTimeout != null) { + return false; + } + + if (bucketName != null ? !bucketName.equals(thatS3TO.bucketName) + : thatS3TO.bucketName != null) { + return false; + } + + if (created != null ? !created.equals(thatS3TO.created) + : thatS3TO.created != null) { + return false; + } + + return true; + + } + + @Override + public int hashCode() { + + int result = id != null ? id.hashCode() : 0; + + result = 31 * result + (accessKey != null ? accessKey.hashCode() : 0); + result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0); + result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0); + result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0); + result = 31 * result + (httpsFlag ? 1 : 0); + result = 31 + * result + + (connectionTimeout != null ? connectionTimeout.hashCode() : 0); + result = 31 * result + + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0); + result = 31 * result + + (socketTimeout != null ? socketTimeout.hashCode() : 0); + + return result; + + } + + public Long getId() { + return this.id; + } + + public void setId(final Long id) { + this.id = id; + } + + public String getUuid() { + return this.uuid; + } + + public void setUuid(final String uuid) { + this.uuid = uuid; + } + + @Override + public String getAccessKey() { + return this.accessKey; + } + + public void setAccessKey(final String accessKey) { + this.accessKey = accessKey; + } + + @Override + public String getSecretKey() { + return this.secretKey; + } + + public void setSecretKey(final String secretKey) { + this.secretKey = secretKey; + } + + @Override + public String getEndPoint() { + return this.endPoint; + } + + public void setEndPoint(final String endPoint) { + this.endPoint = endPoint; + } + + public String getBucketName() { + return this.bucketName; + } + + public void setBucketName(final String bucketName) { + this.bucketName = bucketName; + } + + @Override + public Boolean isHttps() { + return this.httpsFlag; + } + + public void setHttps(final Boolean httpsFlag) { + this.httpsFlag = httpsFlag; + } + + @Override + public Integer getConnectionTimeout() { + return connectionTimeout; + } + + public void setConnectionTimeout(final Integer connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + @Override + public Integer getMaxErrorRetry() { + return maxErrorRetry; + } + + public void setMaxErrorRetry(final Integer maxErrorRetry) { + this.maxErrorRetry = maxErrorRetry; + } + + @Override + public Integer getSocketTimeout() { + return socketTimeout; + } + + public void setSocketTimeout(final Integer socketTimeout) { + this.socketTimeout = socketTimeout; + } + + public Date getCreated() { + return this.created; + } + + public void setCreated(final Date created) { + this.created = created; + } + +} diff --git a/api/src/com/cloud/api/ApiConstants.java b/api/src/com/cloud/api/ApiConstants.java index 78a3deda578..9f1c5eaa33a 100755 --- a/api/src/com/cloud/api/ApiConstants.java +++ b/api/src/com/cloud/api/ApiConstants.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.api; +import org.omg.CORBA.PUBLIC_MEMBER; + public class ApiConstants { public static final String ACCOUNT = "account"; public static final String ACCOUNTS = "accounts"; @@ -383,6 +385,14 @@ public class ApiConstants { public static final String NICIRA_NVP_TRANSPORT_ZONE_UUID = "transportzoneuuid"; public static final String NICIRA_NVP_DEVICE_NAME = "niciradevicename"; public static final String NICIRA_NVP_GATEWAYSERVICE_UUID = "l3gatewayserviceuuid"; + public static final String S3_ACCESS_KEY = "accesskey"; + public static final String S3_SECRET_KEY = "secretkey"; + public static final String S3_END_POINT = "endpoint"; + public static final String S3_BUCKET_NAME = "bucket"; + public static final String S3_HTTPS_FLAG = "usehttps"; + public static final String S3_CONNECTION_TIMEOUT = "connectiontimeout"; + public static final String S3_MAX_ERROR_RETRY = "maxerrorretry"; + public static final String S3_SOCKET_TIMEOUT = "sockettimeout"; public static final String SOURCE = "source"; public static final String COUNTER_ID = "counterid"; diff --git a/api/src/com/cloud/api/ResponseGenerator.java b/api/src/com/cloud/api/ResponseGenerator.java index 4e8fbd824a9..ec6c00ba8d4 100755 --- a/api/src/com/cloud/api/ResponseGenerator.java +++ b/api/src/com/cloud/api/ResponseGenerator.java @@ -64,6 +64,7 @@ import com.cloud.api.response.RemoteAccessVpnResponse; import com.cloud.api.response.ResourceCountResponse; import com.cloud.api.response.ResourceLimitResponse; import com.cloud.api.response.ResourceTagResponse; +import com.cloud.api.response.S3Response; import com.cloud.api.response.SecurityGroupResponse; import com.cloud.api.response.ServiceOfferingResponse; import com.cloud.api.response.ServiceResponse; @@ -141,6 +142,7 @@ import com.cloud.projects.Project; import com.cloud.projects.ProjectAccount; import com.cloud.projects.ProjectInvitation; import com.cloud.server.ResourceTag; +import com.cloud.storage.S3; import com.cloud.storage.Snapshot; import com.cloud.storage.StoragePool; import com.cloud.storage.Swift; @@ -287,6 +289,8 @@ public interface ResponseGenerator { SwiftResponse createSwiftResponse(Swift swift); + S3Response createS3Response(S3 result); + PhysicalNetworkResponse createPhysicalNetworkResponse(PhysicalNetwork result); ServiceResponse createNetworkServiceResponse(Service service); @@ -361,4 +365,5 @@ public interface ResponseGenerator { AutoScaleVmProfileResponse createAutoScaleVmProfileResponse(AutoScaleVmProfile profile); AutoScaleVmGroupResponse createAutoScaleVmGroupResponse(AutoScaleVmGroup vmGroup); + } diff --git a/api/src/com/cloud/api/commands/AddS3Cmd.java b/api/src/com/cloud/api/commands/AddS3Cmd.java new file mode 100644 index 00000000000..e046ccc0160 --- /dev/null +++ b/api/src/com/cloud/api/commands/AddS3Cmd.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.api.commands; + +import static com.cloud.api.ApiConstants.S3_ACCESS_KEY; +import static com.cloud.api.ApiConstants.S3_CONNECTION_TIMEOUT; +import static com.cloud.api.ApiConstants.S3_END_POINT; +import static com.cloud.api.ApiConstants.S3_HTTPS_FLAG; +import static com.cloud.api.ApiConstants.S3_MAX_ERROR_RETRY; +import static com.cloud.api.ApiConstants.S3_SECRET_KEY; +import static com.cloud.api.ApiConstants.S3_SOCKET_TIMEOUT; +import static com.cloud.api.ApiConstants.S3_BUCKET_NAME; +import static com.cloud.api.BaseCmd.CommandType.INTEGER; +import static com.cloud.api.BaseCmd.CommandType.STRING; +import static com.cloud.api.BaseCmd.CommandType.BOOLEAN; +import static com.cloud.user.Account.ACCOUNT_ID_SYSTEM; + +import com.cloud.api.BaseCmd; +import com.cloud.api.Implementation; +import com.cloud.api.Parameter; +import com.cloud.api.ServerApiException; +import com.cloud.api.response.S3Response; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.DiscoveryException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.storage.S3; + +@Implementation(description = "Adds S3", responseObject = S3Response.class, since = "4.0.0") +public final class AddS3Cmd extends BaseCmd { + + private static String COMMAND_NAME = "adds3response"; + + @Parameter(name = S3_ACCESS_KEY, type = STRING, required = true, + description = "S3 access key") + private String accessKey; + + @Parameter(name = S3_SECRET_KEY, type = STRING, required = true, + description = "S3 secret key") + private String secretKey; + + @Parameter(name = S3_END_POINT, type = STRING, required = false, + description = "S3 host name") + private String endPoint = null; + + @Parameter(name = S3_BUCKET_NAME, type = STRING, required = true, + description = "name of the template storage bucket") + private String bucketName; + + @Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false, + description = "connect to the S3 endpoint via HTTPS?") + private Boolean httpsFlag = null; + + @Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false, + description = "connection timeout (milliseconds)") + private Integer connectionTimeout = null; + + @Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false, + description = "maximum number of times to retry on error") + private Integer maxErrorRetry = null; + + @Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false, + description = "socket timeout (milliseconds)") + private Integer socketTimeout = null; + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, + ServerApiException, ConcurrentOperationException, ResourceAllocationException, + NetworkRuleConflictException { + + final S3 result; + + try { + + result = _resourceService.discoverS3(this); + + if (result == null) { + throw new ServerApiException(INTERNAL_ERROR, "Failed to add S3."); + } + + } catch (DiscoveryException e) { + + throw new ServerApiException(INTERNAL_ERROR, "Failed to add S3 due to " + e.getMessage()); + + } + + final S3Response response = _responseGenerator.createS3Response(result); + response.setResponseName(this.getCommandName()); + this.setResponseObject(response); + + } + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null || this.getClass() != thatObject.getClass()) { + return false; + } + + final AddS3Cmd thatAddS3Cmd = (AddS3Cmd) thatObject; + + if (this.httpsFlag != null ? !this.httpsFlag.equals(thatAddS3Cmd.httpsFlag) : thatAddS3Cmd.httpsFlag != null) { + return false; + } + + if (this.accessKey != null ? !this.accessKey.equals(thatAddS3Cmd.accessKey) : thatAddS3Cmd.accessKey != null) { + return false; + } + + if (this.connectionTimeout != null ? !this.connectionTimeout.equals(thatAddS3Cmd.connectionTimeout) : thatAddS3Cmd.connectionTimeout != null) { + return false; + } + + if (this.endPoint != null ? !this.endPoint.equals(thatAddS3Cmd.endPoint) : thatAddS3Cmd.endPoint != null) { + return false; + } + + if (this.maxErrorRetry != null ? !this.maxErrorRetry.equals(thatAddS3Cmd.maxErrorRetry) : thatAddS3Cmd.maxErrorRetry != null) { + return false; + } + + if (this.secretKey != null ? !this.secretKey.equals(thatAddS3Cmd.secretKey) : thatAddS3Cmd.secretKey != null) { + return false; + } + + if (this.socketTimeout != null ? !this.socketTimeout.equals(thatAddS3Cmd.socketTimeout) : thatAddS3Cmd.socketTimeout != null) { + return false; + } + + if (this.bucketName != null ? !this.bucketName.equals(thatAddS3Cmd.bucketName) : thatAddS3Cmd.bucketName != null) { + return false; + } + + return true; + + } + + @Override + public int hashCode() { + + int result = this.accessKey != null ? this.accessKey.hashCode() : 0; + result = 31 * result + (this.secretKey != null ? this.secretKey.hashCode() : 0); + result = 31 * result + (this.endPoint != null ? this.endPoint.hashCode() : 0); + result = 31 * result + (this.bucketName != null ? this.bucketName.hashCode() : 0); + result = 31 * result + (this.httpsFlag != null && this.httpsFlag == true ? 1 : 0); + result = 31 * result + (this.connectionTimeout != null ? this.connectionTimeout.hashCode() : 0); + result = 31 * result + (this.maxErrorRetry != null ? this.maxErrorRetry.hashCode() : 0); + result = 31 * result + (this.socketTimeout != null ? this.socketTimeout.hashCode() : 0); + + return result; + + } + + @Override + public String getCommandName() { + return COMMAND_NAME; + } + + @Override + public long getEntityOwnerId() { + return ACCOUNT_ID_SYSTEM; + } + + public String getAccessKey() { + return this.accessKey; + } + + public String getSecretKey() { + return this.secretKey; + } + + public String getEndPoint() { + return this.endPoint; + } + + public String getBucketName() { + return this.bucketName; + } + + public Boolean getHttpsFlag() { + return this.httpsFlag; + } + + public Integer getConnectionTimeout() { + return this.connectionTimeout; + } + + public Integer getMaxErrorRetry() { + return this.maxErrorRetry; + } + + public Integer getSocketTimeout() { + return this.socketTimeout; + } + +} diff --git a/api/src/com/cloud/api/commands/ListS3sCmd.java b/api/src/com/cloud/api/commands/ListS3sCmd.java new file mode 100644 index 00000000000..507053dd89f --- /dev/null +++ b/api/src/com/cloud/api/commands/ListS3sCmd.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.api.commands; + +import static com.cloud.api.ApiConstants.ID; +import static com.cloud.api.BaseCmd.CommandType.LONG; + +import java.util.ArrayList; +import java.util.List; + +import com.cloud.api.BaseListCmd; +import com.cloud.api.Implementation; +import com.cloud.api.Parameter; +import com.cloud.api.ServerApiException; +import com.cloud.api.response.ListResponse; +import com.cloud.api.response.S3Response; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.storage.S3; + +@Implementation(description = "Lists S3s", responseObject = S3Response.class, since = "4.0.0") +public final class ListS3sCmd extends BaseListCmd { + + private static final String COMMAND_NAME = "lists3sresponse"; + + @Parameter(name = ID, type = LONG, required = true, description = "The ID of the S3") + private Long id; + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, + ServerApiException, ConcurrentOperationException, ResourceAllocationException, + NetworkRuleConflictException { + + final List result = _resourceService.listS3s(this); + final ListResponse response = new ListResponse(); + final List s3Responses = new ArrayList(); + + if (result != null) { + + for (S3 s3 : result) { + + S3Response s3Response = _responseGenerator.createS3Response(s3); + s3Response.setResponseName(this.getCommandName()); + s3Response.setObjectName("s3"); + s3Responses.add(s3Response); + + } + + } + + response.setResponses(s3Responses); + response.setResponseName(this.getCommandName()); + + this.setResponseObject(response); + + } + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null || getClass() != thatObject.getClass()) { + return false; + } + + final ListS3sCmd thatListS3sCmd = (ListS3sCmd) thatObject; + + if (this.id != null ? !this.id.equals(thatListS3sCmd.id) : thatListS3sCmd.id != null) { + return false; + } + + return true; + + } + + @Override + public int hashCode() { + return this.id != null ? this.id.hashCode() : 0; + } + + @Override + public String getCommandName() { + return COMMAND_NAME; + } + + public Long getId() { + return this.id; + } + +} diff --git a/api/src/com/cloud/api/response/S3Response.java b/api/src/com/cloud/api/response/S3Response.java new file mode 100644 index 00000000000..1efbe23b05e --- /dev/null +++ b/api/src/com/cloud/api/response/S3Response.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.api.response; + +import com.cloud.serializer.Param; +import com.cloud.utils.IdentityProxy; +import com.google.gson.annotations.SerializedName; + +import static com.cloud.api.ApiConstants.*; + +public class S3Response extends BaseResponse { + + @SerializedName(ID) + @Param(description = "The ID of the S3 configuration") + private IdentityProxy id = new IdentityProxy("s3"); + + @SerializedName(S3_ACCESS_KEY) + @Param(description = "The S3 access key") + private String accessKey; + + @SerializedName(S3_SECRET_KEY) + @Param(description = "The S3 secret key") + private String secretKey; + + @SerializedName(S3_END_POINT) + @Param(description = "The S3 end point") + private String endPoint; + + @SerializedName(S3_BUCKET_NAME) + @Param(description = "The name of the template storage bucket") + private String bucketName; + + @SerializedName(S3_HTTPS_FLAG) + @Param(description = "Connect to S3 using HTTPS?") + private Integer httpsFlag; + + @SerializedName(S3_CONNECTION_TIMEOUT) + @Param(description = "The connection timeout (milliseconds)") + private Integer connectionTimeout; + + @SerializedName(S3_MAX_ERROR_RETRY) + @Param(description = "The maximum number of time to retry a connection on error.") + private Integer maxErrorRetry; + + @SerializedName(S3_SOCKET_TIMEOUT) + @Param(description = "The connection socket (milliseconds)") + private Integer socketTimeout; + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null || this.getClass() != thatObject.getClass()) { + return false; + } + + final S3Response thatS3Response = (S3Response) thatObject; + + if (this.httpsFlag != null ? !this.httpsFlag.equals(thatS3Response.httpsFlag) : thatS3Response.httpsFlag != null) { + return false; + } + + if (this.accessKey != null ? !this.accessKey.equals(thatS3Response.accessKey) : thatS3Response.accessKey != null) { + return false; + } + + if (this.connectionTimeout != null ? !this.connectionTimeout.equals(thatS3Response.connectionTimeout) : thatS3Response.connectionTimeout != null) { + return false; + } + + if (this.endPoint != null ? !this.endPoint.equals(thatS3Response.endPoint) : thatS3Response.endPoint != null) { + return false; + } + + if (this.id != null ? !this.id.equals(thatS3Response.id) : thatS3Response.id != null) { + return false; + } + + if (this.maxErrorRetry != null ? !this.maxErrorRetry.equals(thatS3Response.maxErrorRetry) : thatS3Response.maxErrorRetry != null) { + return false; + } + + if (this.secretKey != null ? !this.secretKey.equals(thatS3Response.secretKey) : thatS3Response.secretKey != null) { + return false; + } + + if (this.socketTimeout != null ? !this.socketTimeout.equals(thatS3Response.socketTimeout) : thatS3Response.socketTimeout != null) { + return false; + } + + if (this.bucketName != null ? !this.bucketName.equals(thatS3Response.bucketName) : thatS3Response.bucketName != null) { + return false; + } + + return true; + + } + + @Override + public int hashCode() { + + int result = this.id != null ? this.id.hashCode() : 0; + result = 31 * result + (this.accessKey != null ? this.accessKey.hashCode() : 0); + result = 31 * result + (this.secretKey != null ? this.secretKey.hashCode() : 0); + result = 31 * result + (this.endPoint != null ? this.endPoint.hashCode() : 0); + result = 31 * result + (this.bucketName != null ? this.bucketName.hashCode() : 0); + result = 31 * result + (this.httpsFlag != null ? this.httpsFlag : 0); + result = 31 * result + (this.connectionTimeout != null ? this.connectionTimeout.hashCode() : 0); + result = 31 * result + (this.maxErrorRetry != null ? this.maxErrorRetry.hashCode() : 0); + result = 31 * result + (this.socketTimeout != null ? this.socketTimeout.hashCode() : 0); + + return result; + + } + + @Override + public Long getObjectId() { + return this.id.getValue(); + } + + public void setObjectId(Long id) { + this.id.setValue(id); + } + + public String getAccessKey() { + return this.accessKey; + } + + public void setAccessKey(final String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return this.secretKey; + } + + public void setSecretKey(final String secretKey) { + this.secretKey = secretKey; + } + + public String getEndPoint() { + return this.endPoint; + } + + public void setEndPoint(final String endPoint) { + this.endPoint = endPoint; + } + + + public String getTemplateBucketName() { + return this.bucketName; + } + + public void setTemplateBucketName(final String templateBucketName) { + this.bucketName = templateBucketName; + } + + public Integer getHttpsFlag() { + return this.httpsFlag; + } + + public void setHttpsFlag(final Integer httpsFlag) { + this.httpsFlag = httpsFlag; + } + + public Integer getConnectionTimeout() { + return this.connectionTimeout; + } + + public void setConnectionTimeout(final Integer connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + public Integer getMaxErrorRetry() { + return this.maxErrorRetry; + } + + public void setMaxErrorRetry(final Integer maxErrorRetry) { + this.maxErrorRetry = maxErrorRetry; + } + + public Integer getSocketTimeout() { + return this.socketTimeout; + } + + public void setSocketTimeout(final Integer socketTimeout) { + this.socketTimeout = socketTimeout; + } + +} diff --git a/api/src/com/cloud/resource/ResourceService.java b/api/src/com/cloud/resource/ResourceService.java index 10654539bb6..4cdb31a72ab 100755 --- a/api/src/com/cloud/resource/ResourceService.java +++ b/api/src/com/cloud/resource/ResourceService.java @@ -20,10 +20,12 @@ import java.util.List; import com.cloud.api.commands.AddClusterCmd; import com.cloud.api.commands.AddHostCmd; +import com.cloud.api.commands.AddS3Cmd; import com.cloud.api.commands.AddSecondaryStorageCmd; import com.cloud.api.commands.AddSwiftCmd; import com.cloud.api.commands.CancelMaintenanceCmd; import com.cloud.api.commands.DeleteClusterCmd; +import com.cloud.api.commands.ListS3sCmd; import com.cloud.api.commands.ListSwiftsCmd; import com.cloud.api.commands.PrepareForMaintenanceCmd; import com.cloud.api.commands.ReconnectHostCmd; @@ -35,6 +37,7 @@ import com.cloud.exception.ResourceInUseException; import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.org.Cluster; +import com.cloud.storage.S3; import com.cloud.storage.Swift; import com.cloud.utils.fsm.NoTransitionException; @@ -93,8 +96,13 @@ public interface ResourceService { Cluster getCluster(Long clusterId); Swift discoverSwift(AddSwiftCmd addSwiftCmd) throws DiscoveryException; + + S3 discoverS3(AddS3Cmd cmd) throws DiscoveryException; List getSupportedHypervisorTypes(long zoneId, boolean forVirtualRouter, Long podId); List listSwifts(ListSwiftsCmd cmd); + + List listS3s(ListS3sCmd cmd); + } diff --git a/api/src/com/cloud/storage/S3.java b/api/src/com/cloud/storage/S3.java new file mode 100644 index 00000000000..7a679f2adc5 --- /dev/null +++ b/api/src/com/cloud/storage/S3.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage; + +import com.cloud.agent.api.to.S3TO; + +import java.util.Date; + +public interface S3 { + + long getId(); + + String getUuid(); + + String getAccessKey(); + + String getSecretKey(); + + String getEndPoint(); + + String getBucketName(); + + Integer getHttpsFlag(); + + Integer getConnectionTimeout(); + + Integer getMaxErrorRetry(); + + Integer getSocketTimeout(); + + Date getCreated(); + + S3TO toS3TO(); + +} diff --git a/client/WEB-INF/classes/resources/messages.properties b/client/WEB-INF/classes/resources/messages.properties index 48514e8f29d..e0dce474fc5 100644 --- a/client/WEB-INF/classes/resources/messages.properties +++ b/client/WEB-INF/classes/resources/messages.properties @@ -32,6 +32,17 @@ label.destroy=Destroy label.restore=Restore label.isolation.uri=Isolation URI label.broadcast.uri=Broadcast URI +label.enable.s3=Enable S3-backed Secondary Storage +confirm.enable.s3=Please fill in the following information to enable support for S3-backed Secondary Storage +message.after.enable.s3=S3-backed Secondary Storage configured. Note: When you leave this page, you will not be able to re-configure S3 again. +label.s3.access_key=Access Key +label.s3.secret_key=Secret Key +label.s3.bucket=Bucket +label.s3.endpoint=Endpoint +label.s3.use_https=Use HTTPS +label.s3.connection_timeout=Connection Timeout +label.s3.max_error_retry=Max Error Retry +label.s3.socket_timeout=Socket Timeout #new labels (end) ************************************************************************************************ diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in index 149547ee575..437c8d458f5 100755 --- a/client/tomcatconf/commands.properties.in +++ b/client/tomcatconf/commands.properties.in @@ -227,6 +227,9 @@ listCapacity=com.cloud.api.commands.ListCapacityCmd;3 addSwift=com.cloud.api.commands.AddSwiftCmd;1 listSwifts=com.cloud.api.commands.ListSwiftsCmd;1 +#### s3 commands +addS3=com.cloud.api.commands.AddS3Cmd;1 +listS3s=com.cloud.api.commands.ListS3sCmd;1 #### host commands addHost=com.cloud.api.commands.AddHostCmd;3 diff --git a/core/pom.xml b/core/pom.xml index 15f0f7b7302..3d6356e561e 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -43,6 +43,11 @@ + + commons-codec + commons-codec + ${cs.codec.version} + install diff --git a/core/src/com/cloud/storage/S3VO.java b/core/src/com/cloud/storage/S3VO.java new file mode 100644 index 00000000000..89f233b87fe --- /dev/null +++ b/core/src/com/cloud/storage/S3VO.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage; + +import com.cloud.agent.api.to.S3TO; +import com.cloud.api.Identity; +import com.cloud.utils.db.GenericDao; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.Date; + +@Entity +@Table(name = "s3") +public class S3VO implements S3, Identity { + + public static final String ID_COLUMN_NAME = "id"; + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = ID_COLUMN_NAME) + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "access_key") + private String accessKey; + + @Column(name = "secret_key") + private String secretKey; + + @Column(name = "end_point") + private String endPoint; + + @Column(name = "bucket") + private String bucketName; + + @Column(name = "https") + private Integer httpsFlag; + + @Column(name = "connection_timeout") + private Integer connectionTimeout; + + @Column(name = "max_error_retry") + private Integer maxErrorRetry; + + @Column(name = "socket_timeout") + private Integer socketTimeout; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + public S3VO() { + super(); + } + + public S3VO(final String uuid, final String accessKey, + final String secretKey, final String endPoint, + final String bucketName, final Boolean httpsFlag, + final Integer connectionTimeout, final Integer maxErrorRetry, + final Integer socketTimeout, final Date created) { + + super(); + + this.uuid = uuid; + this.accessKey = accessKey; + this.secretKey = secretKey; + this.endPoint = endPoint; + this.bucketName = bucketName; + + Integer value = null; + if (httpsFlag != null) { + value = httpsFlag == false ? 0 : 1; + } + this.httpsFlag = value; + + this.connectionTimeout = connectionTimeout; + this.maxErrorRetry = maxErrorRetry; + this.socketTimeout = socketTimeout; + this.created = created; + + } + + @Override + public S3TO toS3TO() { + + Boolean httpsFlag = null; + if (this.httpsFlag != null) { + httpsFlag = this.httpsFlag == 0 ? false : true; + } + + return new S3TO(this.id, this.uuid, this.accessKey, this.secretKey, + this.endPoint, this.bucketName, httpsFlag, + this.connectionTimeout, this.maxErrorRetry, this.socketTimeout, + this.created); + + } + + public long getId() { + return this.id; + } + + public void setId(final long id) { + this.id = id; + } + + public String getUuid() { + return this.uuid; + } + + public void setUuid(final String uuid) { + this.uuid = uuid; + } + + public String getAccessKey() { + return this.accessKey; + } + + public void setAccessKey(final String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return this.secretKey; + } + + public void setSecretKey(final String secretKey) { + this.secretKey = secretKey; + } + + public String getEndPoint() { + return this.endPoint; + } + + public void setEndPoint(final String endPoint) { + this.endPoint = endPoint; + } + + public String getBucketName() { + return this.bucketName; + } + + public void setBucketName(final String bucketName) { + this.bucketName = bucketName; + } + + public Integer getHttpsFlag() { + return this.httpsFlag; + } + + public void setHttpsFlag(final Integer httpsFlag) { + this.httpsFlag = httpsFlag; + } + + public Integer getConnectionTimeout() { + return this.connectionTimeout; + } + + public void setConnectionTimeout(final int connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + public Integer getMaxErrorRetry() { + return this.maxErrorRetry; + } + + public void setMaxErrorRetry(final int maxErrorRetry) { + this.maxErrorRetry = maxErrorRetry; + } + + public Integer getSocketTimeout() { + return this.socketTimeout; + } + + public void setSocketTimeout(final int socketTimeout) { + this.socketTimeout = socketTimeout; + } + + public Date getCreated() { + return this.created; + } + + public void setCreated(final Date created) { + this.created = created; + } + +} diff --git a/core/src/com/cloud/storage/SnapshotVO.java b/core/src/com/cloud/storage/SnapshotVO.java index 08dfafa6bac..ae4fd6aafef 100644 --- a/core/src/com/cloud/storage/SnapshotVO.java +++ b/core/src/com/cloud/storage/SnapshotVO.java @@ -91,6 +91,9 @@ public class SnapshotVO implements Snapshot, Identity { @Column(name="swift_id") Long swiftId; + @Column(name="s3_id") + Long s3Id; + @Column(name="sechost_id") Long secHostId; @@ -289,4 +292,13 @@ public class SnapshotVO implements Snapshot, Identity { public void setUuid(String uuid) { this.uuid = uuid; } + + public Long getS3Id() { + return s3Id; + } + + public void setS3Id(Long s3Id) { + this.s3Id = s3Id; + } + } diff --git a/core/src/com/cloud/storage/VMTemplateS3VO.java b/core/src/com/cloud/storage/VMTemplateS3VO.java new file mode 100644 index 00000000000..a75c37d192d --- /dev/null +++ b/core/src/com/cloud/storage/VMTemplateS3VO.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage; + +import com.cloud.utils.db.GenericDaoBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import java.text.DateFormat; +import java.util.Date; + +@Entity +@Table(name = "template_s3_ref") +public class VMTemplateS3VO { + + public static final String S3_ID_COLUMN_NAME = "s3_id"; + + public static final String TEMPLATE_ID_COLUMN_NAME = "template_id"; + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private long id; + + @Column(name = S3_ID_COLUMN_NAME) + private long s3Id; + + @Column(name = TEMPLATE_ID_COLUMN_NAME) + private long templateId; + + @Column(name = GenericDaoBase.CREATED_COLUMN) + private Date created; + + @Column(name = "size") + private Long size; + + @Column(name = "physical_size") + private Long physicalSize; + + public VMTemplateS3VO() { + super(); + } + + public VMTemplateS3VO(final long s3Id, final long templateId, + final Date created, final Long size, final Long physicalSize) { + + super(); + + this.s3Id = s3Id; + this.templateId = templateId; + this.created = created; + this.size = size; + this.physicalSize = physicalSize; + + } + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null || getClass() != thatObject.getClass()) { + return false; + } + + final VMTemplateS3VO thatVMTemplateS3VO = (VMTemplateS3VO) thatObject; + + if (this.id != thatVMTemplateS3VO.id) { + return false; + } + + if (this.s3Id != thatVMTemplateS3VO.s3Id) { + return false; + } + + if (this.templateId != thatVMTemplateS3VO.templateId) { + return false; + } + + if (this.created != null ? !created.equals(thatVMTemplateS3VO.created) + : thatVMTemplateS3VO.created != null) { + return false; + } + + if (this.physicalSize != null ? !physicalSize + .equals(thatVMTemplateS3VO.physicalSize) + : thatVMTemplateS3VO.physicalSize != null) { + return false; + } + + if (this.size != null ? !size.equals(thatVMTemplateS3VO.size) + : thatVMTemplateS3VO.size != null) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + + int result = (int) (this.id ^ (this.id >>> 32)); + + result = 31 * result + (int) (this.s3Id ^ (this.s3Id >>> 32)); + result = 31 * result + + (int) (this.templateId ^ (this.templateId >>> 32)); + result = 31 * result + + (this.created != null ? this.created.hashCode() : 0); + result = 31 * result + (this.size != null ? this.size.hashCode() : 0); + result = 31 + * result + + (this.physicalSize != null ? this.physicalSize.hashCode() : 0); + + return result; + + } + + public long getId() { + return this.id; + } + + public void setId(final long id) { + this.id = id; + } + + public long getS3Id() { + return this.s3Id; + } + + public void setS3Id(final long s3Id) { + this.s3Id = s3Id; + } + + public long getTemplateId() { + return this.templateId; + } + + public void setTemplateId(final long templateId) { + this.templateId = templateId; + } + + public Date getCreated() { + return this.created; + } + + public void setCreated(final Date created) { + this.created = created; + } + + public Long getSize() { + return this.size; + } + + public void setSize(final Long size) { + this.size = size; + } + + public Long getPhysicalSize() { + return this.physicalSize; + } + + public void setPhysicalSize(final Long physicalSize) { + this.physicalSize = physicalSize; + } + + @Override + public String toString() { + + final StringBuilder stringBuilder = new StringBuilder( + "VMTemplateS3VO [ id: ").append(id).append(", created: ") + .append(DateFormat.getDateTimeInstance().format(created)) + .append(", physicalSize: ").append(physicalSize) + .append(", size: ").append(size).append(", templateId: ") + .append(templateId).append(", s3Id: ").append(s3Id) + .append(" ]"); + + return stringBuilder.toString(); + + } + +} diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java index 155210df499..d8fdc3a71d6 100755 --- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java +++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java @@ -16,10 +16,20 @@ // under the License. package com.cloud.storage.resource; +import static com.cloud.utils.S3Utils.deleteDirectory; +import static com.cloud.utils.S3Utils.getDirectory; +import static com.cloud.utils.S3Utils.putDirectory; +import static com.cloud.utils.StringUtils.join; +import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock; +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static org.apache.commons.lang.StringUtils.substringAfterLast; + import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileWriter; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; @@ -32,6 +42,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.Callable; import javax.naming.ConfigurationException; @@ -46,6 +57,9 @@ import com.cloud.agent.api.ComputeChecksumCommand; import com.cloud.agent.api.DeleteObjectFromSwiftCommand; import com.cloud.agent.api.DeleteSnapshotBackupCommand; import com.cloud.agent.api.DeleteSnapshotsDirCommand; +import com.cloud.agent.api.DeleteTemplateFromS3Command; +import com.cloud.agent.api.DownloadSnapshotFromS3Command; +import com.cloud.agent.api.DownloadTemplateFromS3ToSecondaryStorageCommand; import com.cloud.agent.api.GetStorageStatsAnswer; import com.cloud.agent.api.GetStorageStatsCommand; import com.cloud.agent.api.PingCommand; @@ -60,6 +74,8 @@ import com.cloud.agent.api.SecStorageSetupCommand.Certificates; import com.cloud.agent.api.StartupSecondaryStorageCommand; import com.cloud.agent.api.SecStorageVMSetupCommand; import com.cloud.agent.api.StartupCommand; +import com.cloud.agent.api.StartupSecondaryStorageCommand; +import com.cloud.agent.api.UploadTemplateToS3FromSecondaryStorageCommand; import com.cloud.agent.api.downloadSnapshotFromSwiftCommand; import com.cloud.agent.api.downloadTemplateFromSwiftToSecondaryStorageCommand; import com.cloud.agent.api.uploadTemplateToSwiftFromSecondaryStorageCommand; @@ -75,6 +91,7 @@ import com.cloud.agent.api.storage.ListVolumeAnswer; import com.cloud.agent.api.storage.ListVolumeCommand; import com.cloud.agent.api.storage.UploadCommand; import com.cloud.agent.api.storage.ssCommand; +import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.api.commands.DeleteVolumeCmd; import com.cloud.exception.InternalErrorException; @@ -90,6 +107,9 @@ import com.cloud.storage.template.TemplateLocation; import com.cloud.storage.template.UploadManager; import com.cloud.storage.template.UploadManagerImpl; import com.cloud.utils.NumbersUtil; +import com.cloud.utils.S3Utils; +import com.cloud.utils.S3Utils.FileNamingStrategy; +import com.cloud.utils.S3Utils.ObjectNamingStrategy; import com.cloud.utils.component.ComponentLocator; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @@ -97,8 +117,15 @@ import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; import com.cloud.vm.SecondaryStorageVm; -public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource { - private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); +public class NfsSecondaryStorageResource extends ServerResourceBase implements + SecondaryStorageResource { + + private static final Logger s_logger = Logger + .getLogger(NfsSecondaryStorageResource.class); + + private static final String TEMPLATE_ROOT_DIR = "template/tmpl"; + private static final String SNAPSHOT_ROOT_DIR = "snapshots"; + int _timeout; String _instance; @@ -168,16 +195,24 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return execute((ListVolumeCommand)cmd); }else if (cmd instanceof downloadSnapshotFromSwiftCommand){ return execute((downloadSnapshotFromSwiftCommand)cmd); + } else if (cmd instanceof DownloadSnapshotFromS3Command) { + return execute((DownloadSnapshotFromS3Command) cmd); } else if (cmd instanceof DeleteSnapshotBackupCommand){ return execute((DeleteSnapshotBackupCommand)cmd); } else if (cmd instanceof DeleteSnapshotsDirCommand){ return execute((DeleteSnapshotsDirCommand)cmd); } else if (cmd instanceof downloadTemplateFromSwiftToSecondaryStorageCommand) { return execute((downloadTemplateFromSwiftToSecondaryStorageCommand) cmd); + } else if (cmd instanceof DownloadTemplateFromS3ToSecondaryStorageCommand) { + return execute((DownloadTemplateFromS3ToSecondaryStorageCommand) cmd); } else if (cmd instanceof uploadTemplateToSwiftFromSecondaryStorageCommand) { return execute((uploadTemplateToSwiftFromSecondaryStorageCommand) cmd); + } else if (cmd instanceof UploadTemplateToS3FromSecondaryStorageCommand) { + return execute((UploadTemplateToS3FromSecondaryStorageCommand) cmd); } else if (cmd instanceof DeleteObjectFromSwiftCommand) { return execute((DeleteObjectFromSwiftCommand) cmd); + } else if (cmd instanceof DeleteTemplateFromS3Command) { + return execute((DeleteTemplateFromS3Command) cmd); } else if (cmd instanceof CleanupSnapshotBackupCommand){ return execute((CleanupSnapshotBackupCommand)cmd); } else { @@ -185,6 +220,69 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + @SuppressWarnings("unchecked") + private String determineS3TemplateDirectory(final Long accountId, + final Long templateId) { + return join(asList(TEMPLATE_ROOT_DIR, accountId, templateId), + S3Utils.SEPARATOR); + } + + @SuppressWarnings("unchecked") + private String determineStorageTemplatePath(final String storagePath, + final Long accountId, final Long templateId) { + return join( + asList(getRootDir(storagePath), TEMPLATE_ROOT_DIR, accountId, + templateId), File.separator); + } + + private Answer execute( + final DownloadTemplateFromS3ToSecondaryStorageCommand cmd) { + + final S3TO s3 = cmd.getS3(); + final String storagePath = cmd.getStoragePath(); + final Long accountId = cmd.getAccountId(); + final Long templateId = cmd.getTemplateId(); + + try { + + final File downloadDirectory = _storage + .getFile(determineStorageTemplatePath(storagePath, + accountId, templateId)); + downloadDirectory.mkdirs(); + + if (!downloadDirectory.exists()) { + final String errMsg = format( + "Unable to create directory " + + "download directory %1$s for download of template id " + + "%2$s from S3.", downloadDirectory.getName(), + templateId); + s_logger.error(errMsg); + return new Answer(cmd, false, errMsg); + } + + getDirectory(s3, s3.getBucketName(), + determineS3TemplateDirectory(accountId, templateId), + downloadDirectory, new FileNamingStrategy() { + @Override + public String determineFileName(final String key) { + return substringAfterLast(key, S3Utils.SEPARATOR); + } + }); + + return new Answer(cmd, true, format("Successfully downloaded " + + "template id %1$s from S3 to directory %2$s", templateId, + downloadDirectory.getName())); + + } catch (Exception e) { + + final String errMsg = format("Failed to upload template id %1$s " + + "due to $2%s", templateId, e.getMessage()); + s_logger.error(errMsg, e); + return new Answer(cmd, false, errMsg); + + } + + } private Answer execute(downloadTemplateFromSwiftToSecondaryStorageCommand cmd) { SwiftTO swift = cmd.getSwift(); @@ -256,6 +354,83 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + private Answer execute(UploadTemplateToS3FromSecondaryStorageCommand cmd) { + + final S3TO s3 = cmd.getS3(); + final Long accountId = cmd.getAccountId(); + final Long templateId = cmd.getTemplateId(); + + try { + + final String templatePath = determineStorageTemplatePath( + cmd.getStoragePath(), accountId, templateId); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Found template id " + templateId + + " account id " + accountId + " from directory " + + templatePath + " to upload to S3."); + } + + if (!_storage.isDirectory(templatePath)) { + final String errMsg = format("S3 Sync Failure: Directory %1$s" + + "for template id %2$s does not exist.", templatePath, + templateId); + s_logger.error(errMsg); + return new Answer(cmd, false, errMsg); + } + + if (!_storage.isFile(templatePath + "/template.properties")) { + final String errMsg = format("S3 Sync Failure: Template id " + + "%1$s does not exist on the file system.", + templatePath); + s_logger.error(errMsg); + return new Answer(cmd, false, errMsg); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug(format( + "Pushing template id %1$s from %2$s to S3...", + templateId, templatePath)); + } + + final String bucket = s3.getBucketName(); + putDirectory(s3, bucket, _storage.getFile(templatePath), + new FilenameFilter() { + @Override + public boolean accept(final File directory, + final String fileName) { + return !fileName.startsWith("."); + } + }, new ObjectNamingStrategy() { + @Override + public String determineKey(final File file) { + s_logger.debug(String + .format("Determining key using account id %1$s and template id %2$s", + accountId, templateId)); + return join( + asList(determineS3TemplateDirectory( + accountId, templateId), file + .getName()), S3Utils.SEPARATOR); + } + }); + + return new Answer( + cmd, + true, + format("Uploaded the contents of directory %1$s for template id %2$s to S3 bucket %3$s", + templatePath, templateId, bucket)); + + } catch (Exception e) { + + final String errMsg = format("Failed to upload template id %1$s", + templateId); + s_logger.error(errMsg, e); + return new Answer(cmd, false, errMsg); + + } + + } + private Answer execute(DeleteObjectFromSwiftCommand cmd) { SwiftTO swift = cmd.getSwift(); String container = cmd.getContainer(); @@ -279,6 +454,47 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } + private Answer execute(final DeleteTemplateFromS3Command cmd) { + + final S3TO s3 = cmd.getS3(); + final Long accountId = cmd.getAccountId(); + final Long templateId = cmd.getTemplateId(); + + if (accountId == null || (accountId != null && accountId <= 0)) { + final String errorMessage = "No account id specified for S3 template deletion."; + s_logger.error(errorMessage); + return new Answer(cmd, false, errorMessage); + } + + if (templateId == null || (templateId != null && templateId <= 0)) { + final String errorMessage = "No template id specified for S3 template deletion."; + s_logger.error(errorMessage); + return new Answer(cmd, false, errorMessage); + } + + if (s3 == null) { + final String errorMessge = "No S3 client options provided"; + s_logger.error(errorMessge); + return new Answer(cmd, false, errorMessge); + } + + final String bucket = s3.getBucketName(); + try { + deleteDirectory(s3, bucket, + determineS3TemplateDirectory(templateId, accountId)); + return new Answer(cmd, true, String.format( + "Deleted template %1%s from bucket %2$s.", templateId, + bucket)); + } catch (Exception e) { + final String errorMessage = String + .format("Failed to delete templaet id %1$s from bucket %2$s due to the following error: %3$s", + templateId, bucket, e.getMessage()); + s_logger.error(errorMessage, e); + return new Answer(cmd, false, errorMessage); + } + + } + String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) { Script command = new Script("/bin/bash", s_logger); command.add("-c"); @@ -451,6 +667,110 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + public Answer execute(final DownloadSnapshotFromS3Command cmd) { + + final S3TO s3 = cmd.getS3(); + final String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); + final Long accountId = cmd.getAccountId(); + final Long volumeId = cmd.getVolumeId(); + + try { + + executeWithNoWaitLock(determineSnapshotLockId(accountId, volumeId), + new Callable() { + + @Override + public Void call() throws Exception { + + final String directoryName = determineSnapshotLocalDirectory( + secondaryStorageUrl, accountId, volumeId); + + String result = createLocalDir(directoryName); + if (result != null) { + throw new InternalErrorException( + format("Failed to create directory %1$s during S3 snapshot download.", + directoryName)); + } + + final String snapshotFileName = determineSnapshotBackupFilename(cmd + .getSnapshotUuid()); + final String key = determineSnapshotS3Key( + accountId, volumeId, snapshotFileName); + final File targetFile = S3Utils.getFile(s3, + s3.getBucketName(), key, + _storage.getFile(directoryName), + new FileNamingStrategy() { + + @Override + public String determineFileName( + String key) { + return snapshotFileName; + } + + }); + + if (cmd.getParent() != null) { + + final String parentPath = join( + File.pathSeparator, directoryName, + determineSnapshotBackupFilename(cmd + .getParent())); + result = setVhdParent( + targetFile.getAbsolutePath(), + parentPath); + if (result != null) { + throw new InternalErrorException( + format("Failed to set the parent for backup %1$s to %2$s due to %3$s.", + targetFile + .getAbsolutePath(), + parentPath, result)); + } + + } + + return null; + + } + + }); + + return new Answer( + cmd, + true, + format("Succesfully retrieved volume id %1$s for account id %2$s to %3$s from S3.", + volumeId, accountId, secondaryStorageUrl)); + + } catch (Exception e) { + final String errMsg = format( + "Failed to retrieve volume id %1$s for account id %2$s to %3$s from S3 due to exception %4$s", + volumeId, accountId, secondaryStorageUrl, e.getMessage()); + s_logger.error(errMsg); + return new Answer(cmd, false, errMsg); + } + + } + + private String determineSnapshotS3Directory(final Long accountId, + final Long volumeId) { + return join(S3Utils.SEPARATOR, SNAPSHOT_ROOT_DIR, accountId, volumeId); + } + + private String determineSnapshotS3Key(final Long accountId, + final Long volumeId, final String snapshotFileName) { + + final String directoryName = determineSnapshotS3Directory(accountId, + volumeId); + return join(S3Utils.SEPARATOR, directoryName, snapshotFileName); + + } + + private String determineSnapshotLocalDirectory( + final String secondaryStorageUrl, final Long accountId, + final Long volumeId) { + return join(File.pathSeparator, getRootDir(secondaryStorageUrl), + SNAPSHOT_ROOT_DIR, accountId, volumeId); + } + public Answer execute(downloadSnapshotFromSwiftCommand cmd){ SwiftTO swift = cmd.getSwift(); String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); @@ -622,6 +942,92 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + private String deleteSnapshotBackupFromLocalFileSystem( + final String secondaryStorageUrl, final Long accountId, + final Long volumeId, final String name, final Boolean deleteAllFlag) { + + final String lPath = determineSnapshotLocalDirectory( + secondaryStorageUrl, accountId, volumeId) + + File.pathSeparator + + (deleteAllFlag ? "*" : "*" + name + "*"); + + final String result = deleteLocalFile(lPath); + + if (result != null) { + return "failed to delete snapshot " + lPath + " , err=" + result; + } + + return null; + + } + + private String deleteSnapshotBackupfromS3(final S3TO s3, + final String secondaryStorageUrl, final Long accountId, + final Long volumeId, final String name, final Boolean deleteAllFlag) { + + try { + + final String bucket = s3.getBucketName(); + + final String result = executeWithNoWaitLock( + determineSnapshotLockId(accountId, volumeId), + new Callable() { + + @Override + public String call() throws Exception { + + final String innerResult = deleteSnapshotBackupFromLocalFileSystem( + secondaryStorageUrl, accountId, volumeId, + name, deleteAllFlag); + if (innerResult != null) { + return innerResult; + } + + if (deleteAllFlag) { + S3Utils.deleteDirectory( + s3, + bucket, + determineSnapshotS3Directory(accountId, + volumeId)); + } else { + S3Utils.deleteObject( + s3, + bucket, + determineSnapshotS3Key( + accountId, + volumeId, + determineSnapshotBackupFilename(name))); + } + + return null; + + } + + }); + + return result; + + } catch (Exception e) { + + s_logger.error( + String.format( + "Failed to delete snapshot backup for account id %1$s volume id %2$sfrom S3.", + accountId, volumeId), e); + return e.getMessage(); + + } + + } + + private String determineSnapshotBackupFilename(final String snapshotUuid) { + return snapshotUuid + ".vhd"; + } + + private String determineSnapshotLockId(final Long accountId, + final Long volumeId) { + return join("_", "SNAPSHOT", accountId, volumeId); + } + protected Answer execute(final DeleteSnapshotBackupCommand cmd) { String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); Long accountId = cmd.getAccountId(); @@ -629,21 +1035,22 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String name = cmd.getSnapshotUuid(); try { SwiftTO swift = cmd.getSwift(); + S3TO s3 = cmd.getS3(); if (swift == null) { - String parent = getRootDir(secondaryStorageUrl); - String filename; - if (cmd.isAll()) { - filename = "*"; - - } else { - filename = "*" + name + "*"; - } - String lPath = parent + "/snapshots/" + String.valueOf(accountId) + "/" + String.valueOf(volumeId) + "/" + filename; - String result = deleteLocalFile(lPath); + final String result = deleteSnapshotBackupFromLocalFileSystem( + secondaryStorageUrl, accountId, volumeId, name, + cmd.isAll()); if (result != null) { - String errMsg = "failed to delete snapshot " + lPath + " , err=" + result; - s_logger.warn(errMsg); - return new Answer(cmd, false, errMsg); + s_logger.warn(result); + return new Answer(cmd, false, result); + } + } else if (s3 != null) { + final String result = deleteSnapshotBackupfromS3(s3, + secondaryStorageUrl, accountId, volumeId, name, + cmd.isAll()); + if (result != null) { + s_logger.warn(result); + return new Answer(cmd, false, result); } } else { String filename; diff --git a/patches/systemvm/debian/config/etc/sysctl.conf b/patches/systemvm/debian/config/etc/sysctl.conf index 7f945b0d4b1..961d471dfa9 100644 --- a/patches/systemvm/debian/config/etc/sysctl.conf +++ b/patches/systemvm/debian/config/etc/sysctl.conf @@ -8,7 +8,7 @@ net.ipv4.ip_forward = 1 # Controls source route verification -net.ipv4.conf.default.rp_filter = 1 +net.ipv4.conf.default.rp_filter = 0 # Do not accept source routing net.ipv4.conf.default.accept_source_route = 0 diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index 967a2993f2e..313703c0a82 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -17,6 +17,10 @@ package com.cloud.hypervisor.xen.resource; +import java.beans.BeanInfo; +import java.beans.IntrospectionException; +import java.beans.Introspector; +import java.beans.PropertyDescriptor; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -24,11 +28,13 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.StringReader; +import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -179,6 +185,7 @@ import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; import com.cloud.agent.api.to.IpAddressTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.PortForwardingRuleTO; +import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.StaticNatRuleTO; import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.agent.api.to.SwiftTO; @@ -217,6 +224,8 @@ import com.cloud.storage.template.TemplateInfo; import com.cloud.template.VirtualMachineTemplate.BootloaderType; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; +import com.cloud.utils.S3Utils; +import com.cloud.utils.StringUtils; import com.cloud.utils.Ternary; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @@ -6507,7 +6516,14 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe } finally { deleteSnapshotBackup(conn, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotBackupUuid); } - } + } else if (cmd.getS3() != null) { + try { + backupSnapshotToS3(conn, cmd.getS3(), snapshotSr.getUuid(conn), snapshotBackupUuid, isISCSI, wait); + snapshotBackupUuid = snapshotBackupUuid + ".vhd"; + } finally { + deleteSnapshotBackup(conn, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotBackupUuid); + } + } success = true; } finally { if( snapshotSr != null) { @@ -6524,6 +6540,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe snapshotBackupUuid = snapshotPaUuid + ".vhd"; } success = true; + } else if (cmd.getS3() != null) { + backupSnapshotToS3(conn, cmd.getS3(), primaryStorageSRUuid, snapshotPaUuid, isISCSI, wait); } else { snapshotBackupUuid = backupSnapshot(conn, primaryStorageSRUuid, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotUuid, prevBackupUuid, isISCSI, wait); success = (snapshotBackupUuid != null); @@ -6546,6 +6564,88 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe return new BackupSnapshotAnswer(cmd, success, details, snapshotBackupUuid, fullbackup); } + private static List serializeProperties(final Object object, + final Class propertySet) { + + assert object != null; + assert propertySet != null; + assert propertySet.isAssignableFrom(object.getClass()); + + try { + + final BeanInfo beanInfo = Introspector.getBeanInfo(propertySet); + final PropertyDescriptor[] descriptors = beanInfo + .getPropertyDescriptors(); + + final List serializedProperties = new ArrayList(); + for (final PropertyDescriptor descriptor : descriptors) { + + serializedProperties.add(descriptor.getName()); + final Object value = descriptor.getReadMethod().invoke(object); + serializedProperties.add(value != null ? value.toString() + : "null"); + + } + + return Collections.unmodifiableList(serializedProperties); + + } catch (IntrospectionException e) { + s_logger.warn( + "Ignored IntrospectionException when serializing class " + + object.getClass().getCanonicalName(), e); + } catch (IllegalArgumentException e) { + s_logger.warn( + "Ignored IllegalArgumentException when serializing class " + + object.getClass().getCanonicalName(), e); + } catch (IllegalAccessException e) { + s_logger.warn( + "Ignored IllegalAccessException when serializing class " + + object.getClass().getCanonicalName(), e); + } catch (InvocationTargetException e) { + s_logger.warn( + "Ignored InvocationTargetException when serializing class " + + object.getClass().getCanonicalName(), e); + } + + return Collections.emptyList(); + + } + + private boolean backupSnapshotToS3(final Connection connection, + final S3TO s3, final String srUuid, final String snapshotUuid, + final Boolean iSCSIFlag, final int wait) { + + final String filename = iSCSIFlag ? "VHD-" + snapshotUuid + : snapshotUuid + ".vhd"; + final String dir = (iSCSIFlag ? "/dev/VG_XenStorage-" + : "/var/run/sr-mount/") + srUuid; + final String key = StringUtils.join("/", "snapshots", snapshotUuid); + + try { + + final List parameters = new ArrayList( + serializeProperties(s3, S3Utils.ClientOptions.class)); + parameters.addAll(Arrays.asList("operation", "put", "directory", + dir, "filename", filename, "iSCSIFlag", + iSCSIFlag.toString(), "key", key)); + final String result = callHostPluginAsync(connection, "s3xen", + "s3", wait, + parameters.toArray(new String[parameters.size()])); + + if (result != null && result.equals("true")) { + return true; + } + + } catch (Exception e) { + s_logger.error(String.format( + "S3 upload failed of snapshot %1$s due to %2$s.", + snapshotUuid, e.toString()), e); + } + + return false; + + } + protected CreateVolumeFromSnapshotAnswer execute(final CreateVolumeFromSnapshotCommand cmd) { Connection conn = getConnection(); String primaryStorageNameLabel = cmd.getPrimaryStoragePoolNameLabel(); diff --git a/pom.xml b/pom.xml index 2c2600b552f..e5aa024b5ac 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ 1.7.1 5.6.100-1-SNAPSHOT 3.1 - 4.0 + 4.1 5.1.21 1.3.1 3.1.3 @@ -82,7 +82,9 @@ 1.0-20081010.060147 4.1 1.9.5 - + 1.3.21.1 + 2.6 + 1.4 diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen new file mode 100644 index 00000000000..4d9c12d6a5a --- /dev/null +++ b/scripts/vm/hypervisor/xenserver/s3xen @@ -0,0 +1,297 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Version @VERSION@ +# +# A plugin for executing script needed by cloud stack +from __future__ import with_statement + +from copy import copy +from datetime import datetime +from httplib import * +from string import join + +import os +import sys +import time +import hashlib +import base64 +import hmac +import traceback +import urllib2 + +import XenAPIPlugin +sys.path.extend(["/opt/xensource/sm/"]) +import util + +NULL = 'null' + +# Value conversion utility functions ... + + +def to_none(value): + return value if value is not None and value.strip() != NULL else None + + +def to_bool(value): + return True if to_none(value) in ['true', 'True', None] else False + + +def to_integer(value, default): + return int(value) if to_none(value) is not None else default + + +def optional_str_value(value, default): + return value if is_not_blank(value) else default + + +def is_not_blank(value): + return True if to_none(value) is not None and value.strip != '' else False + + +def get_optional_key(map, key, default=''): + return map[key] if key in map else default + + +def log(message): + util.SMlog('#### VMOPS %s ####' % message) + + +def echo(fn): + def wrapped(*v, **k): + name = fn.__name__ + log("enter %s ####" % name) + res = fn(*v, **k) + log("exit %s with result %s" % name, res) + return res + return wrapped + + +def require_str_value(value, error_message): + + if is_not_blank(value): + return value + + raise ValueError(error_message) + + +def retry(max_attempts, fn): + + attempts = 1 + while attempts <= max_attempts: + log("Attempting execution {0}/{1} of {2}". + format(attempts, max_attempts, fn.__name__)) + try: + return fn() + except: + if (attempts >= max_attempts): + raise + attempts = attempts + 1 + + +def compute_md5(filename, buffer_size=8192): + + hasher = hashlib.md5() + + with open(filename, 'rb') as file: + data = file.read(buffer_size) + while data != "": + hasher.update(data) + data = file.read(buffer_size) + + return base64.encodestring(hasher.digest())[:-1] + + +class S3Client(object): + + DEFAULT_END_POINT = 's3.amazonaws.com' + DEFAULT_CONNECTION_TIMEOUT = 50000 + DEFAULT_SOCKET_TIMEOUT = 50000 + DEFAULT_MAX_ERROR_RETRY = 3 + + HEADER_CONTENT_MD5 = 'Content-MD5' + HEADER_CONTENT_TYPE = 'Content-Type' + HEADER_CONTENT_LENGTH = 'Content-Length' + + def __init__(self, access_key, secret_key, end_point=None, + https_flag=None, connection_timeout=None, socket_timeout=None, + max_error_retry=None): + + self.access_key = require_str_value( + access_key, 'An access key must be specified.') + self.secret_key = require_str_value( + secret_key, 'A secret key must be specified.') + self.end_point = optional_str_value(end_point, self.DEFAULT_END_POINT) + self.https_flag = to_bool(https_flag) + self.connection_timeout = to_integer( + connection_timeout, self.DEFAULT_CONNECTION_TIMEOUT) + self.socket_timeout = to_integer( + socket_timeout, self.DEFAULT_SOCKET_TIMEOUT) + self.max_error_retry = to_integer( + max_error_retry, self.DEFAULT_MAX_ERROR_RETRY) + + def build_canocialized_resource(self, bucket, key): + + return '/{bucket}/{key}'.format(bucket=bucket, key=key) + + def noop_send_body(): + pass + + def noop_read(response): + return response.read() + + def do_operation( + self, method, bucket, key, input_headers={}, + fn_send_body=noop_send_body, fn_read=noop_read): + + headers = copy(input_headers) + headers['Expect'] = '100-continue' + + uri = self.build_canocialized_resource(bucket, key) + signature, request_date = self.sign_request(method, uri, headers) + headers['Authorization'] = "AWS {0}:{1}".format( + self.access_key, signature) + headers['Date'] = request_date + + connection = HTTPSConnection(self.end_point) \ + if self.https_flag else HTTPConnection(self.end_point) + connection.timeout = self.socket_timeout + + def perform_request(): + + connection.request(method, uri, fn_send_body(), headers) + response = connection.getresponse() + log("Sent {0} request to {1} {2} with headers {3}. \ + Got response status {4}: {5}". + format(method, self.end_point, uri, headers, + response.status, response.reason)) + return fn_read(response) + + try: + return retry(self.max_error_retry, perform_request) + finally: + connection.close() + + ''' + See http://bit.ly/MMC5de for more information regarding the creation of + AWS authorization tokens and header signing + ''' + def sign_request(self, operation, canocialized_resource, headers): + + request_date = datetime.utcnow( + ).strftime('%a, %d %b %Y %H:%M:%S +0000') + + content_hash = get_optional_key(headers, self.HEADER_CONTENT_MD5) + content_type = get_optional_key(headers, self.HEADER_CONTENT_TYPE) + + string_to_sign = join( + [operation, content_hash, content_type, request_date, + canocialized_resource], '\n') + + signature = base64.encodestring( + hmac.new(self.secret_key, string_to_sign.encode('utf8'), + hashlib.sha1).digest())[:-1] + + return signature, request_date + + def put(self, bucket, key, src_filename): + + headers = { + self.HEADER_CONTENT_MD5: compute_md5(src_filename), + self.HEADER_CONTENT_TYPE: 'application/octet-stream', + self.HEADER_CONTENT_LENGTH: os.stat(src_filename).st_size, + } + + def send_body(): + return open(src_filename, 'rb') + + self.do_operation('PUT', bucket, key, headers, send_body) + + def get(self, bucket, key, target_filename): + + def read(response): + + with open(target_filename, 'wb') as file: + while True: + block = response.read(8192) + if not block: + break + file.write(block) + + return self.do_operation('GET', bucket, key, fn_read=read) + + def delete(self, bucket, key): + + return self.do_operation('DELETE', bucket, key) + + +def parseArguments(args): + + # The keys in the args map will correspond to the properties defined on + # the com.cloud.utils.S3Utils#ClientOptions interface + client = S3Client( + args['accessKey'], args['secretKey'], args['endPoint'], + args['isHttps'], args['connectionTimeout'], args['socketTimeout']) + + operation = args['operation'] + bucket = args['bucket'] + key = args['key'] + filename = args['filename'] + + if is_blank(operation): + raise ValueError('An operation must be specified.') + + if is_blank(bucket): + raise ValueError('A bucket must be specified.') + + if is_blank(key): + raise ValueError('A value must be specified.') + + if is_blank(filename): + raise ValueError('A filename must be specified.') + + return client, operation, bucket, key, filename + + +@echo +def s3(session, args): + + client, operation, bucket, key, filename = parseArguments(args) + + try: + + if operation == 'put': + client.put(bucket, key, filename) + elif operation == 'get': + client.get(bucket, key, filename) + elif operation == 'delete': + client.delete(bucket, key, filename) + else: + raise RuntimeError( + "S3 plugin does not support operation {0}.".format(operation)) + + return 'true' + + except: + log("Operation {0} on file {1} from/in bucket {2} key {3}.".format( + operation, filename, bucket, key)) + log(traceback.format_exc()) + return 'false' + +if __name__ == "__main__": + XenAPIPlugin.dispatch({"s3": s3}) diff --git a/scripts/vm/hypervisor/xenserver/xenserver56/patch b/scripts/vm/hypervisor/xenserver/xenserver56/patch index d485414fdcb..36dba3dc06b 100644 --- a/scripts/vm/hypervisor/xenserver/xenserver56/patch +++ b/scripts/vm/hypervisor/xenserver/xenserver56/patch @@ -62,3 +62,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin swift=..,0755,/opt/xensource/bin swiftxen=..,0755,/etc/xapi.d/plugins +s3xen=..,0755,/etc/xapi.d/plugins + diff --git a/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch b/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch index 9fe9740756c..d20e60f2e49 100644 --- a/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch +++ b/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch @@ -61,3 +61,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin swift=..,0755,/opt/xensource/bin swiftxen=..,0755,/etc/xapi.d/plugins +s3xen=..,0755,/etc/xapi.d/plugins + diff --git a/scripts/vm/hypervisor/xenserver/xenserver60/patch b/scripts/vm/hypervisor/xenserver/xenserver60/patch index f0491092749..c9125f4c5b2 100644 --- a/scripts/vm/hypervisor/xenserver/xenserver60/patch +++ b/scripts/vm/hypervisor/xenserver/xenserver60/patch @@ -66,3 +66,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin swift=..,0755,/opt/xensource/bin swiftxen=..,0755,/etc/xapi.d/plugins +s3xen=..,0755,/etc/xapi.d/plugins + diff --git a/server/pom.xml b/server/pom.xml index e3308d85ea2..5ae926f9111 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -62,6 +62,11 @@ jstl ${cs.jstl.version} + + commons-codec + commons-codec + ${cs.codec.version} + org.apache.cloudstack cloud-utils @@ -69,7 +74,6 @@ tests test - install diff --git a/server/src/com/cloud/api/ApiDBUtils.java b/server/src/com/cloud/api/ApiDBUtils.java index 3b5f634f216..cdd5339665c 100755 --- a/server/src/com/cloud/api/ApiDBUtils.java +++ b/server/src/com/cloud/api/ApiDBUtils.java @@ -115,6 +115,7 @@ import com.cloud.storage.StoragePoolVO; import com.cloud.storage.StorageStats; import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateHostVO; +import com.cloud.storage.VMTemplateS3VO; import com.cloud.storage.VMTemplateSwiftVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume.Type; @@ -129,6 +130,7 @@ import com.cloud.storage.dao.UploadDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VMTemplateHostDao; +import com.cloud.storage.dao.VMTemplateS3Dao; import com.cloud.storage.dao.VMTemplateSwiftDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeHostDao; @@ -196,6 +198,7 @@ public class ApiDBUtils { private static VMTemplateDetailsDao _templateDetailsDao; private static VMTemplateHostDao _templateHostDao; private static VMTemplateSwiftDao _templateSwiftDao; + private static VMTemplateS3Dao _templateS3Dao; private static UploadDao _uploadDao; private static UserDao _userDao; private static UserStatisticsDao _userStatsDao; @@ -260,6 +263,7 @@ public class ApiDBUtils { _templateDetailsDao = locator.getDao(VMTemplateDetailsDao.class); _templateHostDao = locator.getDao(VMTemplateHostDao.class); _templateSwiftDao = locator.getDao(VMTemplateSwiftDao.class); + _templateS3Dao = locator.getDao(VMTemplateS3Dao.class); _uploadDao = locator.getDao(UploadDao.class); _userDao = locator.getDao(UserDao.class); _userStatsDao = locator.getDao(UserStatisticsDao.class); @@ -575,6 +579,10 @@ public class ApiDBUtils { return _templateSwiftDao.findOneByTemplateId(templateId); } + public static VMTemplateS3VO findTemplateS3Ref(long templateId) { + return _templateS3Dao.findOneByTemplateId(templateId); + } + public static UploadVO findUploadById(Long id) { return _uploadDao.findById(id); } diff --git a/server/src/com/cloud/api/ApiDispatcher.java b/server/src/com/cloud/api/ApiDispatcher.java index dfe4a1fc789..9fd0b2e9f06 100755 --- a/server/src/com/cloud/api/ApiDispatcher.java +++ b/server/src/com/cloud/api/ApiDispatcher.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.api; +import static org.apache.commons.lang.StringUtils.isNotBlank; + import java.lang.reflect.Field; import java.text.DateFormat; import java.text.ParseException; @@ -41,7 +43,6 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.utils.IdentityProxy; import com.cloud.server.ManagementServer; import com.cloud.user.Account; import com.cloud.user.UserContext; @@ -417,10 +418,20 @@ public class ApiDispatcher { } break; case FLOAT: - field.set(cmdObj, Float.valueOf(paramObj.toString())); + // Assuming that the parameters have been checked for required before now, + // we ignore blank or null values and defer to the command to set a default + // value for optional parameters ... + if (paramObj != null && isNotBlank(paramObj.toString())) { + field.set(cmdObj, Float.valueOf(paramObj.toString())); + } break; case INTEGER: - field.set(cmdObj, Integer.valueOf(paramObj.toString())); + // Assuming that the parameters have been checked for required before now, + // we ignore blank or null values and defer to the command to set a default + // value for optional parameters ... + if (paramObj != null && isNotBlank(paramObj.toString())) { + field.set(cmdObj, Integer.valueOf(paramObj.toString())); + } break; case LIST: List listParam = new ArrayList(); diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index ebe8415048b..a5747101215 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -16,6 +16,9 @@ // under the License. package com.cloud.api; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; + import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Date; @@ -82,6 +85,7 @@ import com.cloud.api.response.RemoteAccessVpnResponse; import com.cloud.api.response.ResourceCountResponse; import com.cloud.api.response.ResourceLimitResponse; import com.cloud.api.response.ResourceTagResponse; +import com.cloud.api.response.S3Response; import com.cloud.api.response.SecurityGroupResponse; import com.cloud.api.response.SecurityGroupResultObject; import com.cloud.api.response.SecurityGroupRuleResponse; @@ -131,7 +135,6 @@ import com.cloud.dc.Vlan.VlanType; import com.cloud.dc.VlanVO; import com.cloud.domain.Domain; import com.cloud.event.Event; -import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.host.HostStats; import com.cloud.host.HostVO; @@ -189,6 +192,7 @@ import com.cloud.server.ResourceTag.TaggedResourceType; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.GuestOS; import com.cloud.storage.GuestOSCategoryVO; +import com.cloud.storage.S3; import com.cloud.storage.Snapshot; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; @@ -200,6 +204,7 @@ import com.cloud.storage.StorageStats; import com.cloud.storage.Swift; import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateHostVO; +import com.cloud.storage.VMTemplateS3VO; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateSwiftVO; import com.cloud.storage.VMTemplateVO; @@ -743,6 +748,25 @@ public class ApiResponseHelper implements ResponseGenerator { return swiftResponse; } + @Override + public S3Response createS3Response(final S3 result) { + + final S3Response response = new S3Response(); + + response.setAccessKey(result.getAccessKey()); + response.setConnectionTimeout(result.getConnectionTimeout()); + response.setEndPoint(result.getEndPoint()); + response.setHttpsFlag(result.getHttpsFlag()); + response.setMaxErrorRetry(result.getMaxErrorRetry()); + response.setObjectId(result.getId()); + response.setSecretKey(result.getSecretKey()); + response.setSocketTimeout(result.getSocketTimeout()); + response.setTemplateBucketName(result.getBucketName()); + + return response; + + } + @Override public VlanIpRangeResponse createVlanIpRangeResponse(Vlan vlan) { Long podId = ApiDBUtils.getPodIdForVlan(vlan.getId()); @@ -2153,7 +2177,7 @@ public class ApiResponseHelper implements ResponseGenerator { @Override public List createIsoResponses(long isoId, Long zoneId, boolean readyOnly) { - List isoResponses = new ArrayList(); + final List isoResponses = new ArrayList(); VirtualMachineTemplate iso = findTemplateById(isoId); if (iso.getTemplateType() == TemplateType.PERHOST) { TemplateResponse isoResponse = new TemplateResponse(); @@ -2191,11 +2215,17 @@ public class ApiResponseHelper implements ResponseGenerator { return isoResponses; } else { if (zoneId == null || zoneId == -1) { - isoResponses = createSwiftIsoResponses(iso); + isoResponses.addAll(createSwiftIsoResponses(iso)); if (!isoResponses.isEmpty()) { return isoResponses; } - List dcs = new ArrayList(); + + isoResponses.addAll(createS3IsoResponses(iso)); + if (!isoResponses.isEmpty()) { + return isoResponses; + } + + final List dcs = new ArrayList(); dcs.addAll(ApiDBUtils.listZones()); for (DataCenterVO dc : dcs) { isoResponses.addAll(createIsoResponses(iso, dc.getId(), readyOnly)); @@ -2207,6 +2237,65 @@ public class ApiResponseHelper implements ResponseGenerator { } } + private List createS3IsoResponses(final VirtualMachineTemplate iso) { + + final VMTemplateS3VO s3Iso = ApiDBUtils.findTemplateS3Ref(iso.getId()); + + if (s3Iso == null) { + return emptyList(); + } + + final TemplateResponse templateResponse = new TemplateResponse(); + + templateResponse.setId(iso.getId()); + templateResponse.setName(iso.getName()); + templateResponse.setDisplayText(iso.getDisplayText()); + templateResponse.setPublic(iso.isPublicTemplate()); + templateResponse.setExtractable(iso.isExtractable()); + templateResponse.setCreated(s3Iso.getCreated()); + templateResponse.setReady(true); + templateResponse.setBootable(iso.isBootable()); + templateResponse.setFeatured(iso.isFeatured()); + templateResponse.setCrossZones(iso.isCrossZones()); + templateResponse.setChecksum(iso.getChecksum()); + templateResponse.setDetails(iso.getDetails()); + + final GuestOS os = ApiDBUtils.findGuestOSById(iso.getGuestOSId()); + + if (os != null) { + templateResponse.setOsTypeId(os.getId()); + templateResponse.setOsTypeName(os.getDisplayName()); + } else { + templateResponse.setOsTypeId(-1L); + templateResponse.setOsTypeName(""); + } + + final Account account = ApiDBUtils.findAccountByIdIncludingRemoved(iso.getAccountId()); + populateAccount(templateResponse, account.getId()); + populateDomain(templateResponse, account.getDomainId()); + + boolean isAdmin = false; + if ((account == null) || BaseCmd.isAdmin(account.getType())) { + isAdmin = true; + } + + // If the user is an admin, add the template download status + if (isAdmin || account.getId() == iso.getAccountId()) { + // add download status + templateResponse.setStatus("Successfully Installed"); + } + + final Long isoSize = s3Iso.getSize(); + if (isoSize > 0) { + templateResponse.setSize(isoSize); + } + + templateResponse.setObjectName("iso"); + + return singletonList(templateResponse); + + } + private List createSwiftIsoResponses(VirtualMachineTemplate iso) { long isoId = iso.getId(); List isoResponses = new ArrayList(); diff --git a/server/src/com/cloud/api/doc/ApiXmlDocWriter.java b/server/src/com/cloud/api/doc/ApiXmlDocWriter.java index d31ef5a8a5f..ffdd5be237b 100644 --- a/server/src/com/cloud/api/doc/ApiXmlDocWriter.java +++ b/server/src/com/cloud/api/doc/ApiXmlDocWriter.java @@ -314,6 +314,11 @@ public class ApiXmlDocWriter { impl = clas.getSuperclass().getAnnotation(Implementation.class); } + if (impl == null) { + throw new IllegalStateException(String.format("An %1$s annotation is required for class %2$s.", + Implementation.class.getCanonicalName(), clas.getCanonicalName())); + } + if (impl.includeInApiDoc()) { String commandDescription = impl.description(); if (commandDescription != null && !commandDescription.isEmpty()) { diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 66ac2762de1..9b82fccfc2f 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -140,8 +140,8 @@ public enum Config { JobExpireMinutes("Advanced", ManagementServer.class, String.class, "job.expire.minutes", "1440", "Time (in minutes) for async-jobs to be kept in system", null), JobCancelThresholdMinutes("Advanced", ManagementServer.class, String.class, "job.cancel.threshold.minutes", "60", "Time (in minutes) for async-jobs to be forcely cancelled if it has been in process for long", null), SwiftEnable("Advanced", ManagementServer.class, Boolean.class, "swift.enable", "false", "enable swift ", null), - - EventPurgeInterval("Advanced", ManagementServer.class, Integer.class, "event.purge.interval", "86400", "The interval (in seconds) to wait before running the event purge thread", null), + S3Enable("Advanced", ManagementServer.class, Boolean.class, "s3.enable", "false", "enable s3 ", null), + EventPurgeInterval("Advanced", ManagementServer.class, Integer.class, "event.purge.interval", "86400", "The interval (in seconds) to wait before running the event purge thread", null), AccountCleanupInterval("Advanced", ManagementServer.class, Integer.class, "account.cleanup.interval", "86400", "The interval (in seconds) between cleanup for removed accounts", null), AllowPublicUserTemplates("Advanced", ManagementServer.class, Integer.class, "allow.public.user.templates", "true", "If false, users will not be able to create public templates.", null), InstanceName("Advanced", AgentManager.class, String.class, "instance.name", "VM", "Name of the deployment instance.", "instanceName"), diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index 33baaf12ca4..2d7dfe290a6 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -63,7 +63,9 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.SwiftVO; import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.S3Dao; import com.cloud.storage.dao.SwiftDao; +import com.cloud.storage.s3.S3Manager; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.swift.SwiftManager; import com.cloud.test.IPRangeConfig; @@ -118,6 +120,8 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura @Inject SwiftDao _swiftDao; @Inject + S3Dao _s3Dao; + @Inject ServiceOfferingDao _serviceOfferingDao; @Inject DiskOfferingDao _diskOfferingDao; @@ -158,6 +162,8 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura @Inject SwiftManager _swiftMgr; @Inject + S3Manager _s3Mgr; + @Inject PhysicalNetworkTrafficTypeDao _trafficTypeDao; @Inject NicDao _nicDao; @@ -419,6 +425,14 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura if (swift != null) { return " can not change " + Config.SwiftEnable.key() + " after you have added Swift"; } + if (this._s3Mgr.isS3Enabled()) { + return String.format("Swift is not supported when S3 is enabled."); + } + } + if (Config.S3Enable.key().equals(name)) { + if (this._swiftMgr.isSwiftEnabled()) { + return String.format("S3-backed Secondary Storage is not supported when Swift is enabled."); + } } return null; } @@ -1520,6 +1534,7 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura createDefaultSystemNetworks(zone.getId()); _swiftMgr.propagateSwiftTmplteOnZone(zone.getId()); + _s3Mgr.propagateTemplatesToZone(zone); txn.commit(); return zone; } catch (Exception ex) { diff --git a/server/src/com/cloud/configuration/DefaultComponentLibrary.java b/server/src/com/cloud/configuration/DefaultComponentLibrary.java index ef61044d06b..226ca49a56b 100755 --- a/server/src/com/cloud/configuration/DefaultComponentLibrary.java +++ b/server/src/com/cloud/configuration/DefaultComponentLibrary.java @@ -147,6 +147,7 @@ import com.cloud.storage.dao.DiskOfferingDaoImpl; import com.cloud.storage.dao.GuestOSCategoryDaoImpl; import com.cloud.storage.dao.GuestOSDaoImpl; import com.cloud.storage.dao.LaunchPermissionDaoImpl; +import com.cloud.storage.dao.S3DaoImpl; import com.cloud.storage.dao.SnapshotDaoImpl; import com.cloud.storage.dao.SnapshotPolicyDaoImpl; import com.cloud.storage.dao.SnapshotScheduleDaoImpl; @@ -159,11 +160,13 @@ import com.cloud.storage.dao.VMTemplateDaoImpl; import com.cloud.storage.dao.VMTemplateDetailsDaoImpl; import com.cloud.storage.dao.VMTemplateHostDaoImpl; import com.cloud.storage.dao.VMTemplatePoolDaoImpl; +import com.cloud.storage.dao.VMTemplateS3DaoImpl; import com.cloud.storage.dao.VMTemplateSwiftDaoImpl; import com.cloud.storage.dao.VMTemplateZoneDaoImpl; import com.cloud.storage.dao.VolumeDaoImpl; import com.cloud.storage.dao.VolumeHostDaoImpl; import com.cloud.storage.download.DownloadMonitorImpl; +import com.cloud.storage.s3.S3ManagerImpl; import com.cloud.storage.secondary.SecondaryStorageManagerImpl; import com.cloud.storage.snapshot.SnapshotManagerImpl; import com.cloud.storage.snapshot.SnapshotSchedulerImpl; @@ -270,6 +273,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com addDao("VMTemplateHostDao", VMTemplateHostDaoImpl.class); addDao("VolumeHostDao", VolumeHostDaoImpl.class); addDao("VMTemplateSwiftDao", VMTemplateSwiftDaoImpl.class); + addDao("VMTemplateS3Dao", VMTemplateS3DaoImpl.class); addDao("UploadDao", UploadDaoImpl.class); addDao("VMTemplatePoolDao", VMTemplatePoolDaoImpl.class); addDao("LaunchPermissionDao", LaunchPermissionDaoImpl.class); @@ -318,6 +322,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com addDao("KeystoreDao", KeystoreDaoImpl.class); addDao("DcDetailsDao", DcDetailsDaoImpl.class); addDao("SwiftDao", SwiftDaoImpl.class); + addDao("S3Dao", S3DaoImpl.class); addDao("AgentTransferMapDao", HostTransferMapDaoImpl.class); addDao("ProjectDao", ProjectDaoImpl.class); addDao("InlineLoadBalancerNicMapDao", InlineLoadBalancerNicMapDaoImpl.class); @@ -403,6 +408,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com info.addParameter("consoleproxy.sslEnabled", "true"); addManager("ProjectManager", ProjectManagerImpl.class); addManager("SwiftManager", SwiftManagerImpl.class); + addManager("S3Manager", S3ManagerImpl.class); addManager("StorageNetworkManager", StorageNetworkManagerImpl.class); addManager("ExternalLoadBalancerUsageManager", ExternalLoadBalancerUsageManagerImpl.class); addManager("HA Manager", HighAvailabilityManagerImpl.class); diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index ced601b179b..8d50886d13a 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -30,6 +30,11 @@ import java.util.Set; import javax.ejb.Local; import javax.naming.ConfigurationException; +import com.cloud.api.commands.AddS3Cmd; +import com.cloud.api.commands.ListS3sCmd; +import com.cloud.storage.S3; +import com.cloud.storage.S3VO; +import com.cloud.storage.s3.S3Manager; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; @@ -181,6 +186,8 @@ public class ResourceManagerImpl implements ResourceManager, ResourceService, Ma @Inject protected SwiftManager _swiftMgr; @Inject + protected S3Manager _s3Mgr; + @Inject protected HostDetailsDao _hostDetailsDao; @Inject protected ConfigurationDao _configDao; @@ -561,6 +568,16 @@ public class ResourceManagerImpl implements ResourceManager, ResourceService, Ma return _swiftMgr.listSwifts(cmd); } + @Override + public S3 discoverS3(final AddS3Cmd cmd) throws DiscoveryException { + return this._s3Mgr.addS3(cmd); + } + + @Override + public List listS3s(final ListS3sCmd cmd) { + return this._s3Mgr.listS3s(cmd); + } + private List discoverHostsFull(Long dcId, Long podId, Long clusterId, String clusterName, String url, String username, String password, String hypervisorType, List hostTags, Map params) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException { URI uri = null; diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index a768610b750..7ecc3ec9611 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -151,11 +151,13 @@ import com.cloud.storage.dao.StoragePoolWorkDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VMTemplateS3Dao; import com.cloud.storage.dao.VMTemplateSwiftDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeHostDao; import com.cloud.storage.download.DownloadMonitor; import com.cloud.storage.listener.StoragePoolMonitor; +import com.cloud.storage.s3.S3Manager; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.storage.snapshot.SnapshotScheduler; @@ -262,6 +264,10 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag @Inject protected VMTemplateSwiftDao _vmTemplateSwiftDao = null; @Inject + protected VMTemplateS3Dao _vmTemplateS3Dao; + @Inject + protected S3Manager _s3Mgr; + @Inject protected VMTemplateDao _vmTemplateDao = null; @Inject protected StoragePoolHostDao _poolHostDao = null; @@ -707,6 +713,8 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag try { if (snapshot.getSwiftId() != null && snapshot.getSwiftId() != 0) { _snapshotMgr.downloadSnapshotsFromSwift(snapshot); + } else if (snapshot.getS3Id() != null && snapshot.getS3Id() != 0) { + _snapshotMgr.downloadSnapshotsFromS3(snapshot); } CreateVolumeFromSnapshotCommand createVolumeFromSnapshotCommand = new CreateVolumeFromSnapshotCommand(primaryStoragePoolNameLabel, secondaryStoragePoolUrl, dcId, accountId, volumeId, backedUpSnapshotUuid, snapshot.getName(), _createVolumeFromSnapshotWait); @@ -2983,6 +2991,14 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag if (tsvs != null && tsvs.size() > 0) { size = tsvs.get(0).getSize(); } + + if (size == null && _s3Mgr.isS3Enabled()) { + VMTemplateS3VO vmTemplateS3VO = _vmTemplateS3Dao.findOneByTemplateId(template.getId()); + if (vmTemplateS3VO != null) { + size = vmTemplateS3VO.getSize(); + } + } + if (size == null) { List sss = _vmTemplateHostDao.search(sc, null); if (sss == null || sss.size() == 0) { diff --git a/server/src/com/cloud/storage/dao/S3Dao.java b/server/src/com/cloud/storage/dao/S3Dao.java new file mode 100644 index 00000000000..ebea3531339 --- /dev/null +++ b/server/src/com/cloud/storage/dao/S3Dao.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.dao; + +import com.cloud.agent.api.to.S3TO; +import com.cloud.storage.S3VO; +import com.cloud.utils.db.GenericDao; + +public interface S3Dao extends GenericDao { + + S3TO getS3TO(final Long id); + +} diff --git a/server/src/com/cloud/storage/dao/S3DaoImpl.java b/server/src/com/cloud/storage/dao/S3DaoImpl.java new file mode 100644 index 00000000000..6162e6ebf73 --- /dev/null +++ b/server/src/com/cloud/storage/dao/S3DaoImpl.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.dao; + +import com.cloud.agent.api.to.S3TO; +import com.cloud.storage.S3VO; +import com.cloud.utils.db.GenericDaoBase; + +import javax.ejb.Local; + +@Local(S3Dao.class) +public class S3DaoImpl extends GenericDaoBase implements S3Dao { + + @Override + public S3TO getS3TO(final Long id) { + + if (id != null) { + + final S3VO s3VO = findById(id); + if (s3VO != null) { + return s3VO.toS3TO(); + } + + } + + // NOTE: Excluded listAll / shuffle operation implemented in SwiftDaoImpl ... + + return null; + + } +} diff --git a/server/src/com/cloud/storage/dao/VMTemplateDao.java b/server/src/com/cloud/storage/dao/VMTemplateDao.java index f5b6913df8a..1284ba103cf 100755 --- a/server/src/com/cloud/storage/dao/VMTemplateDao.java +++ b/server/src/com/cloud/storage/dao/VMTemplateDao.java @@ -71,4 +71,6 @@ public interface VMTemplateDao extends GenericDao { List listPrivateTemplatesByHost(Long hostId); public Long countTemplatesForAccount(long accountId); + List findTemplatesToSyncToS3(); + } diff --git a/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java b/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java index 2a0dfc85559..5c71f1bdb0c 100755 --- a/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -92,6 +92,14 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem private final String SELECT_TEMPLATE_SWIFT_REF = "SELECT t.id, t.unique_name, t.name, t.public, t.featured, t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, " + "t.checksum, t.display_text, t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type FROM vm_template t"; + + private static final String SELECT_S3_CANDIDATE_TEMPLATES = "SELECT t.id, t.unique_name, t.name, t.public, t.featured, " + + "t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, t.checksum, t.display_text, " + + "t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type " + + "FROM vm_template t JOIN template_host_ref r ON t.id=r.template_id JOIN host h ON h.id=r.host_id " + + "WHERE t.hypervisor_type IN (SELECT hypervisor_type FROM host) AND r.download_state = 'DOWNLOADED' AND " + + "r.template_id NOT IN (SELECT template_id FROM template_s3_ref) AND r.destroyed = 0 AND t.type <> 'PERHOST'"; + protected SearchBuilder TemplateNameSearch; protected SearchBuilder UniqueNameSearch; protected SearchBuilder tmpltTypeSearch; @@ -917,5 +925,10 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem (accountType == Account.ACCOUNT_TYPE_DOMAIN_ADMIN) || (accountType == Account.ACCOUNT_TYPE_READ_ONLY_ADMIN)); } - + + @Override + public List findTemplatesToSyncToS3() { + return executeList(SELECT_S3_CANDIDATE_TEMPLATES, new Object[] {}); + } + } diff --git a/server/src/com/cloud/storage/dao/VMTemplateS3Dao.java b/server/src/com/cloud/storage/dao/VMTemplateS3Dao.java new file mode 100644 index 00000000000..d36fb3a2257 --- /dev/null +++ b/server/src/com/cloud/storage/dao/VMTemplateS3Dao.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.dao; + +import com.cloud.storage.VMTemplateS3VO; +import com.cloud.utils.db.GenericDao; + +import java.util.List; + +public interface VMTemplateS3Dao extends GenericDao { + + List listByS3Id(long id); + + VMTemplateS3VO findOneByTemplateId(long id); + + VMTemplateS3VO findOneByS3Template(long s3Id, long templateId); + + void expungeAllByTemplateId(long templateId); + +} diff --git a/server/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java b/server/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java new file mode 100644 index 00000000000..f23b803bf68 --- /dev/null +++ b/server/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.dao; + +import static com.cloud.utils.db.SearchCriteria.Op.*; +import static com.cloud.storage.VMTemplateS3VO.*; + +import com.cloud.storage.VMTemplateS3VO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +import javax.ejb.Local; +import java.util.List; + +@Local(VMTemplateS3Dao.class) +public class VMTemplateS3DaoImpl extends GenericDaoBase + implements VMTemplateS3Dao { + + private final SearchBuilder searchBuilder; + + public VMTemplateS3DaoImpl() { + + super(); + + this.searchBuilder = createSearchBuilder(); + this.searchBuilder + .and(S3_ID_COLUMN_NAME, this.searchBuilder.entity().getS3Id(), + EQ) + .and(TEMPLATE_ID_COLUMN_NAME, + this.searchBuilder.entity().getTemplateId(), EQ).done(); + + } + + @Override + public List listByS3Id(final long s3id) { + + final SearchCriteria criteria = this.searchBuilder + .create(); + + criteria.setParameters(S3_ID_COLUMN_NAME, s3id); + + return this.listBy(criteria); + + } + + @Override + public VMTemplateS3VO findOneByTemplateId(final long templateId) { + + final SearchCriteria criteria = this.searchBuilder + .create(); + + criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); + + return this.findOneBy(criteria); + + } + + @Override + public VMTemplateS3VO findOneByS3Template(final long s3Id, + final long templateId) { + + final SearchCriteria criteria = this.searchBuilder + .create(); + + criteria.setParameters(S3_ID_COLUMN_NAME, s3Id); + criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); + + return this.findOneBy(criteria); + + } + + @Override + public void expungeAllByTemplateId(long templateId) { + + final SearchCriteria criteria = this.searchBuilder + .create(); + + criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); + + this.expunge(criteria); + + } + +} diff --git a/server/src/com/cloud/storage/s3/S3Manager.java b/server/src/com/cloud/storage/s3/S3Manager.java new file mode 100644 index 00000000000..357f2aed463 --- /dev/null +++ b/server/src/com/cloud/storage/s3/S3Manager.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.s3; + +import java.util.List; + +import com.cloud.agent.api.to.S3TO; +import com.cloud.api.commands.AddS3Cmd; +import com.cloud.api.commands.ListS3sCmd; +import com.cloud.dc.DataCenterVO; +import com.cloud.exception.DiscoveryException; +import com.cloud.storage.S3; +import com.cloud.storage.S3VO; +import com.cloud.storage.VMTemplateS3VO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.utils.component.Manager; + +public interface S3Manager extends Manager { + + S3TO getS3TO(); + + S3TO getS3TO(Long s3Id); + + S3 addS3(AddS3Cmd addS3Cmd) throws DiscoveryException; + + Long chooseZoneForTemplateExtract(VMTemplateVO template); + + boolean isS3Enabled(); + + boolean isTemplateInstalled(Long templateId); + + void deleteTemplate(final Long accountId, final Long templateId); + + String downloadTemplateFromS3ToSecondaryStorage(final long dcId, + final long templateId, final int primaryStorageDownloadWait); + + List listS3s(ListS3sCmd listS3sCmd); + + VMTemplateS3VO findByTemplateId(Long templateId); + + void propagateTemplatesToZone(DataCenterVO zone); + + void propagateTemplateToAllZones(VMTemplateS3VO vmTemplateS3VO); + + void uploadTemplateToS3FromSecondaryStorage(final VMTemplateVO template); + +} diff --git a/server/src/com/cloud/storage/s3/S3ManagerImpl.java b/server/src/com/cloud/storage/s3/S3ManagerImpl.java new file mode 100644 index 00000000000..6b072540c66 --- /dev/null +++ b/server/src/com/cloud/storage/s3/S3ManagerImpl.java @@ -0,0 +1,669 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.s3; + +import static com.cloud.storage.S3VO.ID_COLUMN_NAME; +import static com.cloud.utils.DateUtil.now; +import static com.cloud.utils.S3Utils.canConnect; +import static com.cloud.utils.S3Utils.canReadWriteBucket; +import static com.cloud.utils.S3Utils.checkBucketName; +import static com.cloud.utils.S3Utils.checkClientOptions; +import static com.cloud.utils.S3Utils.doesBucketExist; +import static com.cloud.utils.StringUtils.join; +import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock; +import static com.cloud.utils.db.SearchCriteria.Op.EQ; +import static java.lang.Boolean.TRUE; +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.shuffle; +import static java.util.Collections.singletonList; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; + +import javax.ejb.Local; +import javax.naming.ConfigurationException; + +import org.apache.log4j.Logger; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.DeleteTemplateFromS3Command; +import com.cloud.agent.api.DownloadTemplateFromS3ToSecondaryStorageCommand; +import com.cloud.agent.api.UploadTemplateToS3FromSecondaryStorageCommand; +import com.cloud.agent.api.to.S3TO; +import com.cloud.api.commands.AddS3Cmd; +import com.cloud.api.commands.ListS3sCmd; +import com.cloud.configuration.Config; +import com.cloud.configuration.dao.ConfigurationDao; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.exception.DiscoveryException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.storage.S3; +import com.cloud.storage.S3VO; +import com.cloud.storage.VMTemplateHostVO; +import com.cloud.storage.VMTemplateS3VO; +import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.dao.S3Dao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplateHostDao; +import com.cloud.storage.dao.VMTemplateS3Dao; +import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.storage.secondary.SecondaryStorageVmManager; +import com.cloud.utils.S3Utils.ClientOptions; +import com.cloud.utils.component.Inject; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.exception.CloudRuntimeException; + +@Local(value = { S3Manager.class }) +public class S3ManagerImpl implements S3Manager { + + private static final Logger LOGGER = Logger.getLogger(S3ManagerImpl.class); + + private String name; + + @Inject + private AgentManager agentManager; + + @Inject + private S3Dao s3Dao; + + @Inject + private VMTemplateZoneDao vmTemplateZoneDao; + + @Inject + private VMTemplateS3Dao vmTemplateS3Dao; + + @Inject + private VMTemplateHostDao vmTemplateHostDao; + + @Inject + private VMTemplateDao vmTemplateDao; + + @Inject + private ConfigurationDao configurationDao; + + @Inject + private DataCenterDao dataCenterDao; + + @Inject + private HostDao hostDao; + + @Inject + private SecondaryStorageVmManager secondaryStorageVMManager; + + protected S3ManagerImpl() { + super(); + } + + private void verifyConnection(final S3TO s3) throws DiscoveryException { + + if (!canConnect(s3)) { + throw new DiscoveryException(format("Unable to connect to S3 " + + "using access key %1$s, secret key %2$s, and endpoint, " + + "%3$S", s3.getAccessKey(), s3.getSecretKey(), + s3.getEndPoint() != null ? s3.getEndPoint() : "default")); + } + + } + + private void verifyBuckets(S3TO s3) throws DiscoveryException { + + final List errorMessages = new ArrayList(); + + errorMessages.addAll(verifyBucket(s3, s3.getBucketName())); + + throwDiscoveryExceptionFromErrorMessages(errorMessages); + + } + + private List verifyBucket(final ClientOptions clientOptions, + final String bucketName) { + + if (!doesBucketExist(clientOptions, bucketName)) { + return singletonList(format("Bucket %1$s does not exist.", + bucketName)); + } + + if (!canReadWriteBucket(clientOptions, bucketName)) { + return singletonList(format("Can read/write from bucket %1$s.", + bucketName)); + } + + return emptyList(); + } + + private void validateFields(final S3VO s3VO) { + + final List errorMessages = new ArrayList(); + + errorMessages.addAll(checkClientOptions(s3VO.toS3TO())); + + errorMessages.addAll(checkBucketName("template", s3VO.getBucketName())); + + throwDiscoveryExceptionFromErrorMessages(errorMessages); + + } + + private void enforceS3PreConditions() throws DiscoveryException { + + if (!this.isS3Enabled()) { + throw new DiscoveryException("S3 is not enabled."); + } + + if (this.getS3TO() != null) { + throw new DiscoveryException("Attempt to define multiple S3 " + + "instances. Only one instance definition is supported."); + } + + } + + private void throwDiscoveryExceptionFromErrorMessages( + final List errorMessages) { + + if (!errorMessages.isEmpty()) { + throw new CloudRuntimeException(join(errorMessages, " ")); + } + + } + + @SuppressWarnings("unchecked") + private String determineLockId(final long accountId, final long templateId) { + + // TBD The lock scope may be too coarse grained. Deletes need to lock + // the template across all zones where upload and download could + // probably safely scoped to the zone ... + return join(asList("S3_TEMPLATE", accountId, templateId), "_"); + + } + + @Override + public S3TO getS3TO(final Long s3Id) { + return this.s3Dao.getS3TO(s3Id); + } + + @Override + public S3TO getS3TO() { + + final List s3s = this.s3Dao.listAll(); + + if (s3s == null || (s3s != null && s3s.isEmpty())) { + return null; + } + + if (s3s.size() == 1) { + return s3s.get(0).toS3TO(); + } + + throw new CloudRuntimeException("Multiple S3 instances have been " + + "defined. Only one instance configuration is supported."); + + } + + @Override + public S3 addS3(final AddS3Cmd addS3Cmd) throws DiscoveryException { + + this.enforceS3PreConditions(); + + final S3VO s3VO = new S3VO(UUID.randomUUID().toString(), + addS3Cmd.getAccessKey(), addS3Cmd.getSecretKey(), + addS3Cmd.getEndPoint(), addS3Cmd.getBucketName(), + addS3Cmd.getHttpsFlag(), addS3Cmd.getConnectionTimeout(), + addS3Cmd.getMaxErrorRetry(), addS3Cmd.getSocketTimeout(), now()); + + this.validateFields(s3VO); + + final S3TO s3 = s3VO.toS3TO(); + this.verifyConnection(s3); + this.verifyBuckets(s3); + + return this.s3Dao.persist(s3VO); + + } + + @Override + public boolean isS3Enabled() { + return Boolean + .valueOf(configurationDao.getValue(Config.S3Enable.key())); + } + + @Override + public boolean isTemplateInstalled(final Long templateId) { + throw new UnsupportedOperationException( + "S3Manager#isTemplateInstalled (DeleteIsoCmd) has not yet " + + "been implemented"); + } + + @Override + public void deleteTemplate(final Long templateId, final Long accountId) { + + final S3TO s3 = getS3TO(); + + if (s3 == null) { + final String errorMessage = "Delete Template Failed: No S3 configuration defined."; + LOGGER.error(errorMessage); + throw new CloudRuntimeException(errorMessage); + } + + final VMTemplateS3VO vmTemplateS3VO = vmTemplateS3Dao + .findOneByS3Template(s3.getId(), templateId); + if (vmTemplateS3VO == null) { + final String errorMessage = format( + "Delete Template Failed: Unable to find Template %1$s in S3.", + templateId); + LOGGER.error(errorMessage); + throw new CloudRuntimeException(errorMessage); + } + + try { + + executeWithNoWaitLock(determineLockId(accountId, templateId), + new Callable() { + + @Override + public Void call() throws Exception { + + final Answer answer = agentManager.sendToSSVM(null, + new DeleteTemplateFromS3Command(s3, + accountId, templateId)); + if (answer == null || !answer.getResult()) { + final String errorMessage = format( + "Delete Template Failed: Unable to delete template id %1$s from S3 due to following error: %2$s", + templateId, + ((answer == null) ? "answer is null" + : answer.getDetails())); + LOGGER.error(errorMessage); + throw new CloudRuntimeException(errorMessage); + } + + vmTemplateS3Dao.remove(vmTemplateS3VO.getId()); + LOGGER.debug(format( + "Deleted template %1$s from S3.", + templateId)); + + return null; + + } + + }); + + } catch (Exception e) { + + final String errorMessage = format( + "Delete Template Failed: Unable to delete template id %1$s from S3 due to the following error: %2$s.", + templateId, e.getMessage()); + LOGGER.error(errorMessage); + throw new CloudRuntimeException(errorMessage, e); + + } + + } + + @SuppressWarnings("unchecked") + @Override + public String downloadTemplateFromS3ToSecondaryStorage( + final long dataCenterId, final long templateId, + final int primaryStorageDownloadWait) { + + if (!isS3Enabled()) { + return null; + } + + final VMTemplateVO template = vmTemplateDao.findById(templateId); + if (template == null) { + final String errorMessage = String + .format("Failed to download template id %1$s from S3 because the template definition was not found.", + templateId); + LOGGER.error(errorMessage); + return errorMessage; + } + + final VMTemplateS3VO templateS3VO = findByTemplateId(templateId); + if (templateS3VO == null) { + final String errorMessage = format( + "Failed to download template id %1$s from S3 because it does not exist in S3.", + templateId); + LOGGER.error(errorMessage); + return errorMessage; + } + + final S3TO s3 = getS3TO(templateS3VO.getS3Id()); + if (s3 == null) { + final String errorMessage = format( + "Failed to download template id %1$s from S3 because S3 id %2$s does not exist.", + templateId, templateS3VO); + LOGGER.error(errorMessage); + return errorMessage; + } + + final HostVO secondaryStorageHost = secondaryStorageVMManager + .findSecondaryStorageHost(dataCenterId); + if (secondaryStorageHost == null) { + final String errorMessage = format( + "Unable to find secondary storage host for zone id %1$s.", + dataCenterId); + LOGGER.error(errorMessage); + throw new CloudRuntimeException(errorMessage); + } + + final long accountId = template.getAccountId(); + final DownloadTemplateFromS3ToSecondaryStorageCommand cmd = new DownloadTemplateFromS3ToSecondaryStorageCommand( + s3, accountId, templateId, secondaryStorageHost.getName(), + primaryStorageDownloadWait); + + try { + + executeWithNoWaitLock(determineLockId(accountId, templateId), + new Callable() { + + @Override + public Void call() throws Exception { + + final Answer answer = agentManager.sendToSSVM( + dataCenterId, cmd); + + if (answer == null || !answer.getResult()) { + final String errMsg = String + .format("Failed to download template from S3 to secondary storage due to %1$s", + (answer == null ? "answer is null" + : answer.getDetails())); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + + final String installPath = join( + asList("template", "tmpl", accountId, + templateId), File.separator); + final VMTemplateHostVO tmpltHost = new VMTemplateHostVO( + secondaryStorageHost.getId(), templateId, + now(), 100, Status.DOWNLOADED, null, null, + null, installPath, template.getUrl()); + tmpltHost.setSize(templateS3VO.getSize()); + tmpltHost.setPhysicalSize(templateS3VO + .getPhysicalSize()); + vmTemplateHostDao.persist(tmpltHost); + + return null; + + } + + }); + + } catch (Exception e) { + final String errMsg = "Failed to download template from S3 to secondary storage due to " + + e.toString(); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + + return null; + + } + + @Override + public List listS3s(final ListS3sCmd cmd) { + + final Filter filter = new Filter(S3VO.class, ID_COLUMN_NAME, TRUE, + cmd.getStartIndex(), cmd.getPageSizeVal()); + final SearchCriteria criteria = this.s3Dao.createSearchCriteria(); + + if (cmd.getId() != null) { + criteria.addAnd(ID_COLUMN_NAME, EQ, cmd.getId()); + } + + return this.s3Dao.search(criteria, filter); + + } + + @Override + public VMTemplateS3VO findByTemplateId(final Long templateId) { + throw new UnsupportedOperationException( + "S3Manager#findByTemplateId(Long) has not yet " + + "been implemented"); + } + + @Override + public void propagateTemplatesToZone(final DataCenterVO zone) { + + if (!isS3Enabled()) { + return; + } + + final List s3VMTemplateRefs = this.vmTemplateS3Dao + .listAll(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(format("Propagating %1$s templates to zone %2$s.", + s3VMTemplateRefs.size(), zone.getName())); + } + + for (final VMTemplateS3VO templateS3VO : s3VMTemplateRefs) { + this.vmTemplateZoneDao.persist(new VMTemplateZoneVO(zone.getId(), + templateS3VO.getTemplateId(), now())); + } + + } + + @Override + public boolean configure(final String name, final Map params) + throws ConfigurationException { + + if (LOGGER.isInfoEnabled()) { + LOGGER.info(format("Configuring S3 Manager %1$s", name)); + } + + this.name = name; + + return true; + + } + + @Override + public boolean start() { + LOGGER.info("Starting S3 Manager"); + return true; + } + + @Override + public boolean stop() { + LOGGER.info("Stopping S3 Manager"); + return true; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public void propagateTemplateToAllZones(final VMTemplateS3VO vmTemplateS3VO) { + + final long templateId = vmTemplateS3VO.getId(); + + if (!isS3Enabled()) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(format( + "Attempt to propogate template id %1$s across all zones. However, S3 is not enabled.", + templateId)); + } + return; + + } + + final S3TO s3 = getS3TO(); + + if (s3 == null) { + LOGGER.warn(format( + "Unable to propagate template id %1$s across all zones because S3 is enabled, but not configured.", + templateId)); + return; + } + + if (vmTemplateS3VO != null) { + final List dataCenters = dataCenterDao.listAll(); + for (DataCenterVO dataCenter : dataCenters) { + final VMTemplateZoneVO tmpltZoneVO = new VMTemplateZoneVO( + dataCenter.getId(), templateId, now()); + vmTemplateZoneDao.persist(tmpltZoneVO); + } + } + + } + + @Override + public Long chooseZoneForTemplateExtract(VMTemplateVO template) { + + final S3TO s3 = getS3TO(); + + if (s3 == null) { + return null; + } + + final List templateHosts = vmTemplateHostDao + .listByOnlyTemplateId(template.getId()); + if (templateHosts != null) { + shuffle(templateHosts); + for (VMTemplateHostVO vmTemplateHostVO : templateHosts) { + final HostVO host = hostDao.findById(vmTemplateHostVO + .getHostId()); + if (host != null) { + return host.getDataCenterId(); + } + throw new CloudRuntimeException( + format("Unable to find secondary storage host for template id %1$s.", + template.getId())); + } + } + + final List dataCenters = dataCenterDao.listAll(); + shuffle(dataCenters); + return dataCenters.get(0).getId(); + + } + + @Override + public void uploadTemplateToS3FromSecondaryStorage( + final VMTemplateVO template) { + + final Long templateId = template.getId(); + + final List templateHostRefs = vmTemplateHostDao + .listByTemplateId(templateId); + + if (templateHostRefs == null + || (templateHostRefs != null && templateHostRefs.isEmpty())) { + throw new CloudRuntimeException( + format("Attempt to sync template id %1$s that is not attached to a host.", + templateId)); + } + + final VMTemplateHostVO templateHostRef = templateHostRefs.get(0); + + if (!isS3Enabled()) { + return; + } + + final S3TO s3 = getS3TO(); + if (s3 == null) { + LOGGER.warn("S3 Template Sync Failed: Attempt to sync templates with S3, but no S3 instance defined."); + return; + } + + final HostVO secondaryHost = this.hostDao.findById(templateHostRef + .getHostId()); + if (secondaryHost == null) { + throw new CloudRuntimeException(format( + "Unable to find secondary storage host id %1$s.", + templateHostRef.getHostId())); + } + + final Long dataCenterId = secondaryHost.getDataCenterId(); + final Long accountId = template.getAccountId(); + + try { + + executeWithNoWaitLock(determineLockId(accountId, templateId), + new Callable() { + + @Override + public Void call() throws Exception { + + final UploadTemplateToS3FromSecondaryStorageCommand cmd = new UploadTemplateToS3FromSecondaryStorageCommand( + s3, secondaryHost.getStorageUrl(), + dataCenterId, accountId, templateId); + + final Answer answer = agentManager.sendToSSVM( + dataCenterId, cmd); + if (answer == null || !answer.getResult()) { + + final String reason = answer != null ? answer + .getDetails() + : "S3 template sync failed due to an unspecified error."; + throw new CloudRuntimeException( + format("Failed to upload template id %1$s to S3 from secondary storage due to %2$s.", + templateId, reason)); + + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(format( + "Creating VMTemplateS3VO instance using template id %1s.", + templateId)); + } + + final VMTemplateS3VO vmTemplateS3VO = new VMTemplateS3VO( + s3.getId(), templateId, now(), + templateHostRef.getSize(), templateHostRef + .getPhysicalSize()); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(format("Persisting %1$s", + vmTemplateS3VO)); + } + + vmTemplateS3Dao.persist(vmTemplateS3VO); + propagateTemplateToAllZones(vmTemplateS3VO); + + return null; + + } + + }); + + } catch (Exception e) { + + final String errorMessage = format( + "Failed to upload template id %1$s for zone id %2$s to S3.", + templateId, dataCenterId); + LOGGER.error(errorMessage, e); + + } + + } + +} diff --git a/server/src/com/cloud/storage/snapshot/SnapshotManager.java b/server/src/com/cloud/storage/snapshot/SnapshotManager.java index a10298efb90..a7692de7107 100755 --- a/server/src/com/cloud/storage/snapshot/SnapshotManager.java +++ b/server/src/com/cloud/storage/snapshot/SnapshotManager.java @@ -128,6 +128,8 @@ public interface SnapshotManager { void downloadSnapshotsFromSwift(SnapshotVO ss); + void downloadSnapshotsFromS3(SnapshotVO snapshot); + HostVO getSecondaryStorageHost(SnapshotVO snapshot); String getSecondaryStorageURL(SnapshotVO snapshot); diff --git a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 32e37e63c0d..259feab3bcf 100755 --- a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.storage.snapshot; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; @@ -34,9 +35,11 @@ import com.cloud.agent.api.BackupSnapshotCommand; import com.cloud.agent.api.Command; import com.cloud.agent.api.DeleteSnapshotBackupCommand; import com.cloud.agent.api.DeleteSnapshotsDirCommand; +import com.cloud.agent.api.DownloadSnapshotFromS3Command; import com.cloud.agent.api.ManageSnapshotAnswer; import com.cloud.agent.api.ManageSnapshotCommand; import com.cloud.agent.api.downloadSnapshotFromSwiftCommand; +import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.alert.AlertManager; import com.cloud.api.commands.CreateSnapshotPolicyCmd; @@ -91,6 +94,7 @@ import com.cloud.storage.dao.SnapshotScheduleDao; import com.cloud.storage.dao.StoragePoolDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.s3.S3Manager; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.swift.SwiftManager; import com.cloud.tags.ResourceTagVO; @@ -171,6 +175,8 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma private ResourceLimitService _resourceLimitMgr; @Inject private SwiftManager _swiftMgr; + @Inject + private S3Manager _s3Mgr; @Inject private SecondaryStorageVmManager _ssvmMgr; @Inject @@ -477,11 +483,25 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma return createdSnapshot; } + private static void checkObjectStorageConfiguration(SwiftTO swift, S3TO s3) { + + if (swift != null && s3 != null) { + throw new CloudRuntimeException( + "Swift and S3 are not simultaneously supported for snapshot backup."); + } + + } @Override public void deleteSnapshotsForVolume (String secondaryStoragePoolUrl, Long dcId, Long accountId, Long volumeId ){ SwiftTO swift = _swiftMgr.getSwiftTO(); - DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand(swift, secondaryStoragePoolUrl, dcId, accountId, volumeId, null, true); + S3TO s3 = _s3Mgr.getS3TO(); + + checkObjectStorageConfiguration(swift, s3); + + DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand( + swift, s3, secondaryStoragePoolUrl, dcId, accountId, volumeId, + null, true); try { Answer ans = _agentMgr.sendToSSVM(dcId, cmd); if ( ans == null || !ans.getResult() ) { @@ -543,6 +563,54 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma } + private List determineBackupUuids(final SnapshotVO snapshot) { + + final List backupUuids = new ArrayList(); + backupUuids.add(0, snapshot.getBackupSnapshotId()); + + SnapshotVO tempSnapshot = snapshot; + while (tempSnapshot.getPrevSnapshotId() != 0) { + tempSnapshot = _snapshotDao.findById(tempSnapshot + .getPrevSnapshotId()); + backupUuids.add(0, tempSnapshot.getBackupSnapshotId()); + } + + return Collections.unmodifiableList(backupUuids); + } + + @Override + public void downloadSnapshotsFromS3(final SnapshotVO snapshot) { + + final VolumeVO volume = _volsDao.findById(snapshot.getVolumeId()); + final Long zoneId = volume.getDataCenterId(); + final HostVO secHost = _storageMgr.getSecondaryStorageHost(zoneId); + + final S3TO s3 = _s3Mgr.getS3TO(snapshot.getS3Id()); + final List backupUuids = determineBackupUuids(snapshot); + + try { + String parent = null; + for (final String backupUuid : backupUuids) { + final DownloadSnapshotFromS3Command cmd = new DownloadSnapshotFromS3Command( + s3, parent, secHost.getStorageUrl(), zoneId, + volume.getAccountId(), volume.getId(), backupUuid, + _backupsnapshotwait); + final Answer answer = _agentMgr.sendToSSVM(zoneId, cmd); + if ((answer == null) || !answer.getResult()) { + throw new CloudRuntimeException(String.format( + "S3 snapshot download failed due to %1$s.", + answer != null ? answer.getDetails() + : "unspecified error")); + } + parent = backupUuid; + } + } catch (Exception e) { + throw new CloudRuntimeException( + "Snapshot download from S3 failed due to " + e.toString(), + e); + } + + } @Override @DB @@ -577,6 +645,9 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma SwiftTO swift = _swiftMgr.getSwiftTO(); + S3TO s3 = _s3Mgr.getS3TO(); + + checkObjectStorageConfiguration(swift, s3); long prevSnapshotId = snapshot.getPrevSnapshotId(); if (prevSnapshotId > 0) { @@ -586,7 +657,8 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma prevBackupUuid = prevSnapshot.getBackupSnapshotId(); prevSnapshotUuid = prevSnapshot.getPath(); } - } else if ( prevSnapshot.getSwiftId() != null && swift != null ) { + } else if ((prevSnapshot.getSwiftId() != null && swift != null) + || (prevSnapshot.getS3Id() != null && s3 != null)) { prevBackupUuid = prevSnapshot.getBackupSnapshotId(); prevSnapshotUuid = prevSnapshot.getPath(); } @@ -599,8 +671,10 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma if ( swift != null ) { backupSnapshotCommand.setSwift(swift); + } else if (s3 != null) { + backupSnapshotCommand.setS3(s3); } - + String backedUpSnapshotUuid = null; // By default, assume failed. boolean backedUp = false; @@ -621,6 +695,9 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma if (backupSnapshotCommand.getSwift() != null ) { snapshot.setSwiftId(swift.getId()); snapshot.setBackupSnapshotId(backedUpSnapshotUuid); + } else if (backupSnapshotCommand.getS3() != null) { + snapshot.setS3Id(s3.getId()); + snapshot.setBackupSnapshotId(backedUpSnapshotUuid); } else { snapshot.setSecHostId(secHost.getId()); snapshot.setBackupSnapshotId(backedUpSnapshotUuid); @@ -832,7 +909,13 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma return true; } SwiftTO swift = _swiftMgr.getSwiftTO(snapshot.getSwiftId()); - DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand(swift, secondaryStoragePoolUrl, dcId, accountId, volumeId, backupOfSnapshot, false); + S3TO s3 = _s3Mgr.getS3TO(); + + checkObjectStorageConfiguration(swift, s3); + + DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand( + swift, s3, secondaryStoragePoolUrl, dcId, accountId, volumeId, + backupOfSnapshot, false); Answer answer = _agentMgr.sendToSSVM(dcId, cmd); if ((answer != null) && answer.getResult()) { @@ -979,9 +1062,15 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma } List ssHosts = _ssvmMgr.listSecondaryStorageHostsInOneZone(dcId); SwiftTO swift = _swiftMgr.getSwiftTO(); - if (swift == null) { + S3TO s3 = _s3Mgr.getS3TO(); + + checkObjectStorageConfiguration(swift, s3); + + if (swift == null && s3 == null) { for (HostVO ssHost : ssHosts) { - DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand(null, ssHost.getStorageUrl(), dcId, accountId, volumeId, "", true); + DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand( + null, null, ssHost.getStorageUrl(), dcId, + accountId, volumeId, "", true); Answer answer = null; try { answer = _agentMgr.sendToSSVM(dcId, cmd); @@ -998,12 +1087,14 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma } } } else { - DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand(swift, "", dcId, accountId, volumeId, "", true); + DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand( + swift, s3, "", dcId, accountId, volumeId, "", true); Answer answer = null; try { answer = _agentMgr.sendToSSVM(dcId, cmd); } catch (Exception e) { - s_logger.warn("Failed to delete all snapshot for volume " + volumeId + " on swift"); + final String storeType = s3 != null ? "S3" : "swift"; + s_logger.warn("Failed to delete all snapshot for volume " + volumeId + " on " + storeType); } if ((answer != null) && answer.getResult()) { s_logger.debug("Deleted all snapshots for volume: " + volumeId + " under account: " + accountId); diff --git a/server/src/com/cloud/template/S3SyncTask.java b/server/src/com/cloud/template/S3SyncTask.java new file mode 100644 index 00000000000..ed179dc8961 --- /dev/null +++ b/server/src/com/cloud/template/S3SyncTask.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.template; + +import static java.lang.String.*; + +import java.util.List; + +import org.apache.log4j.Logger; + +import com.cloud.agent.api.to.S3TO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.s3.S3Manager; + +final class S3SyncTask implements Runnable { + + private static final Logger LOGGER = Logger.getLogger(S3SyncTask.class); + + private final VMTemplateDao vmTemplateDao; + private final S3Manager s3Mgr; + + S3SyncTask(final VMTemplateDao vmTemplateDao, final S3Manager s3Mgr) { + + super(); + + assert vmTemplateDao != null; + assert s3Mgr != null; + + this.vmTemplateDao = vmTemplateDao; + this.s3Mgr = s3Mgr; + + } + + @Override + public void run() { + + try { + + final S3TO s3 = s3Mgr.getS3TO(); + + if (s3 == null) { + LOGGER.warn("S3 sync skipped because no S3 instance is configured."); + return; + } + + final List candidateTemplates = vmTemplateDao + .findTemplatesToSyncToS3(); + + if (candidateTemplates.isEmpty()) { + LOGGER.debug("All templates are synced with S3."); + return; + } + + for (VMTemplateVO candidateTemplate : candidateTemplates) { + + if (LOGGER.isInfoEnabled()) { + LOGGER.info(format( + "Uploading template %1$s (id: %2$s) to S3.", + candidateTemplate.getName(), + candidateTemplate.getId())); + } + + s3Mgr.uploadTemplateToS3FromSecondaryStorage(candidateTemplate); + + } + + LOGGER.debug("Completed S3 template sync task."); + + } catch (Exception e) { + LOGGER.warn( + "S3 Sync Task ignored exception, and will continue to execute.", + e); + } + + } + +} diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 1e87de2fecb..82c31f14ffb 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -113,10 +113,12 @@ import com.cloud.storage.dao.UploadDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VMTemplateS3Dao; import com.cloud.storage.dao.VMTemplateSwiftDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.download.DownloadMonitor; +import com.cloud.storage.s3.S3Manager; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.swift.SwiftManager; import com.cloud.storage.upload.UploadMonitor; @@ -179,8 +181,12 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe @Inject SwiftManager _swiftMgr; @Inject + S3Manager _s3Mgr; + @Inject VMTemplateSwiftDao _tmpltSwiftDao; @Inject + VMTemplateS3Dao _vmS3TemplateDao; + @Inject ConfigurationDao _configDao; @Inject ClusterDao _clusterDao; @@ -207,6 +213,7 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe ExecutorService _preloadExecutor; ScheduledExecutorService _swiftTemplateSyncExecutor; + private ScheduledExecutorService _s3TemplateSyncExecutor = null; @Inject (adapter=TemplateAdapter.class) protected Adapters _adapters; @@ -344,10 +351,14 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe } } - if (zoneId == null) { + if (zoneId == null && _swiftMgr.isSwiftEnabled()) { zoneId = _swiftMgr.chooseZoneForTmpltExtract(templateId); } + if (zoneId == null && _s3Mgr.isS3Enabled()) { + zoneId = _s3Mgr.chooseZoneForTemplateExtract(template); + } + if (_dcDao.findById(zoneId) == null) { throw new IllegalArgumentException("Please specify a valid zone."); } @@ -380,7 +391,13 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe if (swift != null && sservers != null) { downloadTemplateFromSwiftToSecondaryStorage(zoneId, templateId); } + } else if (tmpltHostRef == null && _s3Mgr.isS3Enabled()) { + if (sservers != null) { + _s3Mgr.downloadTemplateFromS3ToSecondaryStorage(zoneId, + templateId, _primaryStorageDownloadWait); + } } + if (tmpltHostRef == null) { throw new InvalidParameterValueException("The " + desc + " has not been downloaded "); } @@ -594,6 +611,12 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); return null; } + result = _s3Mgr.downloadTemplateFromS3ToSecondaryStorage(dcId, + templateId, _primaryStorageDownloadWait); + if (result != null) { + s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); + return null; + } templateHostRef = _storageMgr.findVmTemplateHost(templateId, pool); if (templateHostRef == null || templateHostRef.getDownloadState() != Status.DOWNLOADED) { s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); @@ -708,6 +731,12 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); return null; } + result = _s3Mgr.downloadTemplateFromS3ToSecondaryStorage(dcId, + templateId, _primaryStorageDownloadWait); + if (result != null) { + s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); + return null; + } templateHostRef = _storageMgr.findVmTemplateHost(templateId, pool); if (templateHostRef == null || templateHostRef.getDownloadState() != Status.DOWNLOADED) { s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); @@ -823,6 +852,12 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe if (_swiftMgr.isSwiftEnabled()) { throw new CloudRuntimeException("copytemplate API is disabled in Swift setup, templates in Swift can be accessed by all Zones"); } + + if (_s3Mgr.isS3Enabled()) { + throw new CloudRuntimeException( + "copytemplate API is disabled in S3 setup -- S3 templates are accessible in all zones."); + } + //Verify parameters if (sourceZoneId == destZoneId) { throw new InvalidParameterValueException("Please specify different source and destination zones."); @@ -1003,12 +1038,32 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe @Override public boolean start() { _swiftTemplateSyncExecutor.scheduleAtFixedRate(getSwiftTemplateSyncTask(), 60, 60, TimeUnit.SECONDS); + + if (_s3TemplateSyncExecutor != null) { + + final int initialDelay = 60; + final int period = 60; + + _s3TemplateSyncExecutor.scheduleAtFixedRate(new S3SyncTask( + this._tmpltDao, this._s3Mgr), initialDelay, period, + TimeUnit.SECONDS); + s_logger.info(String.format("Started S3 sync task to execute " + + "execute every %1$s after an initial delay of %2$s.", + period, initialDelay)); + + } + return true; } @Override public boolean stop() { _swiftTemplateSyncExecutor.shutdownNow(); + + if (_s3TemplateSyncExecutor != null) { + _s3TemplateSyncExecutor.shutdownNow(); + } + return true; } @@ -1041,7 +1096,16 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe _storagePoolMaxWaitSeconds = NumbersUtil.parseInt(_configDao.getValue(Config.StoragePoolMaxWaitSeconds.key()), 3600); _preloadExecutor = Executors.newFixedThreadPool(8, new NamedThreadFactory("Template-Preloader")); _swiftTemplateSyncExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("swift-template-sync-Executor")); - return false; + + if (_s3Mgr.isS3Enabled()) { + _s3TemplateSyncExecutor = Executors + .newSingleThreadScheduledExecutor(new NamedThreadFactory( + "s3-template-sync")); + } else { + s_logger.info("S3 secondary storage synchronization is disabled."); + } + + return false; } protected TemplateManagerImpl() { @@ -1195,13 +1259,19 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe if (cmd.getZoneId() == null && _swiftMgr.isSwiftEnabled()) { _swiftMgr.deleteTemplate(cmd); } + if (cmd.getZoneId() == null && _s3Mgr.isS3Enabled()) { + _s3Mgr.deleteTemplate(cmd.getId(), caller.getAccountId()); + } + TemplateAdapter adapter = getAdapter(template.getHypervisorType()); TemplateProfile profile = adapter.prepareDelete(cmd); boolean result = adapter.delete(profile); if (result){ - if (cmd.getZoneId() == null && _swiftMgr.isSwiftEnabled()) { - List templateZones = _tmpltZoneDao.listByZoneTemplate(null, templateId); + if (cmd.getZoneId() == null + && (_swiftMgr.isSwiftEnabled() || _s3Mgr.isS3Enabled())) { + List templateZones = _tmpltZoneDao + .listByZoneTemplate(null, templateId); if (templateZones != null) { for (VMTemplateZoneVO templateZone : templateZones) { _tmpltZoneDao.remove(templateZone.getId()); @@ -1234,6 +1304,10 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe if (cmd.getZoneId() == null && _swiftMgr.isSwiftEnabled()) { _swiftMgr.deleteIso(cmd); } + if (cmd.getZoneId() == null && _s3Mgr.isS3Enabled()) { + _s3Mgr.deleteTemplate(caller.getAccountId(), templateId); + } + if (zoneId != null && (_ssvmMgr.findSecondaryStorageHost(zoneId) == null)) { throw new InvalidParameterValueException("Failed to find a secondary storage host in the specified zone."); } @@ -1241,8 +1315,10 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe TemplateProfile profile = adapter.prepareDelete(cmd); boolean result = adapter.delete(profile); if (result) { - if (cmd.getZoneId() == null && _swiftMgr.isSwiftEnabled()) { - List templateZones = _tmpltZoneDao.listByZoneTemplate(null, templateId); + if (cmd.getZoneId() == null + && (_swiftMgr.isSwiftEnabled() || _s3Mgr.isS3Enabled())) { + List templateZones = _tmpltZoneDao + .listByZoneTemplate(null, templateId); if (templateZones != null) { for (VMTemplateZoneVO templateZone : templateZones) { _tmpltZoneDao.remove(templateZone.getId()); diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index b0457d7ef3e..8010e3b88c5 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -146,6 +146,8 @@ DROP TABLE IF EXISTS `cloud`.`s2s_vpn_gateway`; DROP TABLE IF EXISTS `cloud`.`s2s_vpn_connection`; DROP TABLE IF EXISTS `cloud`,`external_nicira_nvp_devices`; DROP TABLE IF EXISTS `cloud`,`nicira_nvp_nic_map`; +DROP TABLE IF EXISTS `cloud`,`s3`; +DROP TABLE IF EXISTS `cloud`,`template_s3_ref`; DROP TABLE IF EXISTS `cloud`,`nicira_nvp_router_map`; DROP TABLE IF EXISTS `cloud`.`autoscale_vmgroup_policy_map`; DROP TABLE IF EXISTS `cloud`.`autoscale_policy_condition_map`; @@ -164,7 +166,7 @@ CREATE TABLE `cloud`.`version` ( INDEX `i_version__version`(`version`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -INSERT INTO `version` (`version`, `updated`, `step`) VALUES('@VERSION@', now(), 'Complete'); +INSERT INTO `version` (`version`, `updated`, `step`) VALUES('4.0.0.2012-09-12T14:47:37Z', now(), 'Complete'); CREATE TABLE `cloud`.`op_it_work` ( `id` char(40) COMMENT 'reservation id', @@ -480,12 +482,14 @@ CREATE TABLE `cloud`.`snapshots` ( `removed` datetime COMMENT 'Date removed. not null if removed', `backup_snap_id` varchar(255) COMMENT 'Back up uuid of the snapshot', `swift_id` bigint unsigned COMMENT 'which swift', + `s3_id` bigint unsigned COMMENT 'S3 to which this snapshot will be stored', `sechost_id` bigint unsigned COMMENT 'secondary storage host id', `prev_snap_id` bigint unsigned COMMENT 'Id of the most recent snapshot', `hypervisor_type` varchar(32) NOT NULL COMMENT 'hypervisor that the snapshot was taken under', `version` varchar(32) COMMENT 'snapshot version', PRIMARY KEY (`id`), CONSTRAINT `uc_snapshots__uuid` UNIQUE (`uuid`), + CONSTRAINT `fk_snapshots__s3_id` FOREIGN KEY `fk_snapshots__s3_id` (`s3_id`) REFERENCES `s3` (`id`), INDEX `i_snapshots__removed`(`removed`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -1850,6 +1854,36 @@ CREATE TABLE `cloud`.`swift` ( CONSTRAINT `uc_swift__uuid` UNIQUE (`uuid`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; +CREATE TABLE `cloud`.`s3` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `uuid` varchar(40), + `access_key` varchar(20) NOT NULL COMMENT ' The S3 access key', + `secret_key` varchar(40) NOT NULL COMMENT ' The S3 secret key', + `end_point` varchar(1024) COMMENT ' The S3 host', + `bucket` varchar(63) NOT NULL COMMENT ' The S3 host', + `https` tinyint unsigned DEFAULT NULL COMMENT ' Flag indicating whether or not to connect over HTTPS', + `connection_timeout` integer COMMENT ' The amount of time to wait (in milliseconds) when initially establishing a connection before giving up and timing out.', + `max_error_retry` integer COMMENT ' The maximum number of retry attempts for failed retryable requests (ex: 5xx error responses from services).', + `socket_timeout` integer COMMENT ' The amount of time to wait (in milliseconds) for data to be transfered over an established, open connection before the connection times out and is closed.', + `created` datetime COMMENT 'date the s3 first signed on', + PRIMARY KEY (`id`), + CONSTRAINT `uc_s3__uuid` UNIQUE (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `cloud`.`template_s3_ref` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `s3_id` bigint unsigned NOT NULL COMMENT ' Associated S3 instance id', + `template_id` bigint unsigned NOT NULL COMMENT ' Associated template id', + `created` DATETIME NOT NULL COMMENT ' The creation timestamp', + `size` bigint unsigned COMMENT ' The size of the object', + `physical_size` bigint unsigned DEFAULT 0 COMMENT ' The physical size of the object', + PRIMARY KEY (`id`), + CONSTRAINT `uc_template_s3_ref__template_id` UNIQUE (`template_id`), + CONSTRAINT `fk_template_s3_ref__s3_id` FOREIGN KEY `fk_template_s3_ref__s3_id` (`s3_id`) REFERENCES `s3` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_template_s3_ref__template_id` FOREIGN KEY `fk_template_s3_ref__template_id` (`template_id`) REFERENCES `vm_template` (`id`), + INDEX `i_template_s3_ref__s3_id`(`s3_id`), + INDEX `i_template_s3_ref__template_id`(`template_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`op_host_transfer` ( `id` bigint unsigned UNIQUE NOT NULL COMMENT 'Id of the host', diff --git a/setup/db/db/schema-40to410.sql b/setup/db/db/schema-40to410.sql new file mode 100644 index 00000000000..b0301d72538 --- /dev/null +++ b/setup/db/db/schema-40to410.sql @@ -0,0 +1,58 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.0.0 to 4.1.0; +--; + +CREATE TABLE `cloud`.`s3` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `uuid` varchar(40), + `access_key` varchar(20) NOT NULL COMMENT ' The S3 access key', + `secret_key` varchar(40) NOT NULL COMMENT ' The S3 secret key', + `end_point` varchar(1024) COMMENT ' The S3 host', + `bucket` varchar(63) NOT NULL COMMENT ' The S3 host', + `https` tinyint unsigned DEFAULT NULL COMMENT ' Flag indicating whether or not to connect over HTTPS', + `connection_timeout` integer COMMENT ' The amount of time to wait (in milliseconds) when initially establishing a connection before giving up and timing out.', + `max_error_retry` integer COMMENT ' The maximum number of retry attempts for failed retryable requests (ex: 5xx error responses from services).', + `socket_timeout` integer COMMENT ' The amount of time to wait (in milliseconds) for data to be transfered over an established, open connection before the connection times out and is closed.', + `created` datetime COMMENT 'date the s3 first signed on', + PRIMARY KEY (`id`), + CONSTRAINT `uc_s3__uuid` UNIQUE (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `cloud`.`template_s3_ref` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `s3_id` bigint unsigned NOT NULL COMMENT ' Associated S3 instance id', + `template_id` bigint unsigned NOT NULL COMMENT ' Associated template id', + `created` DATETIME NOT NULL COMMENT ' The creation timestamp', + `size` bigint unsigned COMMENT ' The size of the object', + `physical_size` bigint unsigned DEFAULT 0 COMMENT ' The physical size of the object', + PRIMARY KEY (`id`), + CONSTRAINT `uc_template_s3_ref__template_id` UNIQUE (`template_id`), + CONSTRAINT `fk_template_s3_ref__s3_id` FOREIGN KEY `fk_template_s3_ref__s3_id` (`s3_id`) REFERENCES `s3` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_template_s3_ref__template_id` FOREIGN KEY `fk_template_s3_ref__template_id` (`template_id`) REFERENCES `vm_template` (`id`), + INDEX `i_template_s3_ref__swift_id`(`s3_id`), + INDEX `i_template_s3_ref__template_id`(`template_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.enable', 'false', 'enable s3'); + +ALTER TABLE `cloud`.`snapshots` ADD COLUMN `s3_id` bigint unsigned COMMENT 'S3 to which this snapshot will be stored'; + +ALTER TABLE `cloud`.`snapshots` ADD CONSTRAINT `fk_snapshots__s3_id` FOREIGN KEY `fk_snapshots__s3_id` (`s3_id`) REFERENCES `s3` (`id`); + diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index eeaf2a29cc2..abff8d15980 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -116,6 +116,7 @@ known_categories = { 'LB': 'Load Balancer', 'ldap': 'LDAP', 'Swift': 'Swift', + 'S3' : 'S3', 'SecondaryStorage': 'Host', 'Project': 'Project', 'Lun': 'Storage', diff --git a/tools/marvin/marvin/cloudstackConnection.py b/tools/marvin/marvin/cloudstackConnection.py index c8052130fe6..8c4e3251690 100644 --- a/tools/marvin/marvin/cloudstackConnection.py +++ b/tools/marvin/marvin/cloudstackConnection.py @@ -42,7 +42,7 @@ class cloudConnection(object): else: self.protocol=protocol self.path = path - if port == 8096: + if port == 8096 or (self.apiKey == None and self.securityKey == None): self.auth = False else: self.auth = True diff --git a/tools/marvin/marvin/deployDataCenter.py b/tools/marvin/marvin/deployDataCenter.py index bdf08cc2d2d..01235fe4027 100644 --- a/tools/marvin/marvin/deployDataCenter.py +++ b/tools/marvin/marvin/deployDataCenter.py @@ -399,9 +399,9 @@ class deployDataCenters(): logging=self.testClientLogger) """config database""" - dbSvr = self.config.dbSvr - self.testClient.dbConfigure(dbSvr.dbSvr, dbSvr.port, dbSvr.user, \ - dbSvr.passwd, dbSvr.db) + #dbSvr = self.config.dbSvr + #self.testClient.dbConfigure(dbSvr.dbSvr, dbSvr.port, dbSvr.user, \ + # dbSvr.passwd, dbSvr.db) self.apiClient = self.testClient.getApiClient() def updateConfiguration(self, globalCfg): diff --git a/ui/dictionary.jsp b/ui/dictionary.jsp index e72481eeefc..20b35bded86 100644 --- a/ui/dictionary.jsp +++ b/ui/dictionary.jsp @@ -25,6 +25,17 @@ under the License. <% long now = System.currentTimeMillis(); %>