diff --git a/.gitignore b/.gitignore index 33f95c7902c..9511e9ede2f 100644 --- a/.gitignore +++ b/.gitignore @@ -67,8 +67,10 @@ awsapi/modules/* .settings/ db.properties.override awsapi/overlays/ +tools/marvin/marvin/cloudstackAPI/* *.egg-info/ docs/tmp docs/publish docs/runbook/tmp docs/runbook/publish +Gemfile.lock diff --git a/api/src/com/cloud/agent/api/BackupSnapshotCommand.java b/api/src/com/cloud/agent/api/BackupSnapshotCommand.java index 9476d7d7065..7ea78356837 100644 --- a/api/src/com/cloud/agent/api/BackupSnapshotCommand.java +++ b/api/src/com/cloud/agent/api/BackupSnapshotCommand.java @@ -17,6 +17,7 @@ package com.cloud.agent.api; import com.cloud.agent.api.LogLevel.Log4jLevel; +import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.storage.StoragePool; @@ -32,6 +33,7 @@ public class BackupSnapshotCommand extends SnapshotCommand { private Long snapshotId; @LogLevel(Log4jLevel.Off) private SwiftTO swift; + private S3TO s3; StorageFilerTO pool; protected BackupSnapshotCommand() { @@ -48,8 +50,7 @@ public class BackupSnapshotCommand extends SnapshotCommand { * @param isFirstSnapshotOfRootVolume true if this is the first snapshot of a root volume. Set the parent of the backup to null. * @param isVolumeInactive True if the volume belongs to a VM that is not running or is detached. */ - public BackupSnapshotCommand(String primaryStoragePoolNameLabel, - String secondaryStoragePoolURL, + public BackupSnapshotCommand(String secondaryStoragePoolURL, Long dcId, Long accountId, Long volumeId, @@ -64,17 +65,16 @@ public class BackupSnapshotCommand extends SnapshotCommand { String vmName, int wait) { - super(primaryStoragePoolNameLabel, secondaryStoragePoolURL, snapshotUuid, snapshotName, dcId, accountId, volumeId); + super(pool, secondaryStoragePoolURL, snapshotUuid, snapshotName, dcId, accountId, volumeId); this.snapshotId = snapshotId; this.prevSnapshotUuid = prevSnapshotUuid; this.prevBackupUuid = prevBackupUuid; this.isVolumeInactive = isVolumeInactive; this.vmName = vmName; - this.pool = new StorageFilerTO(pool); setVolumePath(volumePath); setWait(wait); } - + public String getPrevSnapshotUuid() { return prevSnapshotUuid; } @@ -88,7 +88,7 @@ public class BackupSnapshotCommand extends SnapshotCommand { } public String getVmName() { - return vmName; + return vmName; } public SwiftTO getSwift() { @@ -99,11 +99,16 @@ public class BackupSnapshotCommand extends SnapshotCommand { this.swift = swift; } + public S3TO getS3() { + return s3; + } + + public void setS3(S3TO s3) { + this.s3 = s3; + } + public Long getSnapshotId() { return snapshotId; } - public StorageFilerTO getPool() { - return pool; - } -} \ No newline at end of file +} diff --git a/api/src/com/cloud/agent/api/CreatePrivateTemplateFromSnapshotCommand.java b/api/src/com/cloud/agent/api/CreatePrivateTemplateFromSnapshotCommand.java index bf6f2604cdd..0140a4e0a65 100644 --- a/api/src/com/cloud/agent/api/CreatePrivateTemplateFromSnapshotCommand.java +++ b/api/src/com/cloud/agent/api/CreatePrivateTemplateFromSnapshotCommand.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.agent.api; +import com.cloud.storage.StoragePool; + /** * This currently assumes that both primary and secondary storage are mounted on the XenServer. */ @@ -40,7 +42,8 @@ public class CreatePrivateTemplateFromSnapshotCommand extends SnapshotCommand { * It may not be the UUID of the base copy of the snapshot, if no data was written since last snapshot. * @param origTemplateInstallPath The install path of the original template VHD on the secondary */ - public CreatePrivateTemplateFromSnapshotCommand(String primaryStoragePoolNameLabel, + + public CreatePrivateTemplateFromSnapshotCommand(StoragePool pool, String secondaryStoragePoolURL, Long dcId, Long accountId, @@ -52,7 +55,7 @@ public class CreatePrivateTemplateFromSnapshotCommand extends SnapshotCommand { String templateName, int wait) { - super(primaryStoragePoolNameLabel, secondaryStoragePoolURL, backedUpSnapshotUuid, backedUpSnapshotName, dcId, accountId, volumeId); + super(pool, secondaryStoragePoolURL, backedUpSnapshotUuid, backedUpSnapshotName, dcId, accountId, volumeId); this.origTemplateInstallPath = origTemplateInstallPath; this.newTemplateId = newTemplateId; this.templateName = templateName; @@ -76,4 +79,4 @@ public class CreatePrivateTemplateFromSnapshotCommand extends SnapshotCommand { public String getTemplateName() { return templateName; } -} \ No newline at end of file +} diff --git a/api/src/com/cloud/agent/api/CreatePrivateTemplateFromVolumeCommand.java b/api/src/com/cloud/agent/api/CreatePrivateTemplateFromVolumeCommand.java index b589ed53077..4e0bf32f1b1 100644 --- a/api/src/com/cloud/agent/api/CreatePrivateTemplateFromVolumeCommand.java +++ b/api/src/com/cloud/agent/api/CreatePrivateTemplateFromVolumeCommand.java @@ -16,6 +16,9 @@ // under the License. package com.cloud.agent.api; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.storage.StoragePool; + public class CreatePrivateTemplateFromVolumeCommand extends SnapshotCommand { private String _vmName; private String _volumePath; @@ -23,13 +26,14 @@ public class CreatePrivateTemplateFromVolumeCommand extends SnapshotCommand { private String _uniqueName; private long _templateId; private long _accountId; + StorageFilerTO _primaryPool; // For XenServer private String _secondaryStorageUrl; public CreatePrivateTemplateFromVolumeCommand() { } - public CreatePrivateTemplateFromVolumeCommand(String StoragePoolUUID, String secondaryStorageUrl, long templateId, long accountId, String userSpecifiedName, String uniqueName, String volumePath, String vmName, int wait) { + public CreatePrivateTemplateFromVolumeCommand(StoragePool pool, String secondaryStorageUrl, long templateId, long accountId, String userSpecifiedName, String uniqueName, String volumePath, String vmName, int wait) { _secondaryStorageUrl = secondaryStorageUrl; _templateId = templateId; _accountId = accountId; @@ -37,7 +41,8 @@ public class CreatePrivateTemplateFromVolumeCommand extends SnapshotCommand { _uniqueName = uniqueName; _volumePath = volumePath; _vmName = vmName; - primaryStoragePoolNameLabel = StoragePoolUUID; + primaryStoragePoolNameLabel = pool.getUuid(); + _primaryPool = new StorageFilerTO(pool); setWait(wait); } @@ -46,6 +51,10 @@ public class CreatePrivateTemplateFromVolumeCommand extends SnapshotCommand { return false; } + public StorageFilerTO getPool() { + return _primaryPool; + } + public String getSecondaryStorageUrl() { return _secondaryStorageUrl; } diff --git a/api/src/com/cloud/agent/api/CreateVolumeFromSnapshotCommand.java b/api/src/com/cloud/agent/api/CreateVolumeFromSnapshotCommand.java index f0063299415..9bc9b36ae0c 100644 --- a/api/src/com/cloud/agent/api/CreateVolumeFromSnapshotCommand.java +++ b/api/src/com/cloud/agent/api/CreateVolumeFromSnapshotCommand.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.agent.api; +import com.cloud.storage.StoragePool; + /** * This currently assumes that both primary and secondary storage are mounted on the XenServer. */ @@ -39,7 +41,8 @@ public class CreateVolumeFromSnapshotCommand extends SnapshotCommand { * It may not be the UUID of the base copy of the snapshot, if no data was written since last snapshot. * @param templatePath The install path of the template VHD on the secondary, if this a root volume */ - public CreateVolumeFromSnapshotCommand(String primaryStoragePoolNameLabel, + + public CreateVolumeFromSnapshotCommand(StoragePool pool, String secondaryStoragePoolURL, Long dcId, Long accountId, @@ -48,7 +51,7 @@ public class CreateVolumeFromSnapshotCommand extends SnapshotCommand { String backedUpSnapshotName, int wait) { - super(primaryStoragePoolNameLabel, secondaryStoragePoolURL, backedUpSnapshotUuid, backedUpSnapshotName, dcId, accountId, volumeId); + super(pool, secondaryStoragePoolURL, backedUpSnapshotUuid, backedUpSnapshotName, dcId, accountId, volumeId); setWait(wait); } -} \ No newline at end of file +} diff --git a/api/src/com/cloud/agent/api/DeleteSnapshotBackupCommand.java b/api/src/com/cloud/agent/api/DeleteSnapshotBackupCommand.java index 3fa8c2b7eb4..5e5557002e6 100644 --- a/api/src/com/cloud/agent/api/DeleteSnapshotBackupCommand.java +++ b/api/src/com/cloud/agent/api/DeleteSnapshotBackupCommand.java @@ -17,6 +17,7 @@ package com.cloud.agent.api; import com.cloud.agent.api.LogLevel.Log4jLevel; +import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; /** @@ -26,6 +27,7 @@ import com.cloud.agent.api.to.SwiftTO; public class DeleteSnapshotBackupCommand extends SnapshotCommand { @LogLevel(Log4jLevel.Off) private SwiftTO swift; + private S3TO s3; private Boolean all; public SwiftTO getSwift() { @@ -44,6 +46,10 @@ public class DeleteSnapshotBackupCommand extends SnapshotCommand { this.swift = swift; } + public S3TO getS3() { + return s3; + } + protected DeleteSnapshotBackupCommand() { } @@ -73,6 +79,7 @@ public class DeleteSnapshotBackupCommand extends SnapshotCommand { * @param childUUID The child VHD file of the backup whose parent is reset to its grandparent. */ public DeleteSnapshotBackupCommand(SwiftTO swift, + S3TO s3, String secondaryStoragePoolURL, Long dcId, Long accountId, @@ -81,6 +88,7 @@ public class DeleteSnapshotBackupCommand extends SnapshotCommand { { super(null, secondaryStoragePoolURL, backupUUID, null, dcId, accountId, volumeId); setSwift(swift); + this.s3 = s3; setAll(all); } } \ No newline at end of file diff --git a/api/src/com/cloud/agent/api/DeleteTemplateFromS3Command.java b/api/src/com/cloud/agent/api/DeleteTemplateFromS3Command.java new file mode 100644 index 00000000000..278669b2c97 --- /dev/null +++ b/api/src/com/cloud/agent/api/DeleteTemplateFromS3Command.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.agent.api; + +import com.cloud.agent.api.to.S3TO; + +public class DeleteTemplateFromS3Command extends Command { + + private S3TO s3; + private Long templateId; + private Long accountId; + + protected DeleteTemplateFromS3Command() { + super(); + } + + public DeleteTemplateFromS3Command(final S3TO s3, final Long accountId, + final Long templateId) { + + super(); + + this.s3 = s3; + this.accountId = accountId; + this.templateId = templateId; + + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((accountId == null) ? 0 : accountId.hashCode()); + result = prime * result + ((s3 == null) ? 0 : s3.hashCode()); + result = prime * result + + ((templateId == null) ? 0 : templateId.hashCode()); + return result; + } + + @Override + public boolean equals(Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null) { + return false; + } + + if (getClass() != thatObject.getClass()) { + return false; + } + + final DeleteTemplateFromS3Command thatCommand = (DeleteTemplateFromS3Command) thatObject; + + if (!(accountId == thatCommand.accountId) + || (this.accountId != null && this.accountId + .equals(thatCommand.accountId))) { + return false; + } + + if (!(templateId == thatCommand.templateId) + || (this.templateId != null && this.templateId + .equals(thatCommand.templateId))) { + return false; + } + + return true; + + } + + public S3TO getS3() { + return s3; + } + + public Long getTemplateId() { + return templateId; + } + + public Long getAccountId() { + return accountId; + } + + @Override + public boolean executeInSequence() { + return true; + } + +} diff --git a/api/src/com/cloud/agent/api/DownloadSnapshotFromS3Command.java b/api/src/com/cloud/agent/api/DownloadSnapshotFromS3Command.java new file mode 100644 index 00000000000..edf683a58be --- /dev/null +++ b/api/src/com/cloud/agent/api/DownloadSnapshotFromS3Command.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.agent.api; + +import com.cloud.agent.api.to.S3TO; + +public class DownloadSnapshotFromS3Command extends SnapshotCommand { + + private S3TO s3; + private String parent; + + protected DownloadSnapshotFromS3Command() { + super(); + } + + public DownloadSnapshotFromS3Command(S3TO s3, String parent, + String secondaryStorageUrl, Long dcId, Long accountId, + Long volumeId, String backupUuid, int wait) { + + super(null, secondaryStorageUrl, backupUuid, "", dcId, accountId, + volumeId); + + this.s3 = s3; + this.parent = parent; + setWait(wait); + + } + + public S3TO getS3() { + return s3; + } + + public void setS3(S3TO s3) { + this.s3 = s3; + } + + public String getParent() { + return parent; + } + + public void setParent(String parent) { + this.parent = parent; + } + +} diff --git a/api/src/com/cloud/agent/api/DownloadTemplateFromS3ToSecondaryStorageCommand.java b/api/src/com/cloud/agent/api/DownloadTemplateFromS3ToSecondaryStorageCommand.java new file mode 100644 index 00000000000..af61228c020 --- /dev/null +++ b/api/src/com/cloud/agent/api/DownloadTemplateFromS3ToSecondaryStorageCommand.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.agent.api; + +import com.cloud.agent.api.to.S3TO; + +public final class DownloadTemplateFromS3ToSecondaryStorageCommand extends Command { + + private final S3TO s3; + private final Long accountId; + private final Long templateId; + private final String storagePath; + + public DownloadTemplateFromS3ToSecondaryStorageCommand(final S3TO s3, + final Long accountId, final Long templateId, + final String storagePath, final int wait) { + + super(); + + this.s3 = s3; + this.accountId = accountId; + this.templateId = templateId; + this.storagePath = storagePath; + + setWait(wait); + + } + + public S3TO getS3() { + return this.s3; + } + + public Long getAccountId() { + return this.accountId; + } + + public Long getTemplateId() { + return this.templateId; + } + + public String getStoragePath() { + return this.storagePath; + } + + @Override + public boolean executeInSequence() { + return true; + } + +} diff --git a/api/src/com/cloud/agent/api/SnapshotCommand.java b/api/src/com/cloud/agent/api/SnapshotCommand.java index 892db37c0d7..d8abeb62371 100644 --- a/api/src/com/cloud/agent/api/SnapshotCommand.java +++ b/api/src/com/cloud/agent/api/SnapshotCommand.java @@ -16,12 +16,16 @@ // under the License. package com.cloud.agent.api; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.storage.StoragePool; + /** * This currently assumes that both primary and secondary storage are mounted on * the XenServer. */ public class SnapshotCommand extends Command { protected String primaryStoragePoolNameLabel; + StorageFilerTO primaryPool; private String snapshotUuid; private String snapshotName; private String secondaryStorageUrl; @@ -46,10 +50,11 @@ public class SnapshotCommand extends Command { * is the value of that field If you have better ideas on how to * get it, you are welcome. */ - public SnapshotCommand(String primaryStoragePoolNameLabel, + public SnapshotCommand(StoragePool pool, String secondaryStorageUrl, String snapshotUuid, String snapshotName, Long dcId, Long accountId, Long volumeId) { - this.primaryStoragePoolNameLabel = primaryStoragePoolNameLabel; + this.primaryStoragePoolNameLabel = pool.getUuid(); + this.primaryPool = new StorageFilerTO(pool); this.snapshotUuid = snapshotUuid; this.secondaryStorageUrl = secondaryStorageUrl; this.dcId = dcId; @@ -65,6 +70,13 @@ public class SnapshotCommand extends Command { return primaryStoragePoolNameLabel; } + /** + * @return the primaryPool + */ + public StorageFilerTO getPool() { + return primaryPool; + } + /** * @return the snapshotUuid */ @@ -111,4 +123,4 @@ public class SnapshotCommand extends Command { return false; } -} \ No newline at end of file +} diff --git a/api/src/com/cloud/agent/api/UpgradeSnapshotCommand.java b/api/src/com/cloud/agent/api/UpgradeSnapshotCommand.java index e4517686491..703aea06be0 100644 --- a/api/src/com/cloud/agent/api/UpgradeSnapshotCommand.java +++ b/api/src/com/cloud/agent/api/UpgradeSnapshotCommand.java @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. package com.cloud.agent.api; +import com.cloud.storage.StoragePool; + public class UpgradeSnapshotCommand extends SnapshotCommand { private String version; private Long templateId; @@ -30,7 +32,7 @@ public class UpgradeSnapshotCommand extends SnapshotCommand { * @param snapshotUuid The UUID of the snapshot which is going to be upgraded * @param _version version for this snapshot */ - public UpgradeSnapshotCommand(String primaryStoragePoolNameLabel, + public UpgradeSnapshotCommand(StoragePool pool, String secondaryStoragePoolURL, Long dcId, Long accountId, @@ -42,7 +44,7 @@ public class UpgradeSnapshotCommand extends SnapshotCommand { String snapshotName, String version) { - super(primaryStoragePoolNameLabel, secondaryStoragePoolURL, snapshotUuid, snapshotName, dcId, accountId, volumeId); + super(pool, secondaryStoragePoolURL, snapshotUuid, snapshotName, dcId, accountId, volumeId); this.version = version; this.templateId = templateId; this.tmpltAccountId = tmpltAccountId; diff --git a/api/src/com/cloud/agent/api/UploadTemplateToS3FromSecondaryStorageCommand.java b/api/src/com/cloud/agent/api/UploadTemplateToS3FromSecondaryStorageCommand.java new file mode 100644 index 00000000000..1807cd56315 --- /dev/null +++ b/api/src/com/cloud/agent/api/UploadTemplateToS3FromSecondaryStorageCommand.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.agent.api; + +import com.cloud.agent.api.to.S3TO; + +public class UploadTemplateToS3FromSecondaryStorageCommand extends Command { + + private final S3TO s3; + private final String storagePath; + private final Long dataCenterId; + private final Long accountId; + private final Long templateId; + + public UploadTemplateToS3FromSecondaryStorageCommand(final S3TO s3, + final String storagePath, final Long dataCenterId, final Long accountId, + final Long templateId) { + + super(); + + this.s3 = s3; + this.storagePath = storagePath; + this.dataCenterId = dataCenterId; + this.accountId = accountId; + this.templateId = templateId; + + } + + @Override + public boolean executeInSequence() { + return false; + } + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null || getClass() != thatObject.getClass()) { + return false; + } + + final UploadTemplateToS3FromSecondaryStorageCommand thatCommand = + (UploadTemplateToS3FromSecondaryStorageCommand) thatObject; + + if (this.accountId != null ? !this.accountId.equals(thatCommand + .accountId) : thatCommand.accountId != null) { + return false; + } + + if (this.dataCenterId != null ? !this.dataCenterId.equals(thatCommand + .dataCenterId) : thatCommand.dataCenterId != null) { + return false; + } + + if (this.s3 != null ? !this.s3.equals(thatCommand.s3) : thatCommand.s3 != null) { + return false; + } + + if (this.storagePath != null ? !this.storagePath.equals(thatCommand + .storagePath) : thatCommand.storagePath != null) { + return false; + } + + if (this.templateId != null ? !this.templateId.equals(thatCommand.templateId) : + thatCommand.templateId != null) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + int result = this.s3 != null ? this.s3.hashCode() : 0; + result = 31 * result + (this.storagePath != null ? this.storagePath.hashCode() : 0); + result = 31 * result + (this.dataCenterId != null ? this.dataCenterId.hashCode() : 0); + result = 31 * result + (this.accountId != null ? this.accountId.hashCode() : 0); + result = 31 * result + (this.templateId != null ? this.templateId.hashCode() : 0); + return result; + } + + public S3TO getS3() { + return this.s3; + } + + public String getStoragePath() { + return this.storagePath; + } + + public Long getDataCenterId() { + return this.dataCenterId; + } + + public Long getAccountId() { + return this.accountId; + } + + public Long getTemplateId() { + return this.templateId; + } + +} diff --git a/api/src/com/cloud/agent/api/downloadSnapshotFromSwiftCommand.java b/api/src/com/cloud/agent/api/downloadSnapshotFromSwiftCommand.java index 0586f8cdcfd..6021762dcf0 100644 --- a/api/src/com/cloud/agent/api/downloadSnapshotFromSwiftCommand.java +++ b/api/src/com/cloud/agent/api/downloadSnapshotFromSwiftCommand.java @@ -34,7 +34,7 @@ public class downloadSnapshotFromSwiftCommand extends SnapshotCommand { public downloadSnapshotFromSwiftCommand(SwiftTO swift, String secondaryStorageUrl, Long dcId, Long accountId, Long volumeId, String parent, String BackupUuid, int wait) { - super("", secondaryStorageUrl, BackupUuid, "", dcId, accountId, volumeId); + super(null, secondaryStorageUrl, BackupUuid, "", dcId, accountId, volumeId); setParent(parent); setSwift(swift); setWait(wait); @@ -57,4 +57,4 @@ public class downloadSnapshotFromSwiftCommand extends SnapshotCommand { this._parent = parent; } -} \ No newline at end of file +} diff --git a/api/src/com/cloud/agent/api/storage/PrimaryStorageDownloadCommand.java b/api/src/com/cloud/agent/api/storage/PrimaryStorageDownloadCommand.java index b56317727e2..f66d00eb8cb 100644 --- a/api/src/com/cloud/agent/api/storage/PrimaryStorageDownloadCommand.java +++ b/api/src/com/cloud/agent/api/storage/PrimaryStorageDownloadCommand.java @@ -17,7 +17,8 @@ package com.cloud.agent.api.storage; import com.cloud.storage.Storage.ImageFormat; - +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.storage.StoragePool; /** * @@ -26,6 +27,7 @@ public class PrimaryStorageDownloadCommand extends AbstractDownloadCommand { String localPath; String poolUuid; long poolId; + StorageFilerTO primaryPool; String secondaryStorageUrl; String primaryStorageUrl; @@ -33,10 +35,11 @@ public class PrimaryStorageDownloadCommand extends AbstractDownloadCommand { protected PrimaryStorageDownloadCommand() { } - public PrimaryStorageDownloadCommand(String name, String url, ImageFormat format, long accountId, long poolId, String poolUuid, int wait) { + public PrimaryStorageDownloadCommand(String name, String url, ImageFormat format, long accountId, StoragePool pool, int wait) { super(name, url, format, accountId); - this.poolId = poolId; - this.poolUuid = poolUuid; + this.poolId = pool.getId(); + this.poolUuid = pool.getUuid(); + this.primaryPool = new StorageFilerTO(pool); setWait(wait); } @@ -47,6 +50,10 @@ public class PrimaryStorageDownloadCommand extends AbstractDownloadCommand { public long getPoolId() { return poolId; } + + public StorageFilerTO getPool() { + return primaryPool; + } public void setLocalPath(String path) { this.localPath = path; diff --git a/api/src/com/cloud/agent/api/to/S3TO.java b/api/src/com/cloud/agent/api/to/S3TO.java new file mode 100644 index 00000000000..879df229c31 --- /dev/null +++ b/api/src/com/cloud/agent/api/to/S3TO.java @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.to; + +import com.cloud.utils.S3Utils; + +import java.util.Date; + +public final class S3TO implements S3Utils.ClientOptions { + + private Long id; + private String uuid; + private String accessKey; + private String secretKey; + private String endPoint; + private String bucketName; + private Boolean httpsFlag; + private Integer connectionTimeout; + private Integer maxErrorRetry; + private Integer socketTimeout; + private Date created; + + public S3TO() { + + super(); + + } + + public S3TO(final Long id, final String uuid, final String accessKey, + final String secretKey, final String endPoint, + final String bucketName, final Boolean httpsFlag, + final Integer connectionTimeout, final Integer maxErrorRetry, + final Integer socketTimeout, final Date created) { + + super(); + + this.id = id; + this.uuid = uuid; + this.accessKey = accessKey; + this.secretKey = secretKey; + this.endPoint = endPoint; + this.bucketName = bucketName; + this.httpsFlag = httpsFlag; + this.connectionTimeout = connectionTimeout; + this.maxErrorRetry = maxErrorRetry; + this.socketTimeout = socketTimeout; + this.created = created; + + } + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) + return true; + if (thatObject == null || getClass() != thatObject.getClass()) + return false; + + final S3TO thatS3TO = (S3TO) thatObject; + + if (httpsFlag != null ? !httpsFlag.equals(thatS3TO.httpsFlag) + : thatS3TO.httpsFlag != null) { + return false; + } + + if (accessKey != null ? !accessKey.equals(thatS3TO.accessKey) + : thatS3TO.accessKey != null) { + return false; + } + + if (connectionTimeout != null ? !connectionTimeout + .equals(thatS3TO.connectionTimeout) + : thatS3TO.connectionTimeout != null) { + return false; + } + + if (endPoint != null ? !endPoint.equals(thatS3TO.endPoint) + : thatS3TO.endPoint != null) { + return false; + } + + if (id != null ? !id.equals(thatS3TO.id) : thatS3TO.id != null) { + return false; + } + + if (uuid != null ? !uuid.equals(thatS3TO.uuid) : thatS3TO.uuid != null) { + return false; + } + + if (maxErrorRetry != null ? !maxErrorRetry + .equals(thatS3TO.maxErrorRetry) + : thatS3TO.maxErrorRetry != null) { + return false; + } + + if (secretKey != null ? !secretKey.equals(thatS3TO.secretKey) + : thatS3TO.secretKey != null) { + return false; + } + + if (socketTimeout != null ? !socketTimeout + .equals(thatS3TO.socketTimeout) + : thatS3TO.socketTimeout != null) { + return false; + } + + if (bucketName != null ? !bucketName.equals(thatS3TO.bucketName) + : thatS3TO.bucketName != null) { + return false; + } + + if (created != null ? !created.equals(thatS3TO.created) + : thatS3TO.created != null) { + return false; + } + + return true; + + } + + @Override + public int hashCode() { + + int result = id != null ? id.hashCode() : 0; + + result = 31 * result + (accessKey != null ? accessKey.hashCode() : 0); + result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0); + result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0); + result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0); + result = 31 * result + (httpsFlag ? 1 : 0); + result = 31 + * result + + (connectionTimeout != null ? connectionTimeout.hashCode() : 0); + result = 31 * result + + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0); + result = 31 * result + + (socketTimeout != null ? socketTimeout.hashCode() : 0); + + return result; + + } + + public Long getId() { + return this.id; + } + + public void setId(final Long id) { + this.id = id; + } + + public String getUuid() { + return this.uuid; + } + + public void setUuid(final String uuid) { + this.uuid = uuid; + } + + @Override + public String getAccessKey() { + return this.accessKey; + } + + public void setAccessKey(final String accessKey) { + this.accessKey = accessKey; + } + + @Override + public String getSecretKey() { + return this.secretKey; + } + + public void setSecretKey(final String secretKey) { + this.secretKey = secretKey; + } + + @Override + public String getEndPoint() { + return this.endPoint; + } + + public void setEndPoint(final String endPoint) { + this.endPoint = endPoint; + } + + public String getBucketName() { + return this.bucketName; + } + + public void setBucketName(final String bucketName) { + this.bucketName = bucketName; + } + + @Override + public Boolean isHttps() { + return this.httpsFlag; + } + + public void setHttps(final Boolean httpsFlag) { + this.httpsFlag = httpsFlag; + } + + @Override + public Integer getConnectionTimeout() { + return connectionTimeout; + } + + public void setConnectionTimeout(final Integer connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + @Override + public Integer getMaxErrorRetry() { + return maxErrorRetry; + } + + public void setMaxErrorRetry(final Integer maxErrorRetry) { + this.maxErrorRetry = maxErrorRetry; + } + + @Override + public Integer getSocketTimeout() { + return socketTimeout; + } + + public void setSocketTimeout(final Integer socketTimeout) { + this.socketTimeout = socketTimeout; + } + + public Date getCreated() { + return this.created; + } + + public void setCreated(final Date created) { + this.created = created; + } + +} diff --git a/api/src/com/cloud/api/ApiConstants.java b/api/src/com/cloud/api/ApiConstants.java index 78a3deda578..588cdc6c140 100755 --- a/api/src/com/cloud/api/ApiConstants.java +++ b/api/src/com/cloud/api/ApiConstants.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.api; +import org.omg.CORBA.PUBLIC_MEMBER; + public class ApiConstants { public static final String ACCOUNT = "account"; public static final String ACCOUNTS = "accounts"; @@ -383,6 +385,16 @@ public class ApiConstants { public static final String NICIRA_NVP_TRANSPORT_ZONE_UUID = "transportzoneuuid"; public static final String NICIRA_NVP_DEVICE_NAME = "niciradevicename"; public static final String NICIRA_NVP_GATEWAYSERVICE_UUID = "l3gatewayserviceuuid"; + public static final String S3_ACCESS_KEY = "accesskey"; + public static final String S3_SECRET_KEY = "secretkey"; + public static final String S3_END_POINT = "endpoint"; + public static final String S3_BUCKET_NAME = "bucket"; + public static final String S3_HTTPS_FLAG = "usehttps"; + public static final String S3_CONNECTION_TIMEOUT = "connectiontimeout"; + public static final String S3_MAX_ERROR_RETRY = "maxerrorretry"; + public static final String S3_SOCKET_TIMEOUT = "sockettimeout"; + public static final String INCL_ZONES = "includezones"; + public static final String EXCL_ZONES = "excludezones"; public static final String SOURCE = "source"; public static final String COUNTER_ID = "counterid"; diff --git a/api/src/com/cloud/api/ResponseGenerator.java b/api/src/com/cloud/api/ResponseGenerator.java index 4e8fbd824a9..ec6c00ba8d4 100755 --- a/api/src/com/cloud/api/ResponseGenerator.java +++ b/api/src/com/cloud/api/ResponseGenerator.java @@ -64,6 +64,7 @@ import com.cloud.api.response.RemoteAccessVpnResponse; import com.cloud.api.response.ResourceCountResponse; import com.cloud.api.response.ResourceLimitResponse; import com.cloud.api.response.ResourceTagResponse; +import com.cloud.api.response.S3Response; import com.cloud.api.response.SecurityGroupResponse; import com.cloud.api.response.ServiceOfferingResponse; import com.cloud.api.response.ServiceResponse; @@ -141,6 +142,7 @@ import com.cloud.projects.Project; import com.cloud.projects.ProjectAccount; import com.cloud.projects.ProjectInvitation; import com.cloud.server.ResourceTag; +import com.cloud.storage.S3; import com.cloud.storage.Snapshot; import com.cloud.storage.StoragePool; import com.cloud.storage.Swift; @@ -287,6 +289,8 @@ public interface ResponseGenerator { SwiftResponse createSwiftResponse(Swift swift); + S3Response createS3Response(S3 result); + PhysicalNetworkResponse createPhysicalNetworkResponse(PhysicalNetwork result); ServiceResponse createNetworkServiceResponse(Service service); @@ -361,4 +365,5 @@ public interface ResponseGenerator { AutoScaleVmProfileResponse createAutoScaleVmProfileResponse(AutoScaleVmProfile profile); AutoScaleVmGroupResponse createAutoScaleVmGroupResponse(AutoScaleVmGroup vmGroup); + } diff --git a/api/src/com/cloud/api/commands/AddS3Cmd.java b/api/src/com/cloud/api/commands/AddS3Cmd.java new file mode 100644 index 00000000000..e046ccc0160 --- /dev/null +++ b/api/src/com/cloud/api/commands/AddS3Cmd.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.api.commands; + +import static com.cloud.api.ApiConstants.S3_ACCESS_KEY; +import static com.cloud.api.ApiConstants.S3_CONNECTION_TIMEOUT; +import static com.cloud.api.ApiConstants.S3_END_POINT; +import static com.cloud.api.ApiConstants.S3_HTTPS_FLAG; +import static com.cloud.api.ApiConstants.S3_MAX_ERROR_RETRY; +import static com.cloud.api.ApiConstants.S3_SECRET_KEY; +import static com.cloud.api.ApiConstants.S3_SOCKET_TIMEOUT; +import static com.cloud.api.ApiConstants.S3_BUCKET_NAME; +import static com.cloud.api.BaseCmd.CommandType.INTEGER; +import static com.cloud.api.BaseCmd.CommandType.STRING; +import static com.cloud.api.BaseCmd.CommandType.BOOLEAN; +import static com.cloud.user.Account.ACCOUNT_ID_SYSTEM; + +import com.cloud.api.BaseCmd; +import com.cloud.api.Implementation; +import com.cloud.api.Parameter; +import com.cloud.api.ServerApiException; +import com.cloud.api.response.S3Response; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.DiscoveryException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.storage.S3; + +@Implementation(description = "Adds S3", responseObject = S3Response.class, since = "4.0.0") +public final class AddS3Cmd extends BaseCmd { + + private static String COMMAND_NAME = "adds3response"; + + @Parameter(name = S3_ACCESS_KEY, type = STRING, required = true, + description = "S3 access key") + private String accessKey; + + @Parameter(name = S3_SECRET_KEY, type = STRING, required = true, + description = "S3 secret key") + private String secretKey; + + @Parameter(name = S3_END_POINT, type = STRING, required = false, + description = "S3 host name") + private String endPoint = null; + + @Parameter(name = S3_BUCKET_NAME, type = STRING, required = true, + description = "name of the template storage bucket") + private String bucketName; + + @Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false, + description = "connect to the S3 endpoint via HTTPS?") + private Boolean httpsFlag = null; + + @Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false, + description = "connection timeout (milliseconds)") + private Integer connectionTimeout = null; + + @Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false, + description = "maximum number of times to retry on error") + private Integer maxErrorRetry = null; + + @Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false, + description = "socket timeout (milliseconds)") + private Integer socketTimeout = null; + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, + ServerApiException, ConcurrentOperationException, ResourceAllocationException, + NetworkRuleConflictException { + + final S3 result; + + try { + + result = _resourceService.discoverS3(this); + + if (result == null) { + throw new ServerApiException(INTERNAL_ERROR, "Failed to add S3."); + } + + } catch (DiscoveryException e) { + + throw new ServerApiException(INTERNAL_ERROR, "Failed to add S3 due to " + e.getMessage()); + + } + + final S3Response response = _responseGenerator.createS3Response(result); + response.setResponseName(this.getCommandName()); + this.setResponseObject(response); + + } + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null || this.getClass() != thatObject.getClass()) { + return false; + } + + final AddS3Cmd thatAddS3Cmd = (AddS3Cmd) thatObject; + + if (this.httpsFlag != null ? !this.httpsFlag.equals(thatAddS3Cmd.httpsFlag) : thatAddS3Cmd.httpsFlag != null) { + return false; + } + + if (this.accessKey != null ? !this.accessKey.equals(thatAddS3Cmd.accessKey) : thatAddS3Cmd.accessKey != null) { + return false; + } + + if (this.connectionTimeout != null ? !this.connectionTimeout.equals(thatAddS3Cmd.connectionTimeout) : thatAddS3Cmd.connectionTimeout != null) { + return false; + } + + if (this.endPoint != null ? !this.endPoint.equals(thatAddS3Cmd.endPoint) : thatAddS3Cmd.endPoint != null) { + return false; + } + + if (this.maxErrorRetry != null ? !this.maxErrorRetry.equals(thatAddS3Cmd.maxErrorRetry) : thatAddS3Cmd.maxErrorRetry != null) { + return false; + } + + if (this.secretKey != null ? !this.secretKey.equals(thatAddS3Cmd.secretKey) : thatAddS3Cmd.secretKey != null) { + return false; + } + + if (this.socketTimeout != null ? !this.socketTimeout.equals(thatAddS3Cmd.socketTimeout) : thatAddS3Cmd.socketTimeout != null) { + return false; + } + + if (this.bucketName != null ? !this.bucketName.equals(thatAddS3Cmd.bucketName) : thatAddS3Cmd.bucketName != null) { + return false; + } + + return true; + + } + + @Override + public int hashCode() { + + int result = this.accessKey != null ? this.accessKey.hashCode() : 0; + result = 31 * result + (this.secretKey != null ? this.secretKey.hashCode() : 0); + result = 31 * result + (this.endPoint != null ? this.endPoint.hashCode() : 0); + result = 31 * result + (this.bucketName != null ? this.bucketName.hashCode() : 0); + result = 31 * result + (this.httpsFlag != null && this.httpsFlag == true ? 1 : 0); + result = 31 * result + (this.connectionTimeout != null ? this.connectionTimeout.hashCode() : 0); + result = 31 * result + (this.maxErrorRetry != null ? this.maxErrorRetry.hashCode() : 0); + result = 31 * result + (this.socketTimeout != null ? this.socketTimeout.hashCode() : 0); + + return result; + + } + + @Override + public String getCommandName() { + return COMMAND_NAME; + } + + @Override + public long getEntityOwnerId() { + return ACCOUNT_ID_SYSTEM; + } + + public String getAccessKey() { + return this.accessKey; + } + + public String getSecretKey() { + return this.secretKey; + } + + public String getEndPoint() { + return this.endPoint; + } + + public String getBucketName() { + return this.bucketName; + } + + public Boolean getHttpsFlag() { + return this.httpsFlag; + } + + public Integer getConnectionTimeout() { + return this.connectionTimeout; + } + + public Integer getMaxErrorRetry() { + return this.maxErrorRetry; + } + + public Integer getSocketTimeout() { + return this.socketTimeout; + } + +} diff --git a/api/src/com/cloud/api/commands/ListS3sCmd.java b/api/src/com/cloud/api/commands/ListS3sCmd.java new file mode 100644 index 00000000000..507053dd89f --- /dev/null +++ b/api/src/com/cloud/api/commands/ListS3sCmd.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.api.commands; + +import static com.cloud.api.ApiConstants.ID; +import static com.cloud.api.BaseCmd.CommandType.LONG; + +import java.util.ArrayList; +import java.util.List; + +import com.cloud.api.BaseListCmd; +import com.cloud.api.Implementation; +import com.cloud.api.Parameter; +import com.cloud.api.ServerApiException; +import com.cloud.api.response.ListResponse; +import com.cloud.api.response.S3Response; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.storage.S3; + +@Implementation(description = "Lists S3s", responseObject = S3Response.class, since = "4.0.0") +public final class ListS3sCmd extends BaseListCmd { + + private static final String COMMAND_NAME = "lists3sresponse"; + + @Parameter(name = ID, type = LONG, required = true, description = "The ID of the S3") + private Long id; + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, + ServerApiException, ConcurrentOperationException, ResourceAllocationException, + NetworkRuleConflictException { + + final List result = _resourceService.listS3s(this); + final ListResponse response = new ListResponse(); + final List s3Responses = new ArrayList(); + + if (result != null) { + + for (S3 s3 : result) { + + S3Response s3Response = _responseGenerator.createS3Response(s3); + s3Response.setResponseName(this.getCommandName()); + s3Response.setObjectName("s3"); + s3Responses.add(s3Response); + + } + + } + + response.setResponses(s3Responses); + response.setResponseName(this.getCommandName()); + + this.setResponseObject(response); + + } + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null || getClass() != thatObject.getClass()) { + return false; + } + + final ListS3sCmd thatListS3sCmd = (ListS3sCmd) thatObject; + + if (this.id != null ? !this.id.equals(thatListS3sCmd.id) : thatListS3sCmd.id != null) { + return false; + } + + return true; + + } + + @Override + public int hashCode() { + return this.id != null ? this.id.hashCode() : 0; + } + + @Override + public String getCommandName() { + return COMMAND_NAME; + } + + public Long getId() { + return this.id; + } + +} diff --git a/api/src/com/cloud/api/response/S3Response.java b/api/src/com/cloud/api/response/S3Response.java new file mode 100644 index 00000000000..1efbe23b05e --- /dev/null +++ b/api/src/com/cloud/api/response/S3Response.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.api.response; + +import com.cloud.serializer.Param; +import com.cloud.utils.IdentityProxy; +import com.google.gson.annotations.SerializedName; + +import static com.cloud.api.ApiConstants.*; + +public class S3Response extends BaseResponse { + + @SerializedName(ID) + @Param(description = "The ID of the S3 configuration") + private IdentityProxy id = new IdentityProxy("s3"); + + @SerializedName(S3_ACCESS_KEY) + @Param(description = "The S3 access key") + private String accessKey; + + @SerializedName(S3_SECRET_KEY) + @Param(description = "The S3 secret key") + private String secretKey; + + @SerializedName(S3_END_POINT) + @Param(description = "The S3 end point") + private String endPoint; + + @SerializedName(S3_BUCKET_NAME) + @Param(description = "The name of the template storage bucket") + private String bucketName; + + @SerializedName(S3_HTTPS_FLAG) + @Param(description = "Connect to S3 using HTTPS?") + private Integer httpsFlag; + + @SerializedName(S3_CONNECTION_TIMEOUT) + @Param(description = "The connection timeout (milliseconds)") + private Integer connectionTimeout; + + @SerializedName(S3_MAX_ERROR_RETRY) + @Param(description = "The maximum number of time to retry a connection on error.") + private Integer maxErrorRetry; + + @SerializedName(S3_SOCKET_TIMEOUT) + @Param(description = "The connection socket (milliseconds)") + private Integer socketTimeout; + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null || this.getClass() != thatObject.getClass()) { + return false; + } + + final S3Response thatS3Response = (S3Response) thatObject; + + if (this.httpsFlag != null ? !this.httpsFlag.equals(thatS3Response.httpsFlag) : thatS3Response.httpsFlag != null) { + return false; + } + + if (this.accessKey != null ? !this.accessKey.equals(thatS3Response.accessKey) : thatS3Response.accessKey != null) { + return false; + } + + if (this.connectionTimeout != null ? !this.connectionTimeout.equals(thatS3Response.connectionTimeout) : thatS3Response.connectionTimeout != null) { + return false; + } + + if (this.endPoint != null ? !this.endPoint.equals(thatS3Response.endPoint) : thatS3Response.endPoint != null) { + return false; + } + + if (this.id != null ? !this.id.equals(thatS3Response.id) : thatS3Response.id != null) { + return false; + } + + if (this.maxErrorRetry != null ? !this.maxErrorRetry.equals(thatS3Response.maxErrorRetry) : thatS3Response.maxErrorRetry != null) { + return false; + } + + if (this.secretKey != null ? !this.secretKey.equals(thatS3Response.secretKey) : thatS3Response.secretKey != null) { + return false; + } + + if (this.socketTimeout != null ? !this.socketTimeout.equals(thatS3Response.socketTimeout) : thatS3Response.socketTimeout != null) { + return false; + } + + if (this.bucketName != null ? !this.bucketName.equals(thatS3Response.bucketName) : thatS3Response.bucketName != null) { + return false; + } + + return true; + + } + + @Override + public int hashCode() { + + int result = this.id != null ? this.id.hashCode() : 0; + result = 31 * result + (this.accessKey != null ? this.accessKey.hashCode() : 0); + result = 31 * result + (this.secretKey != null ? this.secretKey.hashCode() : 0); + result = 31 * result + (this.endPoint != null ? this.endPoint.hashCode() : 0); + result = 31 * result + (this.bucketName != null ? this.bucketName.hashCode() : 0); + result = 31 * result + (this.httpsFlag != null ? this.httpsFlag : 0); + result = 31 * result + (this.connectionTimeout != null ? this.connectionTimeout.hashCode() : 0); + result = 31 * result + (this.maxErrorRetry != null ? this.maxErrorRetry.hashCode() : 0); + result = 31 * result + (this.socketTimeout != null ? this.socketTimeout.hashCode() : 0); + + return result; + + } + + @Override + public Long getObjectId() { + return this.id.getValue(); + } + + public void setObjectId(Long id) { + this.id.setValue(id); + } + + public String getAccessKey() { + return this.accessKey; + } + + public void setAccessKey(final String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return this.secretKey; + } + + public void setSecretKey(final String secretKey) { + this.secretKey = secretKey; + } + + public String getEndPoint() { + return this.endPoint; + } + + public void setEndPoint(final String endPoint) { + this.endPoint = endPoint; + } + + + public String getTemplateBucketName() { + return this.bucketName; + } + + public void setTemplateBucketName(final String templateBucketName) { + this.bucketName = templateBucketName; + } + + public Integer getHttpsFlag() { + return this.httpsFlag; + } + + public void setHttpsFlag(final Integer httpsFlag) { + this.httpsFlag = httpsFlag; + } + + public Integer getConnectionTimeout() { + return this.connectionTimeout; + } + + public void setConnectionTimeout(final Integer connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + public Integer getMaxErrorRetry() { + return this.maxErrorRetry; + } + + public void setMaxErrorRetry(final Integer maxErrorRetry) { + this.maxErrorRetry = maxErrorRetry; + } + + public Integer getSocketTimeout() { + return this.socketTimeout; + } + + public void setSocketTimeout(final Integer socketTimeout) { + this.socketTimeout = socketTimeout; + } + +} diff --git a/api/src/com/cloud/resource/ResourceService.java b/api/src/com/cloud/resource/ResourceService.java index 10654539bb6..4cdb31a72ab 100755 --- a/api/src/com/cloud/resource/ResourceService.java +++ b/api/src/com/cloud/resource/ResourceService.java @@ -20,10 +20,12 @@ import java.util.List; import com.cloud.api.commands.AddClusterCmd; import com.cloud.api.commands.AddHostCmd; +import com.cloud.api.commands.AddS3Cmd; import com.cloud.api.commands.AddSecondaryStorageCmd; import com.cloud.api.commands.AddSwiftCmd; import com.cloud.api.commands.CancelMaintenanceCmd; import com.cloud.api.commands.DeleteClusterCmd; +import com.cloud.api.commands.ListS3sCmd; import com.cloud.api.commands.ListSwiftsCmd; import com.cloud.api.commands.PrepareForMaintenanceCmd; import com.cloud.api.commands.ReconnectHostCmd; @@ -35,6 +37,7 @@ import com.cloud.exception.ResourceInUseException; import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.org.Cluster; +import com.cloud.storage.S3; import com.cloud.storage.Swift; import com.cloud.utils.fsm.NoTransitionException; @@ -93,8 +96,13 @@ public interface ResourceService { Cluster getCluster(Long clusterId); Swift discoverSwift(AddSwiftCmd addSwiftCmd) throws DiscoveryException; + + S3 discoverS3(AddS3Cmd cmd) throws DiscoveryException; List getSupportedHypervisorTypes(long zoneId, boolean forVirtualRouter, Long podId); List listSwifts(ListSwiftsCmd cmd); + + List listS3s(ListS3sCmd cmd); + } diff --git a/api/src/com/cloud/storage/S3.java b/api/src/com/cloud/storage/S3.java new file mode 100644 index 00000000000..7a679f2adc5 --- /dev/null +++ b/api/src/com/cloud/storage/S3.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage; + +import com.cloud.agent.api.to.S3TO; + +import java.util.Date; + +public interface S3 { + + long getId(); + + String getUuid(); + + String getAccessKey(); + + String getSecretKey(); + + String getEndPoint(); + + String getBucketName(); + + Integer getHttpsFlag(); + + Integer getConnectionTimeout(); + + Integer getMaxErrorRetry(); + + Integer getSocketTimeout(); + + Date getCreated(); + + S3TO toS3TO(); + +} diff --git a/api/test/src/com/cloud/agent/api/test/BackupSnapshotAnswerTest.java b/api/test/src/com/cloud/agent/api/test/BackupSnapshotAnswerTest.java index aa1c0fb7f42..b3f2847cd51 100644 --- a/api/test/src/com/cloud/agent/api/test/BackupSnapshotAnswerTest.java +++ b/api/test/src/com/cloud/agent/api/test/BackupSnapshotAnswerTest.java @@ -36,7 +36,7 @@ public class BackupSnapshotAnswerTest { StoragePool pool = Mockito.mock(StoragePool.class); - bsc = new BackupSnapshotCommand("primaryStoragePoolNameLabel", + bsc = new BackupSnapshotCommand( "secondaryStoragePoolURL", 101L, 102L, 103L, 104L, "volumePath", pool, "snapshotUuid", "snapshotName", "prevSnapshotUuid", "prevBackupUuid", false, "vmName", 5); diff --git a/api/test/src/com/cloud/agent/api/test/BackupSnapshotCommandTest.java b/api/test/src/com/cloud/agent/api/test/BackupSnapshotCommandTest.java index baeed5577da..a6241488943 100644 --- a/api/test/src/com/cloud/agent/api/test/BackupSnapshotCommandTest.java +++ b/api/test/src/com/cloud/agent/api/test/BackupSnapshotCommandTest.java @@ -116,14 +116,14 @@ public class BackupSnapshotCommandTest { }; BackupSnapshotCommand bsc = new BackupSnapshotCommand( - "primaryStoragePoolNameLabel", "http://secondary.Storage.Url", + "http://secondary.Storage.Url", 101L, 102L, 103L, 104L, "vPath", pool, "420fa39c-4ef1-a83c-fd93-46dc1ff515ae", "sName", "9012793e-0657-11e2-bebc-0050568b0057", "7167e0b2-f5b0-11e1-8414-0050568b0057", false, "vmName", 5); BackupSnapshotCommand bsc1 = new BackupSnapshotCommand( - "primaryStoragePoolNameLabel", "http://secondary.Storage.Url", + "http://secondary.Storage.Url", 101L, 102L, 103L, 104L, "vPath", pool, "420fa39c-4ef1-a83c-fd93-46dc1ff515ae", "sName", "9012793e-0657-11e2-bebc-0050568b0057", @@ -132,7 +132,7 @@ public class BackupSnapshotCommandTest { @Test public void testGetPrimaryStoragePoolNameLabel() { String label = bsc.getPrimaryStoragePoolNameLabel(); - assertTrue(label.equals("primaryStoragePoolNameLabel")); + assertTrue(label.equals("bed9f83e-cac3-11e1-ac8a-0050568b007e")); } @Test diff --git a/api/test/src/com/cloud/agent/api/test/SnapshotCommandTest.java b/api/test/src/com/cloud/agent/api/test/SnapshotCommandTest.java index 3e37edaf53b..98cdee2a8bf 100644 --- a/api/test/src/com/cloud/agent/api/test/SnapshotCommandTest.java +++ b/api/test/src/com/cloud/agent/api/test/SnapshotCommandTest.java @@ -23,19 +23,109 @@ import static org.junit.Assert.assertTrue; import org.junit.Before; import org.junit.Test; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + import com.cloud.agent.api.SnapshotCommand; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolStatus; public class SnapshotCommandTest { - SnapshotCommand ssc = new SnapshotCommand("primaryStoragePoolNameLabel", + + public StoragePool pool = new StoragePool() { + public long getId() { + return 1L; + }; + + public String getName() { + return "name"; + }; + + public String getUuid() { + return "bed9f83e-cac3-11e1-ac8a-0050568b007e"; + }; + + public StoragePoolType getPoolType() { + return StoragePoolType.Filesystem; + }; + + public Date getCreated() { + Date date = null; + try { + date = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss") + .parse("01/01/1970 12:12:12"); + } catch (ParseException e) { + e.printStackTrace(); + } + return date; + } + + public Date getUpdateTime() { + return new Date(); + }; + + public long getDataCenterId() { + return 0L; + }; + + public long getCapacityBytes() { + return 0L; + }; + + public long getAvailableBytes() { + return 0L; + }; + + public Long getClusterId() { + return 0L; + }; + + public String getHostAddress() { + return "hostAddress"; + }; + + public String getPath() { + return "path"; + }; + + public String getUserInfo() { + return "userInfo"; + }; + + public boolean isShared() { + return false; + }; + + public boolean isLocal() { + return false; + }; + + public StoragePoolStatus getStatus() { + return StoragePoolStatus.Up; + }; + + public int getPort() { + return 25; + }; + + public Long getPodId() { + return 0L; + }; + }; + + SnapshotCommand ssc = new SnapshotCommand(pool, "http://secondary.Storage.Url", "420fa39c-4ef1-a83c-fd93-46dc1ff515ae", "snapshotName", 101L, 102L, 103L); SnapshotCommand ssc1; + @Before public void setUp() { - ssc1 = new SnapshotCommand("primaryStoragePoolNameLabel", + ssc1 = new SnapshotCommand(pool, "secondaryStorageUrl", "snapshotUuid", "snapshotName", 101L, 102L, 103L); } @@ -43,7 +133,7 @@ public class SnapshotCommandTest { @Test public void testGetPrimaryStoragePoolNameLabel() { String label = ssc.getPrimaryStoragePoolNameLabel(); - assertTrue(label.equals("primaryStoragePoolNameLabel")); + assertTrue(label.equals("bed9f83e-cac3-11e1-ac8a-0050568b007e")); } @Test diff --git a/client/WEB-INF/classes/resources/messages.properties b/client/WEB-INF/classes/resources/messages.properties index 48514e8f29d..3dc01aa2008 100644 --- a/client/WEB-INF/classes/resources/messages.properties +++ b/client/WEB-INF/classes/resources/messages.properties @@ -32,6 +32,17 @@ label.destroy=Destroy label.restore=Restore label.isolation.uri=Isolation URI label.broadcast.uri=Broadcast URI +label.enable.s3=Enable S3-backed Secondary Storage +confirm.enable.s3=Please fill in the following information to enable support for S3-backed Secondary Storage +message.after.enable.s3=S3-backed Secondary Storage configured. Note: When you leave this page, you will not be able to re-configure S3 again. +label.s3.access_key=Access Key +label.s3.secret_key=Secret Key +label.s3.bucket=Bucket +label.s3.endpoint=Endpoint +label.s3.use_https=Use HTTPS +label.s3.connection_timeout=Connection Timeout +label.s3.max_error_retry=Max Error Retry +label.s3.socket_timeout=Socket Timeout #new labels (end) ************************************************************************************************ @@ -305,8 +316,6 @@ label.installWizard.subtitle=This tour will aid you in setting up your CloudStac label.continue=Continue label.installWizard.title=Hello and Welcome to CloudStack™ label.agree=Agree -label.license.agreement=License Agreement -label.license.agreement.subtitle=Please accept the CloudStack™ EULA before installing. label.manage.resources=Manage Resources label.port.forwarding.policies=Port forwarding policies label.load.balancing.policies=Load balancing policies diff --git a/client/WEB-INF/classes/resources/messages_fr_FR.properties b/client/WEB-INF/classes/resources/messages_fr_FR.properties index 3710d25a916..4ba3c725892 100644 --- a/client/WEB-INF/classes/resources/messages_fr_FR.properties +++ b/client/WEB-INF/classes/resources/messages_fr_FR.properties @@ -292,8 +292,6 @@ label.installWizard.subtitle=Ce tutorial vous aidera label.continue=Continuer label.installWizard.title=Bonjour et bienvenue dans CloudStack label.agree=Accepter -label.license.agreement=Accord de licence -label.license.agreement.subtitle=Merci d\'accepter l\'EULA CloudStack avant d\'installer. label.manage.resources=Grer les ressources label.port.forwarding.policies=Rgles de transfert de port label.load.balancing.policies=Rgles de rpartition de charge diff --git a/client/WEB-INF/classes/resources/messages_ja.properties b/client/WEB-INF/classes/resources/messages_ja.properties index ec785f6f83e..7555b83dd78 100644 --- a/client/WEB-INF/classes/resources/messages_ja.properties +++ b/client/WEB-INF/classes/resources/messages_ja.properties @@ -295,8 +295,6 @@ label.installWizard.subtitle=このガイド ツアーは CloudStack™ 環 label.continue=続行 label.installWizard.title=CloudStack™ へようこそ label.agree=同意する -label.license.agreement=ライセンス契約 -label.license.agreement.subtitle=インストールの前に CloudStack™ エンド ユーザー ライセンス契約に同意してください。 label.manage.resources=リソースの管理 label.port.forwarding.policies=ポート転送ポリシー label.load.balancing.policies=負荷分散ポリシー diff --git a/client/WEB-INF/classes/resources/messages_pt_BR.properties b/client/WEB-INF/classes/resources/messages_pt_BR.properties index 2b6a5d3ce45..3bc435b9717 100644 --- a/client/WEB-INF/classes/resources/messages_pt_BR.properties +++ b/client/WEB-INF/classes/resources/messages_pt_BR.properties @@ -292,8 +292,6 @@ label.installWizard.subtitle=Este tour vai auxiliar voc label.continue=Continuar label.installWizard.title=Ol, seja bem vindo ao CloudStack&\#8482 label.agree=Concordo -label.license.agreement=Acordo de Licena -label.license.agreement.subtitle=Por favor aceite a EULA do CloudStack&\#8482 antes de instalar. label.manage.resources=Gerenciar Recursos label.port.forwarding.policies=Polticas de redirecionamento de portas label.load.balancing.policies=Polticas de balanceamento de carga diff --git a/client/WEB-INF/classes/resources/messages_ru_RU.properties b/client/WEB-INF/classes/resources/messages_ru_RU.properties index 0c6b962d1cf..dd7e7460744 100644 --- a/client/WEB-INF/classes/resources/messages_ru_RU.properties +++ b/client/WEB-INF/classes/resources/messages_ru_RU.properties @@ -292,8 +292,6 @@ label.installWizard.subtitle=Это руководство настроит ва label.continue=Продолжить label.installWizard.title=Здравствуйте и добро пожаловать в CloudStack\! label.agree=Согласен -label.license.agreement=Лицензионное соглашение -label.license.agreement.subtitle=Для продолжения установки согласитесь с лицензионным соглашением. label.manage.resources=Управление ресурсами label.port.forwarding.policies=Политики проброса портов label.load.balancing.policies=Политики балансировки нагрузки diff --git a/client/WEB-INF/classes/resources/messages_zh_CN.properties b/client/WEB-INF/classes/resources/messages_zh_CN.properties index f86601773cb..62160d98488 100644 --- a/client/WEB-INF/classes/resources/messages_zh_CN.properties +++ b/client/WEB-INF/classes/resources/messages_zh_CN.properties @@ -295,8 +295,6 @@ label.installWizard.subtitle=此教程将帮助您设置 CloudStack™ 安装 label.continue=继续 label.installWizard.title=您好，欢迎使用 CloudStack™ label.agree=同意 -label.license.agreement=许可协议 -label.license.agreement.subtitle=必须先接受 CloudStack™ EULA 才能进行安装。 label.manage.resources=管理资源 label.port.forwarding.policies=端口转发策略 label.load.balancing.policies=负载平衡策略 diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in index 149547ee575..437c8d458f5 100755 --- a/client/tomcatconf/commands.properties.in +++ b/client/tomcatconf/commands.properties.in @@ -227,6 +227,9 @@ listCapacity=com.cloud.api.commands.ListCapacityCmd;3 addSwift=com.cloud.api.commands.AddSwiftCmd;1 listSwifts=com.cloud.api.commands.ListSwiftsCmd;1 +#### s3 commands +addS3=com.cloud.api.commands.AddS3Cmd;1 +listS3s=com.cloud.api.commands.ListS3sCmd;1 #### host commands addHost=com.cloud.api.commands.AddHostCmd;3 diff --git a/client/tomcatconf/db-enc.properties.in b/client/tomcatconf/db-enc.properties.in index 6136f37511f..a9404fa4932 100644 --- a/client/tomcatconf/db-enc.properties.in +++ b/client/tomcatconf/db-enc.properties.in @@ -37,9 +37,6 @@ db.cloud.testOnBorrow=true db.cloud.testWhileIdle=true db.cloud.timeBetweenEvictionRunsMillis=40000 db.cloud.minEvictableIdleTimeMillis=240000 -db.cloud.removeAbandoned=false -db.cloud.removeAbandonedTimeout=300 -db.cloud.logAbandoned=true db.cloud.poolPreparedStatements=false db.cloud.url.params=prepStmtCacheSize=517&cachePrepStmts=true diff --git a/client/tomcatconf/db.properties.in b/client/tomcatconf/db.properties.in index f39d8fe4097..e38366706d4 100644 --- a/client/tomcatconf/db.properties.in +++ b/client/tomcatconf/db.properties.in @@ -37,9 +37,6 @@ db.cloud.testOnBorrow=true db.cloud.testWhileIdle=true db.cloud.timeBetweenEvictionRunsMillis=40000 db.cloud.minEvictableIdleTimeMillis=240000 -db.cloud.removeAbandoned=false -db.cloud.removeAbandonedTimeout=300 -db.cloud.logAbandoned=true db.cloud.poolPreparedStatements=false db.cloud.url.params=prepStmtCacheSize=517&cachePrepStmts=true diff --git a/core/pom.xml b/core/pom.xml index 15f0f7b7302..3d6356e561e 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -43,6 +43,11 @@ + + commons-codec + commons-codec + ${cs.codec.version} + install diff --git a/core/src/com/cloud/agent/api/DirectNetworkUsageCommand.java b/core/src/com/cloud/agent/api/DirectNetworkUsageCommand.java index 97dcf90177e..86e794a1dc6 100644 --- a/core/src/com/cloud/agent/api/DirectNetworkUsageCommand.java +++ b/core/src/com/cloud/agent/api/DirectNetworkUsageCommand.java @@ -24,11 +24,15 @@ public class DirectNetworkUsageCommand extends Command { private List publicIps; private Date start; private Date end; + private String includeZones; + private String excludeZones; - public DirectNetworkUsageCommand(List publicIps, Date start, Date end) { + public DirectNetworkUsageCommand(List publicIps, Date start, Date end, String includeZones, String excludeZones) { this.setPublicIps(publicIps); this.setStart(start); this.setEnd(end); + this.setIncludeZones(includeZones); + this.setExcludeZones(excludeZones); } @Override @@ -59,4 +63,21 @@ public class DirectNetworkUsageCommand extends Command { public Date getEnd() { return end; } + + public String getIncludeZones() { + return includeZones; + } + + public void setIncludeZones(String includeZones) { + this.includeZones = includeZones; + } + + public String getExcludeZones() { + return excludeZones; + } + + public void setExcludeZones(String excludeZones) { + this.excludeZones = excludeZones; + } + } diff --git a/core/src/com/cloud/network/resource/TrafficSentinelResource.java b/core/src/com/cloud/network/resource/TrafficSentinelResource.java index a9088d0eb67..a13e080b7e4 100644 --- a/core/src/com/cloud/network/resource/TrafficSentinelResource.java +++ b/core/src/com/cloud/network/resource/TrafficSentinelResource.java @@ -49,7 +49,6 @@ import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupTrafficMonitorCommand; import com.cloud.host.Host; import com.cloud.resource.ServerResource; -import com.cloud.utils.NumbersUtil; import com.cloud.utils.exception.ExecutionException; public class TrafficSentinelResource implements ServerResource { @@ -59,8 +58,8 @@ public class TrafficSentinelResource implements ServerResource { private String _ip; private String _guid; private String _url; - private static Integer _numRetries; - private static Integer _timeoutInSeconds; + private String _inclZones; + private String _exclZones; private static final Logger s_logger = Logger.getLogger(TrafficSentinelResource.class); @@ -91,9 +90,8 @@ public class TrafficSentinelResource implements ServerResource { throw new ConfigurationException("Unable to find url"); } - _numRetries = NumbersUtil.parseInt((String) params.get("numRetries"), 1); - - _timeoutInSeconds = NumbersUtil.parseInt((String) params.get("timeoutInSeconds"), 300); + _inclZones = (String)params.get("inclZones"); + _exclZones = (String)params.get("exclZones"); return true; } catch (Exception e) { @@ -197,6 +195,15 @@ public class TrafficSentinelResource implements ServerResource { try { //Direct Network Usage URL trafficSentinel; + //Use Global include/exclude zones if there are no per TS zones + if(_inclZones == null){ + _inclZones = cmd.getIncludeZones(); + } + + if(_exclZones == null){ + _exclZones = cmd.getExcludeZones(); + } + try { //Query traffic Sentinel trafficSentinel = new URL(_url+"/inmsf/Query?script="+URLEncoder.encode(getScript(cmd.getPublicIps(), cmd.getStart(), cmd.getEnd()),"UTF-8") @@ -247,12 +254,28 @@ public class TrafficSentinelResource implements ServerResource { IpAddresses += ","; } } + String destZoneCondition = ""; + if(_inclZones !=null && !_inclZones.isEmpty()){ + destZoneCondition = " & destinationzone = "+_inclZones; + } + if(_exclZones !=null && !_exclZones.isEmpty()){ + destZoneCondition += " & destinationzone != "+_exclZones; + } + + String srcZoneCondition = ""; + if(_inclZones !=null && !_inclZones.isEmpty()){ + srcZoneCondition = " & sourcezone = "+_inclZones; + } + if(_exclZones !=null && !_exclZones.isEmpty()){ + srcZoneCondition += " & sourcezone != "+_exclZones; + } + String startDate = getDateString(start); String endtDate = getDateString(end); StringBuffer sb = new StringBuffer(); sb.append("var q = Query.topN(\"historytrmx\","); sb.append(" \"ipsource,bytes\","); - sb.append(" \"ipsource = "+IpAddresses+" & destinationzone = EXTERNAL\","); + sb.append(" \"ipsource = "+IpAddresses+destZoneCondition+"\","); sb.append(" \""+startDate+", "+endtDate+"\","); sb.append(" \"bytes\","); sb.append(" 100000);"); @@ -265,7 +288,7 @@ public class TrafficSentinelResource implements ServerResource { sb.append(" });"); sb.append("var q = Query.topN(\"historytrmx\","); sb.append(" \"ipdestination,bytes\","); - sb.append(" \"ipdestination = "+IpAddresses+" & sourcezone = EXTERNAL\","); + sb.append(" \"ipdestination = "+IpAddresses+srcZoneCondition+"\","); sb.append(" \""+startDate+", "+endtDate+"\","); sb.append(" \"bytes\","); sb.append(" 100000);"); diff --git a/core/src/com/cloud/storage/S3VO.java b/core/src/com/cloud/storage/S3VO.java new file mode 100644 index 00000000000..89f233b87fe --- /dev/null +++ b/core/src/com/cloud/storage/S3VO.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage; + +import com.cloud.agent.api.to.S3TO; +import com.cloud.api.Identity; +import com.cloud.utils.db.GenericDao; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.Date; + +@Entity +@Table(name = "s3") +public class S3VO implements S3, Identity { + + public static final String ID_COLUMN_NAME = "id"; + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = ID_COLUMN_NAME) + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "access_key") + private String accessKey; + + @Column(name = "secret_key") + private String secretKey; + + @Column(name = "end_point") + private String endPoint; + + @Column(name = "bucket") + private String bucketName; + + @Column(name = "https") + private Integer httpsFlag; + + @Column(name = "connection_timeout") + private Integer connectionTimeout; + + @Column(name = "max_error_retry") + private Integer maxErrorRetry; + + @Column(name = "socket_timeout") + private Integer socketTimeout; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + public S3VO() { + super(); + } + + public S3VO(final String uuid, final String accessKey, + final String secretKey, final String endPoint, + final String bucketName, final Boolean httpsFlag, + final Integer connectionTimeout, final Integer maxErrorRetry, + final Integer socketTimeout, final Date created) { + + super(); + + this.uuid = uuid; + this.accessKey = accessKey; + this.secretKey = secretKey; + this.endPoint = endPoint; + this.bucketName = bucketName; + + Integer value = null; + if (httpsFlag != null) { + value = httpsFlag == false ? 0 : 1; + } + this.httpsFlag = value; + + this.connectionTimeout = connectionTimeout; + this.maxErrorRetry = maxErrorRetry; + this.socketTimeout = socketTimeout; + this.created = created; + + } + + @Override + public S3TO toS3TO() { + + Boolean httpsFlag = null; + if (this.httpsFlag != null) { + httpsFlag = this.httpsFlag == 0 ? false : true; + } + + return new S3TO(this.id, this.uuid, this.accessKey, this.secretKey, + this.endPoint, this.bucketName, httpsFlag, + this.connectionTimeout, this.maxErrorRetry, this.socketTimeout, + this.created); + + } + + public long getId() { + return this.id; + } + + public void setId(final long id) { + this.id = id; + } + + public String getUuid() { + return this.uuid; + } + + public void setUuid(final String uuid) { + this.uuid = uuid; + } + + public String getAccessKey() { + return this.accessKey; + } + + public void setAccessKey(final String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return this.secretKey; + } + + public void setSecretKey(final String secretKey) { + this.secretKey = secretKey; + } + + public String getEndPoint() { + return this.endPoint; + } + + public void setEndPoint(final String endPoint) { + this.endPoint = endPoint; + } + + public String getBucketName() { + return this.bucketName; + } + + public void setBucketName(final String bucketName) { + this.bucketName = bucketName; + } + + public Integer getHttpsFlag() { + return this.httpsFlag; + } + + public void setHttpsFlag(final Integer httpsFlag) { + this.httpsFlag = httpsFlag; + } + + public Integer getConnectionTimeout() { + return this.connectionTimeout; + } + + public void setConnectionTimeout(final int connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + public Integer getMaxErrorRetry() { + return this.maxErrorRetry; + } + + public void setMaxErrorRetry(final int maxErrorRetry) { + this.maxErrorRetry = maxErrorRetry; + } + + public Integer getSocketTimeout() { + return this.socketTimeout; + } + + public void setSocketTimeout(final int socketTimeout) { + this.socketTimeout = socketTimeout; + } + + public Date getCreated() { + return this.created; + } + + public void setCreated(final Date created) { + this.created = created; + } + +} diff --git a/core/src/com/cloud/storage/SnapshotVO.java b/core/src/com/cloud/storage/SnapshotVO.java index 08dfafa6bac..ae4fd6aafef 100644 --- a/core/src/com/cloud/storage/SnapshotVO.java +++ b/core/src/com/cloud/storage/SnapshotVO.java @@ -91,6 +91,9 @@ public class SnapshotVO implements Snapshot, Identity { @Column(name="swift_id") Long swiftId; + @Column(name="s3_id") + Long s3Id; + @Column(name="sechost_id") Long secHostId; @@ -289,4 +292,13 @@ public class SnapshotVO implements Snapshot, Identity { public void setUuid(String uuid) { this.uuid = uuid; } + + public Long getS3Id() { + return s3Id; + } + + public void setS3Id(Long s3Id) { + this.s3Id = s3Id; + } + } diff --git a/core/src/com/cloud/storage/VMTemplateS3VO.java b/core/src/com/cloud/storage/VMTemplateS3VO.java new file mode 100644 index 00000000000..a75c37d192d --- /dev/null +++ b/core/src/com/cloud/storage/VMTemplateS3VO.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage; + +import com.cloud.utils.db.GenericDaoBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import java.text.DateFormat; +import java.util.Date; + +@Entity +@Table(name = "template_s3_ref") +public class VMTemplateS3VO { + + public static final String S3_ID_COLUMN_NAME = "s3_id"; + + public static final String TEMPLATE_ID_COLUMN_NAME = "template_id"; + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private long id; + + @Column(name = S3_ID_COLUMN_NAME) + private long s3Id; + + @Column(name = TEMPLATE_ID_COLUMN_NAME) + private long templateId; + + @Column(name = GenericDaoBase.CREATED_COLUMN) + private Date created; + + @Column(name = "size") + private Long size; + + @Column(name = "physical_size") + private Long physicalSize; + + public VMTemplateS3VO() { + super(); + } + + public VMTemplateS3VO(final long s3Id, final long templateId, + final Date created, final Long size, final Long physicalSize) { + + super(); + + this.s3Id = s3Id; + this.templateId = templateId; + this.created = created; + this.size = size; + this.physicalSize = physicalSize; + + } + + @Override + public boolean equals(final Object thatObject) { + + if (this == thatObject) { + return true; + } + + if (thatObject == null || getClass() != thatObject.getClass()) { + return false; + } + + final VMTemplateS3VO thatVMTemplateS3VO = (VMTemplateS3VO) thatObject; + + if (this.id != thatVMTemplateS3VO.id) { + return false; + } + + if (this.s3Id != thatVMTemplateS3VO.s3Id) { + return false; + } + + if (this.templateId != thatVMTemplateS3VO.templateId) { + return false; + } + + if (this.created != null ? !created.equals(thatVMTemplateS3VO.created) + : thatVMTemplateS3VO.created != null) { + return false; + } + + if (this.physicalSize != null ? !physicalSize + .equals(thatVMTemplateS3VO.physicalSize) + : thatVMTemplateS3VO.physicalSize != null) { + return false; + } + + if (this.size != null ? !size.equals(thatVMTemplateS3VO.size) + : thatVMTemplateS3VO.size != null) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + + int result = (int) (this.id ^ (this.id >>> 32)); + + result = 31 * result + (int) (this.s3Id ^ (this.s3Id >>> 32)); + result = 31 * result + + (int) (this.templateId ^ (this.templateId >>> 32)); + result = 31 * result + + (this.created != null ? this.created.hashCode() : 0); + result = 31 * result + (this.size != null ? this.size.hashCode() : 0); + result = 31 + * result + + (this.physicalSize != null ? this.physicalSize.hashCode() : 0); + + return result; + + } + + public long getId() { + return this.id; + } + + public void setId(final long id) { + this.id = id; + } + + public long getS3Id() { + return this.s3Id; + } + + public void setS3Id(final long s3Id) { + this.s3Id = s3Id; + } + + public long getTemplateId() { + return this.templateId; + } + + public void setTemplateId(final long templateId) { + this.templateId = templateId; + } + + public Date getCreated() { + return this.created; + } + + public void setCreated(final Date created) { + this.created = created; + } + + public Long getSize() { + return this.size; + } + + public void setSize(final Long size) { + this.size = size; + } + + public Long getPhysicalSize() { + return this.physicalSize; + } + + public void setPhysicalSize(final Long physicalSize) { + this.physicalSize = physicalSize; + } + + @Override + public String toString() { + + final StringBuilder stringBuilder = new StringBuilder( + "VMTemplateS3VO [ id: ").append(id).append(", created: ") + .append(DateFormat.getDateTimeInstance().format(created)) + .append(", physicalSize: ").append(physicalSize) + .append(", size: ").append(size).append(", templateId: ") + .append(templateId).append(", s3Id: ").append(s3Id) + .append(" ]"); + + return stringBuilder.toString(); + + } + +} diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java index 155210df499..d8fdc3a71d6 100755 --- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java +++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java @@ -16,10 +16,20 @@ // under the License. package com.cloud.storage.resource; +import static com.cloud.utils.S3Utils.deleteDirectory; +import static com.cloud.utils.S3Utils.getDirectory; +import static com.cloud.utils.S3Utils.putDirectory; +import static com.cloud.utils.StringUtils.join; +import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock; +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static org.apache.commons.lang.StringUtils.substringAfterLast; + import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileWriter; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; @@ -32,6 +42,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.Callable; import javax.naming.ConfigurationException; @@ -46,6 +57,9 @@ import com.cloud.agent.api.ComputeChecksumCommand; import com.cloud.agent.api.DeleteObjectFromSwiftCommand; import com.cloud.agent.api.DeleteSnapshotBackupCommand; import com.cloud.agent.api.DeleteSnapshotsDirCommand; +import com.cloud.agent.api.DeleteTemplateFromS3Command; +import com.cloud.agent.api.DownloadSnapshotFromS3Command; +import com.cloud.agent.api.DownloadTemplateFromS3ToSecondaryStorageCommand; import com.cloud.agent.api.GetStorageStatsAnswer; import com.cloud.agent.api.GetStorageStatsCommand; import com.cloud.agent.api.PingCommand; @@ -60,6 +74,8 @@ import com.cloud.agent.api.SecStorageSetupCommand.Certificates; import com.cloud.agent.api.StartupSecondaryStorageCommand; import com.cloud.agent.api.SecStorageVMSetupCommand; import com.cloud.agent.api.StartupCommand; +import com.cloud.agent.api.StartupSecondaryStorageCommand; +import com.cloud.agent.api.UploadTemplateToS3FromSecondaryStorageCommand; import com.cloud.agent.api.downloadSnapshotFromSwiftCommand; import com.cloud.agent.api.downloadTemplateFromSwiftToSecondaryStorageCommand; import com.cloud.agent.api.uploadTemplateToSwiftFromSecondaryStorageCommand; @@ -75,6 +91,7 @@ import com.cloud.agent.api.storage.ListVolumeAnswer; import com.cloud.agent.api.storage.ListVolumeCommand; import com.cloud.agent.api.storage.UploadCommand; import com.cloud.agent.api.storage.ssCommand; +import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.api.commands.DeleteVolumeCmd; import com.cloud.exception.InternalErrorException; @@ -90,6 +107,9 @@ import com.cloud.storage.template.TemplateLocation; import com.cloud.storage.template.UploadManager; import com.cloud.storage.template.UploadManagerImpl; import com.cloud.utils.NumbersUtil; +import com.cloud.utils.S3Utils; +import com.cloud.utils.S3Utils.FileNamingStrategy; +import com.cloud.utils.S3Utils.ObjectNamingStrategy; import com.cloud.utils.component.ComponentLocator; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @@ -97,8 +117,15 @@ import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; import com.cloud.vm.SecondaryStorageVm; -public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource { - private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); +public class NfsSecondaryStorageResource extends ServerResourceBase implements + SecondaryStorageResource { + + private static final Logger s_logger = Logger + .getLogger(NfsSecondaryStorageResource.class); + + private static final String TEMPLATE_ROOT_DIR = "template/tmpl"; + private static final String SNAPSHOT_ROOT_DIR = "snapshots"; + int _timeout; String _instance; @@ -168,16 +195,24 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return execute((ListVolumeCommand)cmd); }else if (cmd instanceof downloadSnapshotFromSwiftCommand){ return execute((downloadSnapshotFromSwiftCommand)cmd); + } else if (cmd instanceof DownloadSnapshotFromS3Command) { + return execute((DownloadSnapshotFromS3Command) cmd); } else if (cmd instanceof DeleteSnapshotBackupCommand){ return execute((DeleteSnapshotBackupCommand)cmd); } else if (cmd instanceof DeleteSnapshotsDirCommand){ return execute((DeleteSnapshotsDirCommand)cmd); } else if (cmd instanceof downloadTemplateFromSwiftToSecondaryStorageCommand) { return execute((downloadTemplateFromSwiftToSecondaryStorageCommand) cmd); + } else if (cmd instanceof DownloadTemplateFromS3ToSecondaryStorageCommand) { + return execute((DownloadTemplateFromS3ToSecondaryStorageCommand) cmd); } else if (cmd instanceof uploadTemplateToSwiftFromSecondaryStorageCommand) { return execute((uploadTemplateToSwiftFromSecondaryStorageCommand) cmd); + } else if (cmd instanceof UploadTemplateToS3FromSecondaryStorageCommand) { + return execute((UploadTemplateToS3FromSecondaryStorageCommand) cmd); } else if (cmd instanceof DeleteObjectFromSwiftCommand) { return execute((DeleteObjectFromSwiftCommand) cmd); + } else if (cmd instanceof DeleteTemplateFromS3Command) { + return execute((DeleteTemplateFromS3Command) cmd); } else if (cmd instanceof CleanupSnapshotBackupCommand){ return execute((CleanupSnapshotBackupCommand)cmd); } else { @@ -185,6 +220,69 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + @SuppressWarnings("unchecked") + private String determineS3TemplateDirectory(final Long accountId, + final Long templateId) { + return join(asList(TEMPLATE_ROOT_DIR, accountId, templateId), + S3Utils.SEPARATOR); + } + + @SuppressWarnings("unchecked") + private String determineStorageTemplatePath(final String storagePath, + final Long accountId, final Long templateId) { + return join( + asList(getRootDir(storagePath), TEMPLATE_ROOT_DIR, accountId, + templateId), File.separator); + } + + private Answer execute( + final DownloadTemplateFromS3ToSecondaryStorageCommand cmd) { + + final S3TO s3 = cmd.getS3(); + final String storagePath = cmd.getStoragePath(); + final Long accountId = cmd.getAccountId(); + final Long templateId = cmd.getTemplateId(); + + try { + + final File downloadDirectory = _storage + .getFile(determineStorageTemplatePath(storagePath, + accountId, templateId)); + downloadDirectory.mkdirs(); + + if (!downloadDirectory.exists()) { + final String errMsg = format( + "Unable to create directory " + + "download directory %1$s for download of template id " + + "%2$s from S3.", downloadDirectory.getName(), + templateId); + s_logger.error(errMsg); + return new Answer(cmd, false, errMsg); + } + + getDirectory(s3, s3.getBucketName(), + determineS3TemplateDirectory(accountId, templateId), + downloadDirectory, new FileNamingStrategy() { + @Override + public String determineFileName(final String key) { + return substringAfterLast(key, S3Utils.SEPARATOR); + } + }); + + return new Answer(cmd, true, format("Successfully downloaded " + + "template id %1$s from S3 to directory %2$s", templateId, + downloadDirectory.getName())); + + } catch (Exception e) { + + final String errMsg = format("Failed to upload template id %1$s " + + "due to $2%s", templateId, e.getMessage()); + s_logger.error(errMsg, e); + return new Answer(cmd, false, errMsg); + + } + + } private Answer execute(downloadTemplateFromSwiftToSecondaryStorageCommand cmd) { SwiftTO swift = cmd.getSwift(); @@ -256,6 +354,83 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + private Answer execute(UploadTemplateToS3FromSecondaryStorageCommand cmd) { + + final S3TO s3 = cmd.getS3(); + final Long accountId = cmd.getAccountId(); + final Long templateId = cmd.getTemplateId(); + + try { + + final String templatePath = determineStorageTemplatePath( + cmd.getStoragePath(), accountId, templateId); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Found template id " + templateId + + " account id " + accountId + " from directory " + + templatePath + " to upload to S3."); + } + + if (!_storage.isDirectory(templatePath)) { + final String errMsg = format("S3 Sync Failure: Directory %1$s" + + "for template id %2$s does not exist.", templatePath, + templateId); + s_logger.error(errMsg); + return new Answer(cmd, false, errMsg); + } + + if (!_storage.isFile(templatePath + "/template.properties")) { + final String errMsg = format("S3 Sync Failure: Template id " + + "%1$s does not exist on the file system.", + templatePath); + s_logger.error(errMsg); + return new Answer(cmd, false, errMsg); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug(format( + "Pushing template id %1$s from %2$s to S3...", + templateId, templatePath)); + } + + final String bucket = s3.getBucketName(); + putDirectory(s3, bucket, _storage.getFile(templatePath), + new FilenameFilter() { + @Override + public boolean accept(final File directory, + final String fileName) { + return !fileName.startsWith("."); + } + }, new ObjectNamingStrategy() { + @Override + public String determineKey(final File file) { + s_logger.debug(String + .format("Determining key using account id %1$s and template id %2$s", + accountId, templateId)); + return join( + asList(determineS3TemplateDirectory( + accountId, templateId), file + .getName()), S3Utils.SEPARATOR); + } + }); + + return new Answer( + cmd, + true, + format("Uploaded the contents of directory %1$s for template id %2$s to S3 bucket %3$s", + templatePath, templateId, bucket)); + + } catch (Exception e) { + + final String errMsg = format("Failed to upload template id %1$s", + templateId); + s_logger.error(errMsg, e); + return new Answer(cmd, false, errMsg); + + } + + } + private Answer execute(DeleteObjectFromSwiftCommand cmd) { SwiftTO swift = cmd.getSwift(); String container = cmd.getContainer(); @@ -279,6 +454,47 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } + private Answer execute(final DeleteTemplateFromS3Command cmd) { + + final S3TO s3 = cmd.getS3(); + final Long accountId = cmd.getAccountId(); + final Long templateId = cmd.getTemplateId(); + + if (accountId == null || (accountId != null && accountId <= 0)) { + final String errorMessage = "No account id specified for S3 template deletion."; + s_logger.error(errorMessage); + return new Answer(cmd, false, errorMessage); + } + + if (templateId == null || (templateId != null && templateId <= 0)) { + final String errorMessage = "No template id specified for S3 template deletion."; + s_logger.error(errorMessage); + return new Answer(cmd, false, errorMessage); + } + + if (s3 == null) { + final String errorMessge = "No S3 client options provided"; + s_logger.error(errorMessge); + return new Answer(cmd, false, errorMessge); + } + + final String bucket = s3.getBucketName(); + try { + deleteDirectory(s3, bucket, + determineS3TemplateDirectory(templateId, accountId)); + return new Answer(cmd, true, String.format( + "Deleted template %1%s from bucket %2$s.", templateId, + bucket)); + } catch (Exception e) { + final String errorMessage = String + .format("Failed to delete templaet id %1$s from bucket %2$s due to the following error: %3$s", + templateId, bucket, e.getMessage()); + s_logger.error(errorMessage, e); + return new Answer(cmd, false, errorMessage); + } + + } + String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) { Script command = new Script("/bin/bash", s_logger); command.add("-c"); @@ -451,6 +667,110 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + public Answer execute(final DownloadSnapshotFromS3Command cmd) { + + final S3TO s3 = cmd.getS3(); + final String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); + final Long accountId = cmd.getAccountId(); + final Long volumeId = cmd.getVolumeId(); + + try { + + executeWithNoWaitLock(determineSnapshotLockId(accountId, volumeId), + new Callable() { + + @Override + public Void call() throws Exception { + + final String directoryName = determineSnapshotLocalDirectory( + secondaryStorageUrl, accountId, volumeId); + + String result = createLocalDir(directoryName); + if (result != null) { + throw new InternalErrorException( + format("Failed to create directory %1$s during S3 snapshot download.", + directoryName)); + } + + final String snapshotFileName = determineSnapshotBackupFilename(cmd + .getSnapshotUuid()); + final String key = determineSnapshotS3Key( + accountId, volumeId, snapshotFileName); + final File targetFile = S3Utils.getFile(s3, + s3.getBucketName(), key, + _storage.getFile(directoryName), + new FileNamingStrategy() { + + @Override + public String determineFileName( + String key) { + return snapshotFileName; + } + + }); + + if (cmd.getParent() != null) { + + final String parentPath = join( + File.pathSeparator, directoryName, + determineSnapshotBackupFilename(cmd + .getParent())); + result = setVhdParent( + targetFile.getAbsolutePath(), + parentPath); + if (result != null) { + throw new InternalErrorException( + format("Failed to set the parent for backup %1$s to %2$s due to %3$s.", + targetFile + .getAbsolutePath(), + parentPath, result)); + } + + } + + return null; + + } + + }); + + return new Answer( + cmd, + true, + format("Succesfully retrieved volume id %1$s for account id %2$s to %3$s from S3.", + volumeId, accountId, secondaryStorageUrl)); + + } catch (Exception e) { + final String errMsg = format( + "Failed to retrieve volume id %1$s for account id %2$s to %3$s from S3 due to exception %4$s", + volumeId, accountId, secondaryStorageUrl, e.getMessage()); + s_logger.error(errMsg); + return new Answer(cmd, false, errMsg); + } + + } + + private String determineSnapshotS3Directory(final Long accountId, + final Long volumeId) { + return join(S3Utils.SEPARATOR, SNAPSHOT_ROOT_DIR, accountId, volumeId); + } + + private String determineSnapshotS3Key(final Long accountId, + final Long volumeId, final String snapshotFileName) { + + final String directoryName = determineSnapshotS3Directory(accountId, + volumeId); + return join(S3Utils.SEPARATOR, directoryName, snapshotFileName); + + } + + private String determineSnapshotLocalDirectory( + final String secondaryStorageUrl, final Long accountId, + final Long volumeId) { + return join(File.pathSeparator, getRootDir(secondaryStorageUrl), + SNAPSHOT_ROOT_DIR, accountId, volumeId); + } + public Answer execute(downloadSnapshotFromSwiftCommand cmd){ SwiftTO swift = cmd.getSwift(); String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); @@ -622,6 +942,92 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + private String deleteSnapshotBackupFromLocalFileSystem( + final String secondaryStorageUrl, final Long accountId, + final Long volumeId, final String name, final Boolean deleteAllFlag) { + + final String lPath = determineSnapshotLocalDirectory( + secondaryStorageUrl, accountId, volumeId) + + File.pathSeparator + + (deleteAllFlag ? "*" : "*" + name + "*"); + + final String result = deleteLocalFile(lPath); + + if (result != null) { + return "failed to delete snapshot " + lPath + " , err=" + result; + } + + return null; + + } + + private String deleteSnapshotBackupfromS3(final S3TO s3, + final String secondaryStorageUrl, final Long accountId, + final Long volumeId, final String name, final Boolean deleteAllFlag) { + + try { + + final String bucket = s3.getBucketName(); + + final String result = executeWithNoWaitLock( + determineSnapshotLockId(accountId, volumeId), + new Callable() { + + @Override + public String call() throws Exception { + + final String innerResult = deleteSnapshotBackupFromLocalFileSystem( + secondaryStorageUrl, accountId, volumeId, + name, deleteAllFlag); + if (innerResult != null) { + return innerResult; + } + + if (deleteAllFlag) { + S3Utils.deleteDirectory( + s3, + bucket, + determineSnapshotS3Directory(accountId, + volumeId)); + } else { + S3Utils.deleteObject( + s3, + bucket, + determineSnapshotS3Key( + accountId, + volumeId, + determineSnapshotBackupFilename(name))); + } + + return null; + + } + + }); + + return result; + + } catch (Exception e) { + + s_logger.error( + String.format( + "Failed to delete snapshot backup for account id %1$s volume id %2$sfrom S3.", + accountId, volumeId), e); + return e.getMessage(); + + } + + } + + private String determineSnapshotBackupFilename(final String snapshotUuid) { + return snapshotUuid + ".vhd"; + } + + private String determineSnapshotLockId(final Long accountId, + final Long volumeId) { + return join("_", "SNAPSHOT", accountId, volumeId); + } + protected Answer execute(final DeleteSnapshotBackupCommand cmd) { String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); Long accountId = cmd.getAccountId(); @@ -629,21 +1035,22 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String name = cmd.getSnapshotUuid(); try { SwiftTO swift = cmd.getSwift(); + S3TO s3 = cmd.getS3(); if (swift == null) { - String parent = getRootDir(secondaryStorageUrl); - String filename; - if (cmd.isAll()) { - filename = "*"; - - } else { - filename = "*" + name + "*"; - } - String lPath = parent + "/snapshots/" + String.valueOf(accountId) + "/" + String.valueOf(volumeId) + "/" + filename; - String result = deleteLocalFile(lPath); + final String result = deleteSnapshotBackupFromLocalFileSystem( + secondaryStorageUrl, accountId, volumeId, name, + cmd.isAll()); if (result != null) { - String errMsg = "failed to delete snapshot " + lPath + " , err=" + result; - s_logger.warn(errMsg); - return new Answer(cmd, false, errMsg); + s_logger.warn(result); + return new Answer(cmd, false, result); + } + } else if (s3 != null) { + final String result = deleteSnapshotBackupfromS3(s3, + secondaryStorageUrl, accountId, volumeId, name, + cmd.isAll()); + if (result != null) { + s_logger.warn(result); + return new Answer(cmd, false, result); } } else { String filename; diff --git a/docs/README.txt b/docs/README.txt index 7f096e4b117..e327fb9101c 100644 --- a/docs/README.txt +++ b/docs/README.txt @@ -201,7 +201,7 @@ TAGS FOR A SECTION
Text of the section title Here's the text of a paragraph in this section. - Always use &PRODUCT; rather than typing CloudStack or CloudPlatform. + Always use &PRODUCT; rather than typing CloudStack. Indent with 4 spaces, not with tab characters. To hyperlink to a URL outside this document: Display text of the link here To hyperlink to another section in this document: diff --git a/docs/en-US/API_Developers_Guide.ent b/docs/en-US/Developers_Guide.ent similarity index 100% rename from docs/en-US/API_Developers_Guide.ent rename to docs/en-US/Developers_Guide.ent diff --git a/docs/en-US/API_Developers_Guide.xml b/docs/en-US/Developers_Guide.xml similarity index 85% rename from docs/en-US/API_Developers_Guide.xml rename to docs/en-US/Developers_Guide.xml index c691ad02cf6..6c09354f6a8 100644 --- a/docs/en-US/API_Developers_Guide.xml +++ b/docs/en-US/Developers_Guide.xml @@ -24,14 +24,14 @@ - &PRODUCT; API Developer's Guide + &PRODUCT; Developer's Guide Apache CloudStack 4.0.0-incubating - How to integrate with &PRODUCT; using the &PRODUCT; API. + This guide shows how to develop &PRODUCT;, use the API for operation and integration, access the usage data and use &PRODUCT; specific tools to ease development, testing and integration. @@ -45,10 +45,12 @@ + + diff --git a/docs/en-US/building-devcloud.xml b/docs/en-US/building-devcloud.xml new file mode 100644 index 00000000000..5f792c375a2 --- /dev/null +++ b/docs/en-US/building-devcloud.xml @@ -0,0 +1,32 @@ + + +%BOOK_ENTITIES; +]> + + + +
+ Building DevCloud + The DevCloud appliance can be downloaded from the wiki at . It can also be built from scratch. Code is being developed to provide this alternative build. It is based on veewee, Vagrant and Puppet. + The goal is to automate the DevCloud build and make this automation capability available to all within the source release of &PRODUCT; + This is under heavy development. The code is located in the source tree under tools/devcloud + A preliminary wiki page describes the build at https://cwiki.pache.org/CLOUDSTACK/building-devcloud.html + +
diff --git a/docs/en-US/building-prerequisites.xml b/docs/en-US/building-prerequisites.xml new file mode 100644 index 00000000000..d97ca40f2a3 --- /dev/null +++ b/docs/en-US/building-prerequisites.xml @@ -0,0 +1,66 @@ + + +%BOOK_ENTITIES; +]> + + + + +
+ Build Procedure Prerequisites + In this section we will assume that you are using the Ubuntu Linux distribution with the Advanced Packaging Tool (APT). If you are using a different distribution or OS and a different packaging tool, adapt the following instructions to your environment. To build &PRODUCT; you will need: + + + git, http://git-scm.com + sudo apt-get install git-core + + + maven, http://maven.apache.org + sudo apt-get install maven + Make sure that you installed maven 3 + $ mvn --version +Apache Maven 3.0.4 +Maven home: /usr/share/maven +Java version: 1.6.0_24, vendor: Sun Microsystems Inc. +Java home: /usr/lib/jvm/java-6-openjdk-amd64/jre +Default locale: en_US, platform encoding: UTF-8 +OS name: "linux", version: "3.2.0-33-generic", arch: "amd64", family: "unix" + + + java + set the JAVA_HOME environment variable + $ export JAVA_HOME=/usr/lib/jvm/java-6-openjdk + + + + In addition, to deploy and run &PRODUCT; in a development environment you will need: + + + Mysql + sudo apt-get install mysql-server-5.5 + Start the mysqld service and create a cloud user with cloud as a password + + + Tomcat 6 + sudo apt-get install tomcat6 + + + +
diff --git a/docs/en-US/building-with-maven-deploy.xml b/docs/en-US/building-with-maven-deploy.xml new file mode 100644 index 00000000000..e4b9801aa30 --- /dev/null +++ b/docs/en-US/building-with-maven-deploy.xml @@ -0,0 +1,39 @@ + + +%BOOK_ENTITIES; +]> + + + +
+ Deployment and Testing Steps + Deploying the &PRODUCT; code that you compiled is a two step process: + + If you have not configured the database or modified its properties do: + mvn -P developer -pl developer -Ddeploydb + + Then you need to run the &PRODUCT; management server. To attach a debugger to it, do: + export MAVEN_OPTS="-Xmx1024 -Xdebug -Xrunjdwp:transport=dt_socket,address=8787,server=y,suspend=n" + mvn -pl :cloud-client-ui jetty:run + + + When dealing with the database, remember that you may wipe it entirely and lose any data center configuration that you may have set previously. +
+ diff --git a/docs/en-US/building-with-maven-steps.xml b/docs/en-US/building-with-maven-steps.xml new file mode 100644 index 00000000000..319654fb290 --- /dev/null +++ b/docs/en-US/building-with-maven-steps.xml @@ -0,0 +1,33 @@ + + +%BOOK_ENTITIES; +]> + + + +
+ Building Steps + &PRODUCT; uses git for source version control, first make sure you have the source code by pulling it: + git clone https://git-wip-us.apache.org/repos/asf/incubator-cloudstack.git + Several Project Object Models (POM) are defined to deal with the various build targets of &PRODUCT;. Certain features require some packages that are not compatible with the Apache license and therefore need to be downloaded on your own. Check the wiki for additional information https://cwiki.apache.org/CLOUDSTACK/building-with-maven.html. In order to build all the open source targets of &PRODUCT; do: + mvn clean install + The resulting jar files will be in the target directory of the subdirectory of the compiled module. +
+ diff --git a/docs/en-US/building-with-maven.xml b/docs/en-US/building-with-maven.xml new file mode 100644 index 00000000000..5363b1d754a --- /dev/null +++ b/docs/en-US/building-with-maven.xml @@ -0,0 +1,32 @@ + + +%BOOK_ENTITIES; +]> + + + + + Using Maven to Build &PRODUCT; + + + + + + diff --git a/docs/en-US/devcloud-usage-mode.xml b/docs/en-US/devcloud-usage-mode.xml new file mode 100644 index 00000000000..bb65f904ccd --- /dev/null +++ b/docs/en-US/devcloud-usage-mode.xml @@ -0,0 +1,60 @@ + + +%BOOK_ENTITIES; +]> + + + +
+ DevCloud Usage Mode + DevCloud can be used in several different ways: + + + Full sandbox. Where &PRODUCT; is run within the DevCloud instance started in Virtual Box. + In this mode, the &PRODUCT; management server runs within the instance and nested virtualization allows instantiation of tiny VMs within DevCloud itself. &PRODUCT; code modifications are done within DevCloud. + The following diagram shows the architecture of the SandBox mode. + + + + + + DevCloud.png: Schematic of the DevCloud SandBox architecture + + + + + A deployment environment. Where &PRODUCT; code is developed in the localhost of the developer and the resulting build is deployed within DevCloud + This mode was used in the testing procedure of &PRODUCT; 4.0.0 incubating release. See the following screencast to see how: http://vimeo.com/54621457 + + + A host-only mode. Where DevCloud is used only as a host. &PRODUCT; management server is run in the localhost of the developer + This mode makes use of a host-only interface defined in the Virtual Box preferences. Check the following screencast to see how: http://vimeo.com/54610161 + The following schematic shows the architecture of the Host-Only mode. + + + + + + DevCloud-hostonly.png: Schematic of the DevCloud host-only architecture + + + + +
diff --git a/docs/en-US/devcloud.xml b/docs/en-US/devcloud.xml new file mode 100644 index 00000000000..677818700ae --- /dev/null +++ b/docs/en-US/devcloud.xml @@ -0,0 +1,38 @@ + + +%BOOK_ENTITIES; +]> + + + +
+ DevCloud + DevCloud is the &PRODUCT; sandbox. It is provided as a Virtual Box appliance. It is meant to be used as a development environment to easily test new &PRODUCT; development. It has also been used for training and &PRODUCT; demos since it provides a Cloud in a box. + + DevCloud is provided as a convenience by community members. It is not an official &PRODUCT; release artifact. + The &PRODUCT; source code however, contains tools to build your own DevCloud. + + + DevCloud is under development and should be considered a Work In Progress (WIP), the wiki is the most up to date documentation: + + + + +
diff --git a/docs/en-US/developer-introduction.xml b/docs/en-US/developer-introduction.xml index bb5251b4f26..9d54f31dae9 100644 --- a/docs/en-US/developer-introduction.xml +++ b/docs/en-US/developer-introduction.xml @@ -23,7 +23,7 @@ --> - Introduction for Developers + Introduction to the &PRODUCT; API diff --git a/docs/en-US/images/DevCloud-hostonly.png b/docs/en-US/images/DevCloud-hostonly.png new file mode 100644 index 00000000000..111f93ac700 Binary files /dev/null and b/docs/en-US/images/DevCloud-hostonly.png differ diff --git a/docs/en-US/images/DevCloud.png b/docs/en-US/images/DevCloud.png new file mode 100644 index 00000000000..5e83ca946c7 Binary files /dev/null and b/docs/en-US/images/DevCloud.png differ diff --git a/docs/en-US/marvin.xml b/docs/en-US/marvin.xml new file mode 100644 index 00000000000..062616ac888 --- /dev/null +++ b/docs/en-US/marvin.xml @@ -0,0 +1,32 @@ + + +%BOOK_ENTITIES; +]> + + + +
+ Marvin + Marvin is the &PRODUCT; automation framework. It originated as a tool for integration testing but is now also used to build DevCloud as well as to provide a Python &PRODUCT; API binding. + + Marvin's complete documenation is on the wiki at https://cwiki.apache.org/CLOUDSTACK/testing-with-python.html + The source code is located at tools/marvin + +
diff --git a/docs/en-US/source-build.xml b/docs/en-US/source-build.xml new file mode 100644 index 00000000000..33218962174 --- /dev/null +++ b/docs/en-US/source-build.xml @@ -0,0 +1,49 @@ + + +%BOOK_ENTITIES; +]> + + + +
+ Building &PRODUCT; from Source + + Prior to the 4.0.0 incubating release, Ant was used to build &PRODUCT;. Starting with 4.0.0 a migration to Maven is underway. + The website and the wiki contain up to date information on the build procedure at: + + https://cwiki.apache.org/CLOUDSTACK/building-with-maven.html + http://incubator.apache.org/cloudstack/develop/environment.html + + + The overarching steps to build &PRODUCT; are:. + + Install the prerequisites and setup your environment + Understand that various Maven profiles and build targets + Deploy and test your build + If needed, learn how to build binaries + + + + Learning Maven is outside the scope of this documentation. + Go to the Maven website at http://maven.apache.org/guides/getting-started/index.html + + +
+ diff --git a/docs/en-US/tools.xml b/docs/en-US/tools.xml new file mode 100644 index 00000000000..66fe894e171 --- /dev/null +++ b/docs/en-US/tools.xml @@ -0,0 +1,29 @@ + + +%BOOK_ENTITIES; +]> + + + + + Tools + + + diff --git a/docs/publican-devguide.cfg b/docs/publican-devguide.cfg index 006c0d16d00..a8e28aea37b 100644 --- a/docs/publican-devguide.cfg +++ b/docs/publican-devguide.cfg @@ -20,7 +20,7 @@ xml_lang: en-US type: Book -docname: API_Developers_Guide +docname: Developers_Guide brand: cloudstack chunk_first: 1 chunk_section_depth: 1 diff --git a/patches/systemvm/debian/config/etc/sysctl.conf b/patches/systemvm/debian/config/etc/sysctl.conf index 7f945b0d4b1..961d471dfa9 100644 --- a/patches/systemvm/debian/config/etc/sysctl.conf +++ b/patches/systemvm/debian/config/etc/sysctl.conf @@ -8,7 +8,7 @@ net.ipv4.ip_forward = 1 # Controls source route verification -net.ipv4.conf.default.rp_filter = 1 +net.ipv4.conf.default.rp_filter = 0 # Do not accept source routing net.ipv4.conf.default.accept_source_route = 0 diff --git a/patches/systemvm/debian/config/root/edithosts.sh b/patches/systemvm/debian/config/root/edithosts.sh index 3c6102d61d5..37c75a53022 100755 --- a/patches/systemvm/debian/config/root/edithosts.sh +++ b/patches/systemvm/debian/config/root/edithosts.sh @@ -88,7 +88,7 @@ echo "0 $mac $ip $host *" >> $DHCP_LEASES #edit hosts file as well sed -i /"$ip "/d $HOSTS -sed -i /"$host "/d $HOSTS +sed -i /" $host$"/d $HOSTS echo "$ip $host" >> $HOSTS if [ "$dflt" != "" ] diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index e0187e67689..b52e2d8a0b0 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -1104,8 +1104,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements String secondaryStorageUrl = cmd.getSecondaryStorageURL(); KVMStoragePool secondaryStoragePool = null; try { - KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(pool - .getUuid()); + KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool( + pool.getType(), + pool.getUuid()); String volumeName = UUID.randomUUID().toString(); if (copyToSecondary) { @@ -1114,20 +1115,21 @@ public class LibvirtComputingResource extends ServerResourceBase implements .getVolumePath()); String volumeDestPath = "/volumes/" + cmd.getVolumeId() + File.separator; - secondaryStoragePool = _storagePoolMgr - .getStoragePoolByURI(secondaryStorageUrl); + secondaryStoragePool = _storagePoolMgr.getStoragePoolByURI( + secondaryStorageUrl); secondaryStoragePool.createFolder(volumeDestPath); secondaryStoragePool.delete(); - secondaryStoragePool = _storagePoolMgr - .getStoragePoolByURI(secondaryStorageUrl - + volumeDestPath); - _storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, - secondaryStoragePool); + secondaryStoragePool = _storagePoolMgr.getStoragePoolByURI( + secondaryStorageUrl + + volumeDestPath); + _storagePoolMgr.copyPhysicalDisk(volume, + destVolumeName,secondaryStoragePool); return new CopyVolumeAnswer(cmd, true, null, null, volumeName); } else { volumePath = "/volumes/" + cmd.getVolumeId() + File.separator; - secondaryStoragePool = _storagePoolMgr - .getStoragePoolByURI(secondaryStorageUrl + volumePath); + secondaryStoragePool = _storagePoolMgr.getStoragePoolByURI( + secondaryStorageUrl + + volumePath); KVMPhysicalDisk volume = secondaryStoragePool .getPhysicalDisk(cmd.getVolumePath() + ".qcow2"); _storagePoolMgr.copyPhysicalDisk(volume, volumeName, @@ -1145,7 +1147,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements protected Answer execute(DeleteStoragePoolCommand cmd) { try { - _storagePoolMgr.deleteStoragePool(cmd.getPool().getUuid()); + _storagePoolMgr.deleteStoragePool(cmd.getPool().getType(), + cmd.getPool().getUuid()); return new Answer(cmd); } catch (CloudRuntimeException e) { return new Answer(cmd, false, e.toString()); @@ -1186,7 +1189,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements KVMPhysicalDisk vol = null; long disksize; try { - primaryPool = _storagePoolMgr.getStoragePool(pool.getUuid()); + primaryPool = _storagePoolMgr.getStoragePool(pool.getType(), + pool.getUuid()); disksize = dskch.getSize(); if (cmd.getTemplateUrl() != null) { @@ -1268,8 +1272,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements VolumeTO vol = cmd.getVolume(); try { - KVMStoragePool pool = _storagePoolMgr.getStoragePool(vol - .getPoolUuid()); + KVMStoragePool pool = _storagePoolMgr.getStoragePool( + vol.getPoolType(), + vol.getPoolUuid()); pool.deletePhysicalDisk(vol.getPath()); String vmName = cmd.getVmName(); String poolPath = pool.getLocalPath(); @@ -1281,7 +1286,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements File patchVbd = new File(poolPath + File.separator + vmName + "-patchdisk"); if(patchVbd.exists()){ try { - _storagePoolMgr.deleteVbdByPath(patchVbd.getAbsolutePath()); + _storagePoolMgr.deleteVbdByPath(vol.getPoolType(),patchVbd.getAbsolutePath()); } catch(CloudRuntimeException e) { s_logger.warn("unable to destroy patch disk '" + patchVbd.getAbsolutePath() + "' while removing root disk for " + vmName + " : " + e); @@ -1643,8 +1648,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements } } - KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(cmd - .getPool().getUuid()); + KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool( + cmd.getPool().getType(), + cmd.getPool().getUuid()); if (primaryPool.getType() == StoragePoolType.RBD) { s_logger.debug("Snapshots are not supported on RBD volumes"); @@ -1721,8 +1727,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements try { Connect conn = LibvirtConnection.getConnection(); - secondaryStoragePool = _storagePoolMgr - .getStoragePoolByURI(secondaryStoragePoolUrl); + secondaryStoragePool = _storagePoolMgr.getStoragePoolByURI( + secondaryStoragePoolUrl); String ssPmountPath = secondaryStoragePool.getLocalPath(); snapshotRelPath = File.separator + "snapshots" + File.separator @@ -1732,8 +1738,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements snapshotDestPath = ssPmountPath + File.separator + "snapshots" + File.separator + dcId + File.separator + accountId + File.separator + volumeId; - KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(cmd - .getPrimaryStoragePoolNameLabel()); + KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool( + cmd.getPool().getType(), + cmd.getPrimaryStoragePoolNameLabel()); KVMPhysicalDisk snapshotDisk = primaryPool.getPhysicalDisk(cmd .getVolumePath()); Script command = new Script(_manageSnapshotPath, _cmdsTimeout, @@ -1760,8 +1767,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements } } - KVMStoragePool primaryStorage = _storagePoolMgr.getStoragePool(cmd - .getPool().getUuid()); + KVMStoragePool primaryStorage = _storagePoolMgr.getStoragePool( + cmd.getPool().getType(), + cmd.getPool().getUuid()); if (state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING && !primaryStorage.isExternalSnapshot()) { String vmUuid = vm.getUUIDString(); @@ -1845,7 +1853,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements KVMStoragePool secondaryStoragePool = null; try { secondaryStoragePool = _storagePoolMgr.getStoragePoolByURI(cmd - .getSecondaryStorageUrl()); + .getSecondaryStorageUrl()); String ssPmountPath = secondaryStoragePool.getLocalPath(); String snapshotDestPath = ssPmountPath + File.separator @@ -1875,15 +1883,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements String snapshotPath = cmd.getSnapshotUuid(); int index = snapshotPath.lastIndexOf("/"); snapshotPath = snapshotPath.substring(0, index); - KVMStoragePool secondaryPool = _storagePoolMgr - .getStoragePoolByURI(cmd.getSecondaryStorageUrl() - + snapshotPath); + KVMStoragePool secondaryPool = _storagePoolMgr.getStoragePoolByURI( + cmd.getSecondaryStorageUrl() + + snapshotPath); KVMPhysicalDisk snapshot = secondaryPool.getPhysicalDisk(cmd .getSnapshotName()); String primaryUuid = cmd.getPrimaryStoragePoolNameLabel(); KVMStoragePool primaryPool = _storagePoolMgr - .getStoragePool(primaryUuid); + .getStoragePool(cmd.getPool().getType(), + primaryUuid); String volUuid = UUID.randomUUID().toString(); KVMPhysicalDisk disk = _storagePoolMgr.copyPhysicalDisk(snapshot, volUuid, primaryPool); @@ -1918,8 +1927,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements KVMPhysicalDisk snapshot = snapshotPool.getPhysicalDisk(cmd .getSnapshotName()); - secondaryPool = _storagePoolMgr.getStoragePoolByURI(cmd - .getSecondaryStorageUrl()); + secondaryPool = _storagePoolMgr.getStoragePoolByURI( + cmd.getSecondaryStorageUrl()); String templatePath = secondaryPool.getLocalPath() + File.separator + templateInstallFolder; @@ -1968,8 +1977,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements protected GetStorageStatsAnswer execute(final GetStorageStatsCommand cmd) { try { - KVMStoragePool sp = _storagePoolMgr.getStoragePool(cmd - .getStorageId()); + KVMStoragePool sp = _storagePoolMgr.getStoragePool( + cmd.getPooltype(), + cmd.getStorageId()); return new GetStorageStatsAnswer(cmd, sp.getCapacity(), sp.getUsed()); } catch (CloudRuntimeException e) { @@ -1988,11 +1998,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements + cmd.getTemplateId() + File.separator; String templateInstallFolder = "/template/tmpl/" + templateFolder; - secondaryStorage = _storagePoolMgr - .getStoragePoolByURI(secondaryStorageURL); + secondaryStorage = _storagePoolMgr.getStoragePoolByURI( + secondaryStorageURL); - KVMStoragePool primary = _storagePoolMgr.getStoragePool(cmd - .getPrimaryStoragePoolNameLabel()); + KVMStoragePool primary = _storagePoolMgr.getStoragePool( + cmd.getPool().getType(), + cmd.getPrimaryStoragePoolNameLabel()); KVMPhysicalDisk disk = primary.getPhysicalDisk(cmd.getVolumePath()); String tmpltPath = secondaryStorage.getLocalPath() + File.separator + templateInstallFolder; @@ -2114,8 +2125,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements } /* Copy volume to primary storage */ - KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(cmd - .getPoolUuid()); + KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool( + cmd.getPool().getType(), + cmd.getPoolUuid()); KVMPhysicalDisk primaryVol = _storagePoolMgr.copyPhysicalDisk( tmplVol, UUID.randomUUID().toString(), primaryPool); @@ -2136,9 +2148,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements } protected Answer execute(ModifyStoragePoolCommand cmd) { + String poolType = cmd.getPool().getType().toString(); KVMStoragePool storagepool = _storagePoolMgr.createStoragePool(cmd - .getPool().getUuid(), cmd.getPool().getHost(), cmd.getPool().getPort(), - cmd.getPool().getPath(), cmd.getPool().getUserInfo(), cmd.getPool().getType()); + .getPool().getUuid(), cmd.getPool().getHost(), + cmd.getPool().getPort(), cmd.getPool().getPath(), + cmd.getPool().getUserInfo(), cmd.getPool().getType()); if (storagepool == null) { return new Answer(cmd, false, " Failed to create storage pool"); } @@ -2276,8 +2290,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements private AttachVolumeAnswer execute(AttachVolumeCommand cmd) { try { Connect conn = LibvirtConnection.getConnection(); - KVMStoragePool primary = _storagePoolMgr.getStoragePool(cmd - .getPoolUuid()); + KVMStoragePool primary = _storagePoolMgr.getStoragePool( + cmd.getPooltype(), + cmd.getPoolUuid()); KVMPhysicalDisk disk = primary.getPhysicalDisk(cmd.getVolumePath()); attachOrDetachDisk(conn, cmd.getAttach(), cmd.getVmName(), disk, cmd.getDeviceId().intValue()); @@ -2924,8 +2939,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements int index = isoPath.lastIndexOf("/"); String path = isoPath.substring(0, index); String name = isoPath.substring(index + 1); - KVMStoragePool secondaryPool = _storagePoolMgr - .getStoragePoolByURI(path); + KVMStoragePool secondaryPool = _storagePoolMgr.getStoragePoolByURI( + path); KVMPhysicalDisk isoVol = secondaryPool.getPhysicalDisk(name); return isoVol.getPath(); } else { @@ -2952,11 +2967,13 @@ public class LibvirtComputingResource extends ServerResourceBase implements int index = volPath.lastIndexOf("/"); String volDir = volPath.substring(0, index); String volName = volPath.substring(index + 1); - KVMStoragePool secondaryStorage = _storagePoolMgr - .getStoragePoolByURI(volDir); + KVMStoragePool secondaryStorage = _storagePoolMgr. + getStoragePoolByURI(volDir); physicalDisk = secondaryStorage.getPhysicalDisk(volName); } else if (volume.getType() != Volume.Type.ISO) { - pool = _storagePoolMgr.getStoragePool(volume.getPoolUuid()); + pool = _storagePoolMgr.getStoragePool( + volume.getPoolType(), + volume.getPoolUuid()); physicalDisk = pool.getPhysicalDisk(volume.getPath()); } @@ -3034,7 +3051,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements DiskDef rootDisk = disks.get(0); VolumeTO rootVol = getVolume(vmSpec, Volume.Type.ROOT); String patchName = vmName + "-patchdisk"; - KVMStoragePool pool = _storagePoolMgr.getStoragePool(rootVol.getPoolUuid()); + KVMStoragePool pool = _storagePoolMgr.getStoragePool( + rootVol.getPoolType(), + rootVol.getPoolUuid()); String patchDiskPath = pool.getLocalPath() + "/" + patchName; List phyDisks = pool.listPhysicalDisks(); @@ -3130,7 +3149,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements } try { - KVMStoragePool pool = _storagePoolMgr.getStoragePool(poolUuid); + //we use libvirt since we passed a libvirt connection to cleanupDisk + KVMStoragePool pool = _storagePoolMgr.getStoragePool(null, poolUuid); if (pool != null) { pool.delete(); } @@ -3148,8 +3168,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements int index = isoPath.lastIndexOf("/"); String path = isoPath.substring(0, index); String name = isoPath.substring(index + 1); - KVMStoragePool secondaryPool = _storagePoolMgr - .getStoragePoolByURI(path); + KVMStoragePool secondaryPool = _storagePoolMgr.getStoragePoolByURI( + path); KVMPhysicalDisk isoVol = secondaryPool.getPhysicalDisk(name); isoPath = isoVol.getPath(); @@ -4416,4 +4436,5 @@ public class LibvirtComputingResource extends ServerResourceBase implements return new Answer(cmd, success, ""); } + } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 2c0e0ac9b6f..ac611275e5a 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -16,8 +16,12 @@ // under the License. package com.cloud.hypervisor.kvm.storage; +import java.net.URI; +import java.net.URISyntaxException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.HashMap; +import java.util.UUID; import com.cloud.hypervisor.kvm.resource.KVMHABase; import com.cloud.hypervisor.kvm.resource.KVMHABase.PoolType; @@ -25,11 +29,22 @@ import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk.PhysicalDiskFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageLayer; +import com.cloud.utils.exception.CloudRuntimeException; public class KVMStoragePoolManager { private StorageAdaptor _storageAdaptor; private KVMHAMonitor _haMonitor; private final Map _storagePools = new ConcurrentHashMap(); + private final Map _storageMapper = new HashMap(); + + private StorageAdaptor getStorageAdaptor(StoragePoolType type) { + StorageAdaptor adaptor = _storageMapper.get(type.toString()); + if (adaptor == null) { + // LibvirtStorageAdaptor is selected by default + adaptor = _storageMapper.get("libvirt"); + } + return adaptor; + } private void addStoragePool(String uuid) { synchronized (_storagePools) { @@ -42,20 +57,49 @@ public class KVMStoragePoolManager { public KVMStoragePoolManager(StorageLayer storagelayer, KVMHAMonitor monitor) { this._storageAdaptor = new LibvirtStorageAdaptor(storagelayer); this._haMonitor = monitor; + this._storageMapper.put("libvirt", new LibvirtStorageAdaptor(storagelayer)); + // add other storage adaptors here + // this._storageMapper.put("newadaptor", new NewStorageAdaptor(storagelayer)); } - public KVMStoragePool getStoragePool(String uuid) { - return this._storageAdaptor.getStoragePool(uuid); + public KVMStoragePool getStoragePool(StoragePoolType type, String uuid) { + StorageAdaptor adaptor = getStorageAdaptor(type); + return adaptor.getStoragePool(uuid); } public KVMStoragePool getStoragePoolByURI(String uri) { - return this._storageAdaptor.getStoragePoolByURI(uri); + URI storageUri = null; + + try { + storageUri = new URI(uri); + } catch (URISyntaxException e) { + throw new CloudRuntimeException(e.toString()); + } + + String sourcePath = null; + String uuid = null; + String sourceHost = ""; + StoragePoolType protocol = null; + if (storageUri.getScheme().equalsIgnoreCase("nfs")) { + sourcePath = storageUri.getPath(); + sourcePath = sourcePath.replace("//", "/"); + sourceHost = storageUri.getHost(); + uuid = UUID.nameUUIDFromBytes( + new String(sourceHost + sourcePath).getBytes()).toString(); + protocol = StoragePoolType.NetworkFilesystem; + } + + return createStoragePool(uuid, sourceHost, 0, sourcePath, "", protocol); } - public KVMStoragePool createStoragePool(String name, String host, int port, String path, - String userInfo, StoragePoolType type) { - KVMStoragePool pool = this._storageAdaptor.createStoragePool(name, + public KVMStoragePool createStoragePool( String name, String host, int port, + String path, String userInfo, + StoragePoolType type) { + StorageAdaptor adaptor = getStorageAdaptor(type); + KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type); + + // LibvirtStorageAdaptor-specific statement if (type == StoragePoolType.NetworkFilesystem) { KVMHABase.NfsStoragePool nfspool = new KVMHABase.NfsStoragePool( pool.getUuid(), host, path, pool.getLocalPath(), @@ -66,28 +110,33 @@ public class KVMStoragePoolManager { return pool; } - public boolean deleteStoragePool(String uuid) { + public boolean deleteStoragePool(StoragePoolType type, String uuid) { + StorageAdaptor adaptor = getStorageAdaptor(type); _haMonitor.removeStoragePool(uuid); - this._storageAdaptor.deleteStoragePool(uuid); + adaptor.deleteStoragePool(uuid); _storagePools.remove(uuid); return true; } - public boolean deleteVbdByPath(String diskPath) { - return this._storageAdaptor.deleteVbdByPath(diskPath); + public boolean deleteVbdByPath(StoragePoolType type, String diskPath) { + StorageAdaptor adaptor = getStorageAdaptor(type); + return adaptor.deleteVbdByPath(diskPath); } public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, KVMStoragePool destPool) { + StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); + + // LibvirtStorageAdaptor-specific statement if (destPool.getType() == StoragePoolType.RBD) { - return this._storageAdaptor.createDiskFromTemplate(template, name, + return adaptor.createDiskFromTemplate(template, name, KVMPhysicalDisk.PhysicalDiskFormat.RAW, template.getSize(), destPool); } else if (destPool.getType() == StoragePoolType.CLVM) { - return this._storageAdaptor.createDiskFromTemplate(template, name, + return adaptor.createDiskFromTemplate(template, name, KVMPhysicalDisk.PhysicalDiskFormat.RAW, template.getSize(), destPool); } else { - return this._storageAdaptor.createDiskFromTemplate(template, name, + return adaptor.createDiskFromTemplate(template, name, KVMPhysicalDisk.PhysicalDiskFormat.QCOW2, template.getSize(), destPool); } @@ -96,22 +145,22 @@ public class KVMStoragePoolManager { public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool) { - return this._storageAdaptor.createTemplateFromDisk(disk, name, format, + StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); + return adaptor.createTemplateFromDisk(disk, name, format, size, destPool); } public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool) { - return this._storageAdaptor.copyPhysicalDisk(disk, name, destPool); + StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); + return adaptor.copyPhysicalDisk(disk, name, destPool); } public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool) { - return this._storageAdaptor.createDiskFromSnapshot(snapshot, + StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); + return adaptor.createDiskFromSnapshot(snapshot, snapshotName, name, destPool); } - public KVMPhysicalDisk getPhysicalDiskFromUrl(String url) { - return this._storageAdaptor.getPhysicalDiskFromURI(url); - } } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 2a603cb0f47..d350ef994fb 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -725,38 +725,6 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { return newDisk; } - @Override - public KVMStoragePool getStoragePoolByURI(String uri) { - URI storageUri = null; - - try { - storageUri = new URI(uri); - } catch (URISyntaxException e) { - throw new CloudRuntimeException(e.toString()); - } - - String sourcePath = null; - String uuid = null; - String sourceHost = ""; - StoragePoolType protocal = null; - if (storageUri.getScheme().equalsIgnoreCase("nfs")) { - sourcePath = storageUri.getPath(); - sourcePath = sourcePath.replace("//", "/"); - sourceHost = storageUri.getHost(); - uuid = UUID.nameUUIDFromBytes( - new String(sourceHost + sourcePath).getBytes()).toString(); - protocal = StoragePoolType.NetworkFilesystem; - } - - return createStoragePool(uuid, sourceHost, 0, sourcePath, "", protocal); - } - - @Override - public KVMPhysicalDisk getPhysicalDiskFromURI(String uri) { - // TODO Auto-generated method stub - return null; - } - @Override public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool) { diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java index ef1e7c9302a..79c3b926b33 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java @@ -55,10 +55,6 @@ public interface StorageAdaptor { public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool); - public KVMStoragePool getStoragePoolByURI(String uri); - - public KVMPhysicalDisk getPhysicalDiskFromURI(String uri); - public boolean refresh(KVMStoragePool pool); public boolean deleteStoragePool(KVMStoragePool pool); diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index 967a2993f2e..313703c0a82 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -17,6 +17,10 @@ package com.cloud.hypervisor.xen.resource; +import java.beans.BeanInfo; +import java.beans.IntrospectionException; +import java.beans.Introspector; +import java.beans.PropertyDescriptor; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -24,11 +28,13 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.StringReader; +import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -179,6 +185,7 @@ import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; import com.cloud.agent.api.to.IpAddressTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.PortForwardingRuleTO; +import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.StaticNatRuleTO; import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.agent.api.to.SwiftTO; @@ -217,6 +224,8 @@ import com.cloud.storage.template.TemplateInfo; import com.cloud.template.VirtualMachineTemplate.BootloaderType; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; +import com.cloud.utils.S3Utils; +import com.cloud.utils.StringUtils; import com.cloud.utils.Ternary; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @@ -6507,7 +6516,14 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe } finally { deleteSnapshotBackup(conn, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotBackupUuid); } - } + } else if (cmd.getS3() != null) { + try { + backupSnapshotToS3(conn, cmd.getS3(), snapshotSr.getUuid(conn), snapshotBackupUuid, isISCSI, wait); + snapshotBackupUuid = snapshotBackupUuid + ".vhd"; + } finally { + deleteSnapshotBackup(conn, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotBackupUuid); + } + } success = true; } finally { if( snapshotSr != null) { @@ -6524,6 +6540,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe snapshotBackupUuid = snapshotPaUuid + ".vhd"; } success = true; + } else if (cmd.getS3() != null) { + backupSnapshotToS3(conn, cmd.getS3(), primaryStorageSRUuid, snapshotPaUuid, isISCSI, wait); } else { snapshotBackupUuid = backupSnapshot(conn, primaryStorageSRUuid, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotUuid, prevBackupUuid, isISCSI, wait); success = (snapshotBackupUuid != null); @@ -6546,6 +6564,88 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe return new BackupSnapshotAnswer(cmd, success, details, snapshotBackupUuid, fullbackup); } + private static List serializeProperties(final Object object, + final Class propertySet) { + + assert object != null; + assert propertySet != null; + assert propertySet.isAssignableFrom(object.getClass()); + + try { + + final BeanInfo beanInfo = Introspector.getBeanInfo(propertySet); + final PropertyDescriptor[] descriptors = beanInfo + .getPropertyDescriptors(); + + final List serializedProperties = new ArrayList(); + for (final PropertyDescriptor descriptor : descriptors) { + + serializedProperties.add(descriptor.getName()); + final Object value = descriptor.getReadMethod().invoke(object); + serializedProperties.add(value != null ? value.toString() + : "null"); + + } + + return Collections.unmodifiableList(serializedProperties); + + } catch (IntrospectionException e) { + s_logger.warn( + "Ignored IntrospectionException when serializing class " + + object.getClass().getCanonicalName(), e); + } catch (IllegalArgumentException e) { + s_logger.warn( + "Ignored IllegalArgumentException when serializing class " + + object.getClass().getCanonicalName(), e); + } catch (IllegalAccessException e) { + s_logger.warn( + "Ignored IllegalAccessException when serializing class " + + object.getClass().getCanonicalName(), e); + } catch (InvocationTargetException e) { + s_logger.warn( + "Ignored InvocationTargetException when serializing class " + + object.getClass().getCanonicalName(), e); + } + + return Collections.emptyList(); + + } + + private boolean backupSnapshotToS3(final Connection connection, + final S3TO s3, final String srUuid, final String snapshotUuid, + final Boolean iSCSIFlag, final int wait) { + + final String filename = iSCSIFlag ? "VHD-" + snapshotUuid + : snapshotUuid + ".vhd"; + final String dir = (iSCSIFlag ? "/dev/VG_XenStorage-" + : "/var/run/sr-mount/") + srUuid; + final String key = StringUtils.join("/", "snapshots", snapshotUuid); + + try { + + final List parameters = new ArrayList( + serializeProperties(s3, S3Utils.ClientOptions.class)); + parameters.addAll(Arrays.asList("operation", "put", "directory", + dir, "filename", filename, "iSCSIFlag", + iSCSIFlag.toString(), "key", key)); + final String result = callHostPluginAsync(connection, "s3xen", + "s3", wait, + parameters.toArray(new String[parameters.size()])); + + if (result != null && result.equals("true")) { + return true; + } + + } catch (Exception e) { + s_logger.error(String.format( + "S3 upload failed of snapshot %1$s due to %2$s.", + snapshotUuid, e.toString()), e); + } + + return false; + + } + protected CreateVolumeFromSnapshotAnswer execute(final CreateVolumeFromSnapshotCommand cmd) { Connection conn = getConnection(); String primaryStorageNameLabel = cmd.getPrimaryStoragePoolNameLabel(); diff --git a/plugins/network-elements/dns-notifier/pom.xml b/plugins/network-elements/dns-notifier/pom.xml new file mode 100644 index 00000000000..0b4c981af98 --- /dev/null +++ b/plugins/network-elements/dns-notifier/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + + org.apache.cloudstack + cloudstack-plugins + 4.1.0-SNAPSHOT + ../../pom.xml + + org.apache.cloudstack + cloud-plugin-example-dns-notifier + 4.1.0-SNAPSHOT + CloudStack Dns Notifier Example + This is sample source code on how to write a plugin for CloudStack + + install + src + test + + + + org.apache.cloudstack + cloud-api + ${project.version} + + + org.apache.cloudstack + cloud-utils + ${project.version} + + + diff --git a/plugins/network-elements/dns-notifier/resources/components-example.xml b/plugins/network-elements/dns-notifier/resources/components-example.xml new file mode 100755 index 00000000000..592b01c3d2a --- /dev/null +++ b/plugins/network-elements/dns-notifier/resources/components-example.xml @@ -0,0 +1,220 @@ + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/plugins/network-elements/dns-notifier/src/org/apache/cloudstack/network/element/DnsNotifier.java b/plugins/network-elements/dns-notifier/src/org/apache/cloudstack/network/element/DnsNotifier.java new file mode 100644 index 00000000000..f9aa063973b --- /dev/null +++ b/plugins/network-elements/dns-notifier/src/org/apache/cloudstack/network/element/DnsNotifier.java @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.network.element; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import javax.ejb.Local; +import javax.naming.ConfigurationException; + +import com.cloud.deploy.DeployDestination; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.Network; +import com.cloud.network.Network.Capability; +import com.cloud.network.Network.Provider; +import com.cloud.network.Network.Service; +import com.cloud.network.PhysicalNetworkServiceProvider; +import com.cloud.network.element.NetworkElement; +import com.cloud.offering.NetworkOffering; +import com.cloud.vm.NicProfile; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; + +/** + * @author ahuang + * + */ +@Local(NetworkElement.class) +public class DnsNotifier implements NetworkElement { + String _name = null; + + public DnsNotifier() { + + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + _name = name; + return true; + } + + @Override + public String getName() { + return _name; + } + + @Override + public boolean start() { + return true; + } + + @Override + public boolean stop() { + return true; + } + + @Override + public Map> getCapabilities() { + Map> caps = new HashMap>(); + caps.put(Service.Dns, new HashMap()); + return caps; + } + + @Override + public Provider getProvider() { + return null; + } + + @Override + public boolean implement(Network network, NetworkOffering offering, DeployDestination dest, ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException, + InsufficientCapacityException { + return true; + } + + @Override + public boolean prepare(Network network, NicProfile nic, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context) throws ConcurrentOperationException, + ResourceUnavailableException, InsufficientCapacityException { + // signal to the dns server that this vm is up and running and set the ip address to hostname mapping. + vm.getHostName(); + nic.getIp4Address(); + nic.getIp6Address(); + return true; + } + + @Override + public boolean release(Network network, NicProfile nic, VirtualMachineProfile vm, ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException { + vm.getHostName(); + nic.getIp4Address(); + nic.getIp6Address(); + // signal to the dns server that the vm is being shutdown and remove the mapping. + return true; + } + + @Override + public boolean shutdown(Network network, ReservationContext context, boolean cleanup) throws ConcurrentOperationException, ResourceUnavailableException { + return true; + } + + @Override + public boolean destroy(Network network, ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException { + return true; + } + + @Override + public boolean isReady(PhysicalNetworkServiceProvider provider) { + return true; + } + + @Override + public boolean shutdownProviderInstances(PhysicalNetworkServiceProvider provider, ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException { + return true; + } + + @Override + public boolean canEnableIndividualServices() { + return true; + } + + @Override + public boolean verifyServicesCombination(Set services) { + return true; + } + +} diff --git a/plugins/network-elements/nicira-nvp/src/com/cloud/network/guru/NiciraNvpGuestNetworkGuru.java b/plugins/network-elements/nicira-nvp/src/com/cloud/network/guru/NiciraNvpGuestNetworkGuru.java index a386ae53b47..1046a5a96b7 100644 --- a/plugins/network-elements/nicira-nvp/src/com/cloud/network/guru/NiciraNvpGuestNetworkGuru.java +++ b/plugins/network-elements/nicira-nvp/src/com/cloud/network/guru/NiciraNvpGuestNetworkGuru.java @@ -39,6 +39,7 @@ import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostDetailsDao; import com.cloud.network.Network; +import com.cloud.network.Network.Service; import com.cloud.network.NetworkManager; import com.cloud.network.NetworkProfile; import com.cloud.network.NetworkVO; @@ -50,9 +51,11 @@ import com.cloud.network.PhysicalNetwork; import com.cloud.network.PhysicalNetwork.IsolationMethod; import com.cloud.network.PhysicalNetworkVO; import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkServiceMapDao; import com.cloud.network.dao.NiciraNvpDao; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.offering.NetworkOffering; +import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; import com.cloud.resource.ResourceManager; import com.cloud.user.Account; import com.cloud.user.dao.AccountDao; @@ -88,6 +91,8 @@ public class NiciraNvpGuestNetworkGuru extends GuestNetworkGuru { AgentManager _agentMgr; @Inject HostDetailsDao _hostDetailsDao; + @Inject + NetworkOfferingServiceMapDao _ntwkOfferingSrvcDao; public NiciraNvpGuestNetworkGuru() { super(); @@ -100,7 +105,8 @@ public class NiciraNvpGuestNetworkGuru extends GuestNetworkGuru { if (networkType == NetworkType.Advanced && isMyTrafficType(offering.getTrafficType()) && offering.getGuestType() == Network.GuestType.Isolated - && isMyIsolationMethod(physicalNetwork)) { + && isMyIsolationMethod(physicalNetwork) + && _ntwkOfferingSrvcDao.areServicesSupportedByNetworkOffering(offering.getId(), Service.Connectivity)) { return true; } else { s_logger.trace("We only take care of Guest networks of type " + GuestType.Isolated + " in zone of type " + NetworkType.Advanced); diff --git a/plugins/network-elements/nicira-nvp/test/com/cloud/network/guru/NiciraNvpGuestNetworkGuruTest.java b/plugins/network-elements/nicira-nvp/test/com/cloud/network/guru/NiciraNvpGuestNetworkGuruTest.java new file mode 100644 index 00000000000..f8d9652da60 --- /dev/null +++ b/plugins/network-elements/nicira-nvp/test/com/cloud/network/guru/NiciraNvpGuestNetworkGuruTest.java @@ -0,0 +1,343 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.guru; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.CreateLogicalSwitchAnswer; +import com.cloud.agent.api.DeleteLogicalSwitchAnswer; +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.domain.Domain; +import com.cloud.exception.InsufficientVirtualNetworkCapcityException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.network.Network; +import com.cloud.network.Network.GuestType; +import com.cloud.network.Network.Service; +import com.cloud.network.Network.State; +import com.cloud.network.NetworkManager; +import com.cloud.network.NetworkProfile; +import com.cloud.network.NetworkVO; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.Networks.TrafficType; +import com.cloud.network.NiciraNvpDeviceVO; +import com.cloud.network.PhysicalNetworkVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NiciraNvpDao; +import com.cloud.network.dao.PhysicalNetworkDao; +import com.cloud.offering.NetworkOffering; +import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; +import com.cloud.user.Account; +import com.cloud.vm.ReservationContext; + +import edu.emory.mathcs.backport.java.util.Arrays; + +public class NiciraNvpGuestNetworkGuruTest { + PhysicalNetworkDao physnetdao = mock (PhysicalNetworkDao.class); + NiciraNvpDao nvpdao = mock(NiciraNvpDao.class); + DataCenterDao dcdao = mock(DataCenterDao.class); + NetworkOfferingServiceMapDao nosd = mock(NetworkOfferingServiceMapDao.class); + AgentManager agentmgr = mock (AgentManager.class); + NetworkManager netmgr = mock (NetworkManager.class); + HostDao hostdao = mock (HostDao.class); + NetworkDao netdao = mock(NetworkDao.class); + NiciraNvpGuestNetworkGuru guru; + + + @Before + public void setUp() { + guru = new NiciraNvpGuestNetworkGuru(); + ((GuestNetworkGuru) guru)._physicalNetworkDao = physnetdao; + guru._physicalNetworkDao = physnetdao; + guru._niciraNvpDao = nvpdao; + guru._dcDao = dcdao; + guru._ntwkOfferingSrvcDao = nosd; + guru._networkMgr = netmgr; + guru._hostDao = hostdao; + guru._agentMgr = agentmgr; + guru._networkDao = netdao; + + DataCenterVO dc = mock(DataCenterVO.class); + when(dc.getNetworkType()).thenReturn(NetworkType.Advanced); + when(dc.getGuestNetworkCidr()).thenReturn("10.1.1.1/24"); + + when(dcdao.findById((Long) any())).thenReturn((DataCenterVO) dc); + } + + @Test + public void testCanHandle() { + NetworkOffering offering = mock(NetworkOffering.class); + when(offering.getId()).thenReturn(42L); + when(offering.getTrafficType()).thenReturn(TrafficType.Guest); + when(offering.getGuestType()).thenReturn(GuestType.Isolated); + + PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class); + when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" })); + when(physnet.getId()).thenReturn(42L); + + when(nosd.areServicesSupportedByNetworkOffering(42L, Service.Connectivity)).thenReturn(true); + + assertTrue(guru.canHandle(offering, NetworkType.Advanced, physnet) == true); + + // Not supported TrafficType != Guest + when(offering.getTrafficType()).thenReturn(TrafficType.Management); + assertFalse(guru.canHandle(offering, NetworkType.Advanced, physnet) == true); + + // Not supported: GuestType Shared + when(offering.getTrafficType()).thenReturn(TrafficType.Guest); + when(offering.getGuestType()).thenReturn(GuestType.Shared); + assertFalse(guru.canHandle(offering, NetworkType.Advanced, physnet) == true); + + // Not supported: Basic networking + when(offering.getGuestType()).thenReturn(GuestType.Isolated); + assertFalse(guru.canHandle(offering, NetworkType.Basic, physnet) == true); + + // Not supported: IsolationMethod != STT + when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "VLAN" })); + assertFalse(guru.canHandle(offering, NetworkType.Advanced, physnet) == true); + + } + + + @Test + public void testDesign() { + PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class); + when(physnetdao.findById((Long) any())).thenReturn(physnet); + when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" })); + when(physnet.getId()).thenReturn(42L); + + NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class); + when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Arrays.asList(new NiciraNvpDeviceVO[] { device })); + when(device.getId()).thenReturn(1L); + + NetworkOffering offering = mock(NetworkOffering.class); + when(offering.getId()).thenReturn(42L); + when(offering.getTrafficType()).thenReturn(TrafficType.Guest); + when(offering.getGuestType()).thenReturn(GuestType.Isolated); + + when(nosd.areServicesSupportedByNetworkOffering(42L, Service.Connectivity)).thenReturn(true); + + DeploymentPlan plan = mock(DeploymentPlan.class); + Network network = mock(Network.class); + Account account = mock(Account.class); + + Network designednetwork = guru.design(offering, plan, network, account); + assertTrue(designednetwork != null); + assertTrue(designednetwork.getBroadcastDomainType() == BroadcastDomainType.Lswitch); + } + + @Test + public void testDesignNoElementOnPhysicalNetwork() { + PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class); + when(physnetdao.findById((Long) any())).thenReturn(physnet); + when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" })); + when(physnet.getId()).thenReturn(42L); + + NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class); + when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Collections. emptyList()); + + NetworkOffering offering = mock(NetworkOffering.class); + when(offering.getId()).thenReturn(42L); + when(offering.getTrafficType()).thenReturn(TrafficType.Guest); + when(offering.getGuestType()).thenReturn(GuestType.Isolated); + + DeploymentPlan plan = mock(DeploymentPlan.class); + Network network = mock(Network.class); + Account account = mock(Account.class); + + Network designednetwork = guru.design(offering, plan, network, account); + assertTrue(designednetwork == null); + } + + @Test + public void testDesignNoIsolationMethodSTT() { + PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class); + when(physnetdao.findById((Long) any())).thenReturn(physnet); + when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "VLAN" })); + when(physnet.getId()).thenReturn(42L); + + NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class); + when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Collections. emptyList()); + + NetworkOffering offering = mock(NetworkOffering.class); + when(offering.getId()).thenReturn(42L); + when(offering.getTrafficType()).thenReturn(TrafficType.Guest); + when(offering.getGuestType()).thenReturn(GuestType.Isolated); + + DeploymentPlan plan = mock(DeploymentPlan.class); + Network network = mock(Network.class); + Account account = mock(Account.class); + + Network designednetwork = guru.design(offering, plan, network, account); + assertTrue(designednetwork == null); + } + + @Test + public void testDesignNoConnectivityInOffering() { + PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class); + when(physnetdao.findById((Long) any())).thenReturn(physnet); + when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" })); + when(physnet.getId()).thenReturn(42L); + + NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class); + when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Arrays.asList(new NiciraNvpDeviceVO[] { device })); + when(device.getId()).thenReturn(1L); + + NetworkOffering offering = mock(NetworkOffering.class); + when(offering.getId()).thenReturn(42L); + when(offering.getTrafficType()).thenReturn(TrafficType.Guest); + when(offering.getGuestType()).thenReturn(GuestType.Isolated); + + when(nosd.areServicesSupportedByNetworkOffering(42L, Service.Connectivity)).thenReturn(false); + + DeploymentPlan plan = mock(DeploymentPlan.class); + Network network = mock(Network.class); + Account account = mock(Account.class); + + Network designednetwork = guru.design(offering, plan, network, account); + assertTrue(designednetwork == null); + } + + @Test + public void testImplement() throws InsufficientVirtualNetworkCapcityException { + PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class); + when(physnetdao.findById((Long) any())).thenReturn(physnet); + when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" })); + when(physnet.getId()).thenReturn(42L); + + NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class); + when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Arrays.asList(new NiciraNvpDeviceVO[] { device })); + when(device.getId()).thenReturn(1L); + + NetworkOffering offering = mock(NetworkOffering.class); + when(offering.getId()).thenReturn(42L); + when(offering.getTrafficType()).thenReturn(TrafficType.Guest); + when(offering.getGuestType()).thenReturn(GuestType.Isolated); + + when(nosd.areServicesSupportedByNetworkOffering(42L, Service.Connectivity)).thenReturn(false); + + DeploymentPlan plan = mock(DeploymentPlan.class); + + NetworkVO network = mock(NetworkVO.class); + when(network.getName()).thenReturn("testnetwork"); + when(network.getState()).thenReturn(State.Implementing); + + DeployDestination dest = mock(DeployDestination.class); + + DataCenter dc = mock(DataCenter.class); + when(dest.getDataCenter()).thenReturn(dc); + + HostVO niciraHost = mock(HostVO.class); + when(hostdao.findById(anyLong())).thenReturn(niciraHost); + when(niciraHost.getDetail("transportzoneuuid")).thenReturn("aaaa"); + when(niciraHost.getDetail("transportzoneisotype")).thenReturn("stt"); + when(niciraHost.getId()).thenReturn(42L); + + when(netmgr.findPhysicalNetworkId(anyLong(), (String) any(), (TrafficType) any())).thenReturn(42L); + Domain dom = mock(Domain.class); + when(dom.getName()).thenReturn("domain"); + Account acc = mock(Account.class); + when(acc.getAccountName()).thenReturn("accountname"); + ReservationContext res = mock(ReservationContext.class); + when(res.getDomain()).thenReturn(dom); + when(res.getAccount()).thenReturn(acc); + + CreateLogicalSwitchAnswer answer = mock(CreateLogicalSwitchAnswer.class); + when(answer.getResult()).thenReturn(true); + when(agentmgr.easySend(eq(42L), (Command)any())).thenReturn(answer); + + Network implementednetwork = guru.implement(network, offering, dest, res); + assertTrue(implementednetwork != null); + verify(agentmgr, times(1)).easySend(eq(42L), (Command)any()); + } + + @Test + public void testShutdown() throws InsufficientVirtualNetworkCapcityException, URISyntaxException { + PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class); + when(physnetdao.findById((Long) any())).thenReturn(physnet); + when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" })); + when(physnet.getId()).thenReturn(42L); + + NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class); + when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Arrays.asList(new NiciraNvpDeviceVO[] { device })); + when(device.getId()).thenReturn(1L); + + NetworkOffering offering = mock(NetworkOffering.class); + when(offering.getId()).thenReturn(42L); + when(offering.getTrafficType()).thenReturn(TrafficType.Guest); + when(offering.getGuestType()).thenReturn(GuestType.Isolated); + + when(nosd.areServicesSupportedByNetworkOffering(42L, Service.Connectivity)).thenReturn(false); + + DeploymentPlan plan = mock(DeploymentPlan.class); + + NetworkVO network = mock(NetworkVO.class); + when(network.getName()).thenReturn("testnetwork"); + when(network.getState()).thenReturn(State.Implementing); + when(network.getBroadcastDomainType()).thenReturn(BroadcastDomainType.Lswitch); + when(network.getBroadcastUri()).thenReturn(new URI("lswitch:aaaaa")); + when(network.getPhysicalNetworkId()).thenReturn(42L); + when(netdao.findById(42L)).thenReturn(network); + + DeployDestination dest = mock(DeployDestination.class); + + DataCenter dc = mock(DataCenter.class); + when(dest.getDataCenter()).thenReturn(dc); + + HostVO niciraHost = mock(HostVO.class); + when(hostdao.findById(anyLong())).thenReturn(niciraHost); + when(niciraHost.getDetail("transportzoneuuid")).thenReturn("aaaa"); + when(niciraHost.getDetail("transportzoneisotype")).thenReturn("stt"); + when(niciraHost.getId()).thenReturn(42L); + + when(netmgr.findPhysicalNetworkId(anyLong(), (String) any(), (TrafficType) any())).thenReturn(42L); + Domain dom = mock(Domain.class); + when(dom.getName()).thenReturn("domain"); + Account acc = mock(Account.class); + when(acc.getAccountName()).thenReturn("accountname"); + ReservationContext res = mock(ReservationContext.class); + when(res.getDomain()).thenReturn(dom); + when(res.getAccount()).thenReturn(acc); + + DeleteLogicalSwitchAnswer answer = mock(DeleteLogicalSwitchAnswer.class); + when(answer.getResult()).thenReturn(true); + when(agentmgr.easySend(eq(42L), (Command)any())).thenReturn(answer); + + NetworkProfile implementednetwork = mock(NetworkProfile.class); + when(implementednetwork.getId()).thenReturn(42L); + when(implementednetwork.getBroadcastUri()).thenReturn(new URI("lswitch:aaaa")); + when(offering.getSpecifyVlan()).thenReturn(false); + + guru.shutdown(implementednetwork, offering); + verify(agentmgr, times(1)).easySend(eq(42L), (Command)any()); + verify(implementednetwork, times(1)).setBroadcastUri(null); + } +} diff --git a/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElement.java b/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElement.java index b7a978e72d6..5e8639c3188 100644 --- a/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElement.java +++ b/plugins/network-elements/ovs/src/com/cloud/network/element/OvsElement.java @@ -16,7 +16,6 @@ // under the License. package com.cloud.network.element; -import java.util.List; import java.util.Map; import java.util.Set; @@ -65,18 +64,18 @@ public class OvsElement extends AdapterBase implements NetworkElement { @Override public boolean implement(Network network, NetworkOffering offering, DeployDestination dest, ReservationContext context) - throws ConcurrentOperationException, ResourceUnavailableException, - InsufficientCapacityException { + throws ConcurrentOperationException, ResourceUnavailableException, + InsufficientCapacityException { //Consider actually implementing the network here - return true; + return true; } @Override public boolean prepare(Network network, NicProfile nic, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context) - throws ConcurrentOperationException, ResourceUnavailableException, - InsufficientCapacityException { + throws ConcurrentOperationException, ResourceUnavailableException, + InsufficientCapacityException { if (nic.getBroadcastType() != Networks.BroadcastDomainType.Vswitch) { return true; } @@ -116,7 +115,7 @@ public class OvsElement extends AdapterBase implements NetworkElement { @Override public boolean isReady(PhysicalNetworkServiceProvider provider) { - return true; + return true; } @Override diff --git a/plugins/pom.xml b/plugins/pom.xml index 2009302423e..d71bd45aa29 100644 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -46,6 +46,7 @@ user-authenticators/md5 user-authenticators/plain-text user-authenticators/sha256salted + network-elements/dns-notifier diff --git a/pom.xml b/pom.xml index 2c2600b552f..4d31cded68c 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ 1.7.1 5.6.100-1-SNAPSHOT 3.1 - 4.0 + 4.1 5.1.21 1.3.1 3.1.3 @@ -82,7 +82,9 @@ 1.0-20081010.060147 4.1 1.9.5 - + 1.3.21.1 + 2.6 + 1.4 @@ -206,8 +208,12 @@ CHANGES INSTALL.md + .idea/ + *.log + **/*.patch **/.classpath **/.project + **/*.iml **/.settings/** .metadata/** .git/** diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen new file mode 100644 index 00000000000..4d9c12d6a5a --- /dev/null +++ b/scripts/vm/hypervisor/xenserver/s3xen @@ -0,0 +1,297 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Version @VERSION@ +# +# A plugin for executing script needed by cloud stack +from __future__ import with_statement + +from copy import copy +from datetime import datetime +from httplib import * +from string import join + +import os +import sys +import time +import hashlib +import base64 +import hmac +import traceback +import urllib2 + +import XenAPIPlugin +sys.path.extend(["/opt/xensource/sm/"]) +import util + +NULL = 'null' + +# Value conversion utility functions ... + + +def to_none(value): + return value if value is not None and value.strip() != NULL else None + + +def to_bool(value): + return True if to_none(value) in ['true', 'True', None] else False + + +def to_integer(value, default): + return int(value) if to_none(value) is not None else default + + +def optional_str_value(value, default): + return value if is_not_blank(value) else default + + +def is_not_blank(value): + return True if to_none(value) is not None and value.strip != '' else False + + +def get_optional_key(map, key, default=''): + return map[key] if key in map else default + + +def log(message): + util.SMlog('#### VMOPS %s ####' % message) + + +def echo(fn): + def wrapped(*v, **k): + name = fn.__name__ + log("enter %s ####" % name) + res = fn(*v, **k) + log("exit %s with result %s" % name, res) + return res + return wrapped + + +def require_str_value(value, error_message): + + if is_not_blank(value): + return value + + raise ValueError(error_message) + + +def retry(max_attempts, fn): + + attempts = 1 + while attempts <= max_attempts: + log("Attempting execution {0}/{1} of {2}". + format(attempts, max_attempts, fn.__name__)) + try: + return fn() + except: + if (attempts >= max_attempts): + raise + attempts = attempts + 1 + + +def compute_md5(filename, buffer_size=8192): + + hasher = hashlib.md5() + + with open(filename, 'rb') as file: + data = file.read(buffer_size) + while data != "": + hasher.update(data) + data = file.read(buffer_size) + + return base64.encodestring(hasher.digest())[:-1] + + +class S3Client(object): + + DEFAULT_END_POINT = 's3.amazonaws.com' + DEFAULT_CONNECTION_TIMEOUT = 50000 + DEFAULT_SOCKET_TIMEOUT = 50000 + DEFAULT_MAX_ERROR_RETRY = 3 + + HEADER_CONTENT_MD5 = 'Content-MD5' + HEADER_CONTENT_TYPE = 'Content-Type' + HEADER_CONTENT_LENGTH = 'Content-Length' + + def __init__(self, access_key, secret_key, end_point=None, + https_flag=None, connection_timeout=None, socket_timeout=None, + max_error_retry=None): + + self.access_key = require_str_value( + access_key, 'An access key must be specified.') + self.secret_key = require_str_value( + secret_key, 'A secret key must be specified.') + self.end_point = optional_str_value(end_point, self.DEFAULT_END_POINT) + self.https_flag = to_bool(https_flag) + self.connection_timeout = to_integer( + connection_timeout, self.DEFAULT_CONNECTION_TIMEOUT) + self.socket_timeout = to_integer( + socket_timeout, self.DEFAULT_SOCKET_TIMEOUT) + self.max_error_retry = to_integer( + max_error_retry, self.DEFAULT_MAX_ERROR_RETRY) + + def build_canocialized_resource(self, bucket, key): + + return '/{bucket}/{key}'.format(bucket=bucket, key=key) + + def noop_send_body(): + pass + + def noop_read(response): + return response.read() + + def do_operation( + self, method, bucket, key, input_headers={}, + fn_send_body=noop_send_body, fn_read=noop_read): + + headers = copy(input_headers) + headers['Expect'] = '100-continue' + + uri = self.build_canocialized_resource(bucket, key) + signature, request_date = self.sign_request(method, uri, headers) + headers['Authorization'] = "AWS {0}:{1}".format( + self.access_key, signature) + headers['Date'] = request_date + + connection = HTTPSConnection(self.end_point) \ + if self.https_flag else HTTPConnection(self.end_point) + connection.timeout = self.socket_timeout + + def perform_request(): + + connection.request(method, uri, fn_send_body(), headers) + response = connection.getresponse() + log("Sent {0} request to {1} {2} with headers {3}. \ + Got response status {4}: {5}". + format(method, self.end_point, uri, headers, + response.status, response.reason)) + return fn_read(response) + + try: + return retry(self.max_error_retry, perform_request) + finally: + connection.close() + + ''' + See http://bit.ly/MMC5de for more information regarding the creation of + AWS authorization tokens and header signing + ''' + def sign_request(self, operation, canocialized_resource, headers): + + request_date = datetime.utcnow( + ).strftime('%a, %d %b %Y %H:%M:%S +0000') + + content_hash = get_optional_key(headers, self.HEADER_CONTENT_MD5) + content_type = get_optional_key(headers, self.HEADER_CONTENT_TYPE) + + string_to_sign = join( + [operation, content_hash, content_type, request_date, + canocialized_resource], '\n') + + signature = base64.encodestring( + hmac.new(self.secret_key, string_to_sign.encode('utf8'), + hashlib.sha1).digest())[:-1] + + return signature, request_date + + def put(self, bucket, key, src_filename): + + headers = { + self.HEADER_CONTENT_MD5: compute_md5(src_filename), + self.HEADER_CONTENT_TYPE: 'application/octet-stream', + self.HEADER_CONTENT_LENGTH: os.stat(src_filename).st_size, + } + + def send_body(): + return open(src_filename, 'rb') + + self.do_operation('PUT', bucket, key, headers, send_body) + + def get(self, bucket, key, target_filename): + + def read(response): + + with open(target_filename, 'wb') as file: + while True: + block = response.read(8192) + if not block: + break + file.write(block) + + return self.do_operation('GET', bucket, key, fn_read=read) + + def delete(self, bucket, key): + + return self.do_operation('DELETE', bucket, key) + + +def parseArguments(args): + + # The keys in the args map will correspond to the properties defined on + # the com.cloud.utils.S3Utils#ClientOptions interface + client = S3Client( + args['accessKey'], args['secretKey'], args['endPoint'], + args['isHttps'], args['connectionTimeout'], args['socketTimeout']) + + operation = args['operation'] + bucket = args['bucket'] + key = args['key'] + filename = args['filename'] + + if is_blank(operation): + raise ValueError('An operation must be specified.') + + if is_blank(bucket): + raise ValueError('A bucket must be specified.') + + if is_blank(key): + raise ValueError('A value must be specified.') + + if is_blank(filename): + raise ValueError('A filename must be specified.') + + return client, operation, bucket, key, filename + + +@echo +def s3(session, args): + + client, operation, bucket, key, filename = parseArguments(args) + + try: + + if operation == 'put': + client.put(bucket, key, filename) + elif operation == 'get': + client.get(bucket, key, filename) + elif operation == 'delete': + client.delete(bucket, key, filename) + else: + raise RuntimeError( + "S3 plugin does not support operation {0}.".format(operation)) + + return 'true' + + except: + log("Operation {0} on file {1} from/in bucket {2} key {3}.".format( + operation, filename, bucket, key)) + log(traceback.format_exc()) + return 'false' + +if __name__ == "__main__": + XenAPIPlugin.dispatch({"s3": s3}) diff --git a/scripts/vm/hypervisor/xenserver/xenserver56/patch b/scripts/vm/hypervisor/xenserver/xenserver56/patch index d485414fdcb..36dba3dc06b 100644 --- a/scripts/vm/hypervisor/xenserver/xenserver56/patch +++ b/scripts/vm/hypervisor/xenserver/xenserver56/patch @@ -62,3 +62,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin swift=..,0755,/opt/xensource/bin swiftxen=..,0755,/etc/xapi.d/plugins +s3xen=..,0755,/etc/xapi.d/plugins + diff --git a/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch b/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch index 9fe9740756c..d20e60f2e49 100644 --- a/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch +++ b/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch @@ -61,3 +61,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin swift=..,0755,/opt/xensource/bin swiftxen=..,0755,/etc/xapi.d/plugins +s3xen=..,0755,/etc/xapi.d/plugins + diff --git a/scripts/vm/hypervisor/xenserver/xenserver60/patch b/scripts/vm/hypervisor/xenserver/xenserver60/patch index f0491092749..c9125f4c5b2 100644 --- a/scripts/vm/hypervisor/xenserver/xenserver60/patch +++ b/scripts/vm/hypervisor/xenserver/xenserver60/patch @@ -66,3 +66,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin swift=..,0755,/opt/xensource/bin swiftxen=..,0755,/etc/xapi.d/plugins +s3xen=..,0755,/etc/xapi.d/plugins + diff --git a/scripts/vm/network/security_group.py b/scripts/vm/network/security_group.py index b0798907c8a..dcb01a7604b 100755 --- a/scripts/vm/network/security_group.py +++ b/scripts/vm/network/security_group.py @@ -24,6 +24,7 @@ import sys import os import xml.dom.minidom from optparse import OptionParser, OptionGroup, OptParseError, BadOptionError, OptionError, OptionConflictError, OptionValueError +import re iptables = Command("iptables") bash = Command("/bin/bash") virsh = Command("virsh") @@ -420,12 +421,11 @@ def network_rules_for_rebooted_vm(vmName): delete_rules_for_vm_in_bridge_firewall_chain(vm_name) - brName = execute("iptables-save |grep physdev-is-bridged |grep FORWARD |grep BF |grep '\-o' |awk '{print $9}'").split("\n") - if brName is None: + brName = execute("iptables-save |grep physdev-is-bridged |grep FORWARD |grep BF |grep '\-o' |awk '{print $9}' | head -1").strip() + if brName is None or brName is "": brName = "cloudbr0" else: - brName.pop() - brName = brName[0].split("-")[1] + brName = re.sub("^BF-", "", brName) if 1 in [ vm_name.startswith(c) for c in ['r-', 's-', 'v-'] ]: diff --git a/server/pom.xml b/server/pom.xml index e3308d85ea2..5ae926f9111 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -62,6 +62,11 @@ jstl ${cs.jstl.version} + + commons-codec + commons-codec + ${cs.codec.version} + org.apache.cloudstack cloud-utils @@ -69,7 +74,6 @@ tests test - install diff --git a/server/src/com/cloud/api/ApiDBUtils.java b/server/src/com/cloud/api/ApiDBUtils.java index 3b5f634f216..cdd5339665c 100755 --- a/server/src/com/cloud/api/ApiDBUtils.java +++ b/server/src/com/cloud/api/ApiDBUtils.java @@ -115,6 +115,7 @@ import com.cloud.storage.StoragePoolVO; import com.cloud.storage.StorageStats; import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateHostVO; +import com.cloud.storage.VMTemplateS3VO; import com.cloud.storage.VMTemplateSwiftVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume.Type; @@ -129,6 +130,7 @@ import com.cloud.storage.dao.UploadDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VMTemplateHostDao; +import com.cloud.storage.dao.VMTemplateS3Dao; import com.cloud.storage.dao.VMTemplateSwiftDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeHostDao; @@ -196,6 +198,7 @@ public class ApiDBUtils { private static VMTemplateDetailsDao _templateDetailsDao; private static VMTemplateHostDao _templateHostDao; private static VMTemplateSwiftDao _templateSwiftDao; + private static VMTemplateS3Dao _templateS3Dao; private static UploadDao _uploadDao; private static UserDao _userDao; private static UserStatisticsDao _userStatsDao; @@ -260,6 +263,7 @@ public class ApiDBUtils { _templateDetailsDao = locator.getDao(VMTemplateDetailsDao.class); _templateHostDao = locator.getDao(VMTemplateHostDao.class); _templateSwiftDao = locator.getDao(VMTemplateSwiftDao.class); + _templateS3Dao = locator.getDao(VMTemplateS3Dao.class); _uploadDao = locator.getDao(UploadDao.class); _userDao = locator.getDao(UserDao.class); _userStatsDao = locator.getDao(UserStatisticsDao.class); @@ -575,6 +579,10 @@ public class ApiDBUtils { return _templateSwiftDao.findOneByTemplateId(templateId); } + public static VMTemplateS3VO findTemplateS3Ref(long templateId) { + return _templateS3Dao.findOneByTemplateId(templateId); + } + public static UploadVO findUploadById(Long id) { return _uploadDao.findById(id); } diff --git a/server/src/com/cloud/api/ApiDispatcher.java b/server/src/com/cloud/api/ApiDispatcher.java index dfe4a1fc789..9fd0b2e9f06 100755 --- a/server/src/com/cloud/api/ApiDispatcher.java +++ b/server/src/com/cloud/api/ApiDispatcher.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.api; +import static org.apache.commons.lang.StringUtils.isNotBlank; + import java.lang.reflect.Field; import java.text.DateFormat; import java.text.ParseException; @@ -41,7 +43,6 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.utils.IdentityProxy; import com.cloud.server.ManagementServer; import com.cloud.user.Account; import com.cloud.user.UserContext; @@ -417,10 +418,20 @@ public class ApiDispatcher { } break; case FLOAT: - field.set(cmdObj, Float.valueOf(paramObj.toString())); + // Assuming that the parameters have been checked for required before now, + // we ignore blank or null values and defer to the command to set a default + // value for optional parameters ... + if (paramObj != null && isNotBlank(paramObj.toString())) { + field.set(cmdObj, Float.valueOf(paramObj.toString())); + } break; case INTEGER: - field.set(cmdObj, Integer.valueOf(paramObj.toString())); + // Assuming that the parameters have been checked for required before now, + // we ignore blank or null values and defer to the command to set a default + // value for optional parameters ... + if (paramObj != null && isNotBlank(paramObj.toString())) { + field.set(cmdObj, Integer.valueOf(paramObj.toString())); + } break; case LIST: List listParam = new ArrayList(); diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index ebe8415048b..a5747101215 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -16,6 +16,9 @@ // under the License. package com.cloud.api; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; + import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Date; @@ -82,6 +85,7 @@ import com.cloud.api.response.RemoteAccessVpnResponse; import com.cloud.api.response.ResourceCountResponse; import com.cloud.api.response.ResourceLimitResponse; import com.cloud.api.response.ResourceTagResponse; +import com.cloud.api.response.S3Response; import com.cloud.api.response.SecurityGroupResponse; import com.cloud.api.response.SecurityGroupResultObject; import com.cloud.api.response.SecurityGroupRuleResponse; @@ -131,7 +135,6 @@ import com.cloud.dc.Vlan.VlanType; import com.cloud.dc.VlanVO; import com.cloud.domain.Domain; import com.cloud.event.Event; -import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.host.HostStats; import com.cloud.host.HostVO; @@ -189,6 +192,7 @@ import com.cloud.server.ResourceTag.TaggedResourceType; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.GuestOS; import com.cloud.storage.GuestOSCategoryVO; +import com.cloud.storage.S3; import com.cloud.storage.Snapshot; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; @@ -200,6 +204,7 @@ import com.cloud.storage.StorageStats; import com.cloud.storage.Swift; import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateHostVO; +import com.cloud.storage.VMTemplateS3VO; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateSwiftVO; import com.cloud.storage.VMTemplateVO; @@ -743,6 +748,25 @@ public class ApiResponseHelper implements ResponseGenerator { return swiftResponse; } + @Override + public S3Response createS3Response(final S3 result) { + + final S3Response response = new S3Response(); + + response.setAccessKey(result.getAccessKey()); + response.setConnectionTimeout(result.getConnectionTimeout()); + response.setEndPoint(result.getEndPoint()); + response.setHttpsFlag(result.getHttpsFlag()); + response.setMaxErrorRetry(result.getMaxErrorRetry()); + response.setObjectId(result.getId()); + response.setSecretKey(result.getSecretKey()); + response.setSocketTimeout(result.getSocketTimeout()); + response.setTemplateBucketName(result.getBucketName()); + + return response; + + } + @Override public VlanIpRangeResponse createVlanIpRangeResponse(Vlan vlan) { Long podId = ApiDBUtils.getPodIdForVlan(vlan.getId()); @@ -2153,7 +2177,7 @@ public class ApiResponseHelper implements ResponseGenerator { @Override public List createIsoResponses(long isoId, Long zoneId, boolean readyOnly) { - List isoResponses = new ArrayList(); + final List isoResponses = new ArrayList(); VirtualMachineTemplate iso = findTemplateById(isoId); if (iso.getTemplateType() == TemplateType.PERHOST) { TemplateResponse isoResponse = new TemplateResponse(); @@ -2191,11 +2215,17 @@ public class ApiResponseHelper implements ResponseGenerator { return isoResponses; } else { if (zoneId == null || zoneId == -1) { - isoResponses = createSwiftIsoResponses(iso); + isoResponses.addAll(createSwiftIsoResponses(iso)); if (!isoResponses.isEmpty()) { return isoResponses; } - List dcs = new ArrayList(); + + isoResponses.addAll(createS3IsoResponses(iso)); + if (!isoResponses.isEmpty()) { + return isoResponses; + } + + final List dcs = new ArrayList(); dcs.addAll(ApiDBUtils.listZones()); for (DataCenterVO dc : dcs) { isoResponses.addAll(createIsoResponses(iso, dc.getId(), readyOnly)); @@ -2207,6 +2237,65 @@ public class ApiResponseHelper implements ResponseGenerator { } } + private List createS3IsoResponses(final VirtualMachineTemplate iso) { + + final VMTemplateS3VO s3Iso = ApiDBUtils.findTemplateS3Ref(iso.getId()); + + if (s3Iso == null) { + return emptyList(); + } + + final TemplateResponse templateResponse = new TemplateResponse(); + + templateResponse.setId(iso.getId()); + templateResponse.setName(iso.getName()); + templateResponse.setDisplayText(iso.getDisplayText()); + templateResponse.setPublic(iso.isPublicTemplate()); + templateResponse.setExtractable(iso.isExtractable()); + templateResponse.setCreated(s3Iso.getCreated()); + templateResponse.setReady(true); + templateResponse.setBootable(iso.isBootable()); + templateResponse.setFeatured(iso.isFeatured()); + templateResponse.setCrossZones(iso.isCrossZones()); + templateResponse.setChecksum(iso.getChecksum()); + templateResponse.setDetails(iso.getDetails()); + + final GuestOS os = ApiDBUtils.findGuestOSById(iso.getGuestOSId()); + + if (os != null) { + templateResponse.setOsTypeId(os.getId()); + templateResponse.setOsTypeName(os.getDisplayName()); + } else { + templateResponse.setOsTypeId(-1L); + templateResponse.setOsTypeName(""); + } + + final Account account = ApiDBUtils.findAccountByIdIncludingRemoved(iso.getAccountId()); + populateAccount(templateResponse, account.getId()); + populateDomain(templateResponse, account.getDomainId()); + + boolean isAdmin = false; + if ((account == null) || BaseCmd.isAdmin(account.getType())) { + isAdmin = true; + } + + // If the user is an admin, add the template download status + if (isAdmin || account.getId() == iso.getAccountId()) { + // add download status + templateResponse.setStatus("Successfully Installed"); + } + + final Long isoSize = s3Iso.getSize(); + if (isoSize > 0) { + templateResponse.setSize(isoSize); + } + + templateResponse.setObjectName("iso"); + + return singletonList(templateResponse); + + } + private List createSwiftIsoResponses(VirtualMachineTemplate iso) { long isoId = iso.getId(); List isoResponses = new ArrayList(); diff --git a/server/src/com/cloud/api/ApiServer.java b/server/src/com/cloud/api/ApiServer.java index 7365b36622f..f42025c1d23 100755 --- a/server/src/com/cloud/api/ApiServer.java +++ b/server/src/com/cloud/api/ApiServer.java @@ -108,6 +108,7 @@ import com.cloud.user.UserVO; import com.cloud.utils.IdentityProxy; import com.cloud.utils.Pair; import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.StringUtils; import com.cloud.utils.component.ComponentLocator; import com.cloud.utils.component.PluggableService; import com.cloud.utils.concurrency.NamedThreadFactory; @@ -313,7 +314,7 @@ public class ApiServer implements HttpRequestHandler { InetAddress remoteAddr = ((SocketHttpServerConnection) connObj).getRemoteAddress(); sb.append(remoteAddr.toString() + " -- "); } - sb.append(request.getRequestLine()); + sb.append(StringUtils.cleanString(request.getRequestLine().toString())); try { String uri = request.getRequestLine().getUri(); @@ -589,22 +590,10 @@ public class ApiServer implements HttpRequestHandler { if (command.equals("createSSHKeyPair")){ auditTrailSb.append("This result was not logged because it contains sensitive data."); } else { - auditTrailSb.append(result); + auditTrailSb.append(StringUtils.cleanString(result)); } - /* - * if (command.equals("queryAsyncJobResult")){ //For this command we need to also log job status and job - * resultcode for - * (Pair pair : resultValues){ String key = pair.first(); if (key.equals("jobstatus")){ - * auditTrailSb.append(" "); auditTrailSb.append(key); auditTrailSb.append("="); - * auditTrailSb.append(pair.second()); - * }else if (key.equals("jobresultcode")){ auditTrailSb.append(" "); auditTrailSb.append(key); - * auditTrailSb.append("="); - * auditTrailSb.append(pair.second()); } } }else { for (Pair pair : resultValues){ if - * (pair.first().equals("jobid")){ // Its an async job so report the jobid auditTrailSb.append(" "); - * auditTrailSb.append(pair.first()); auditTrailSb.append("="); auditTrailSb.append(pair.second()); } } } - */ } - + private static boolean isCommandAvailable(String commandName) { boolean isCommandAvailable = false; isCommandAvailable = s_allCommands.contains(commandName); diff --git a/server/src/com/cloud/api/ApiServlet.java b/server/src/com/cloud/api/ApiServlet.java index b5c42725104..091bf189cc2 100755 --- a/server/src/com/cloud/api/ApiServlet.java +++ b/server/src/com/cloud/api/ApiServlet.java @@ -36,6 +36,7 @@ import com.cloud.server.ManagementServer; import com.cloud.user.Account; import com.cloud.user.AccountService; import com.cloud.user.UserContext; +import com.cloud.utils.StringUtils; import com.cloud.utils.component.ComponentLocator; import com.cloud.utils.exception.CloudRuntimeException; @@ -126,7 +127,7 @@ public class ApiServlet extends HttpServlet { String reqStr = ""; if (s_logger.isDebugEnabled()) { reqStr = auditTrailSb.toString() + " " + req.getQueryString(); - s_logger.debug("===START=== " + reqStr); + s_logger.debug("===START=== " + StringUtils.cleanString(reqStr)); } try { @@ -343,7 +344,7 @@ public class ApiServlet extends HttpServlet { } finally { s_accessLogger.info(auditTrailSb.toString()); if (s_logger.isDebugEnabled()) { - s_logger.debug("===END=== " + reqStr); + s_logger.debug("===END=== " + StringUtils.cleanString(reqStr)); } // cleanup user context to prevent from being peeked in other request context UserContext.unregisterContext(); diff --git a/server/src/com/cloud/api/commands/AddTrafficMonitorCmd.java b/server/src/com/cloud/api/commands/AddTrafficMonitorCmd.java index 6bbdde7ecc3..f253b378bc1 100644 --- a/server/src/com/cloud/api/commands/AddTrafficMonitorCmd.java +++ b/server/src/com/cloud/api/commands/AddTrafficMonitorCmd.java @@ -50,11 +50,25 @@ public class AddTrafficMonitorCmd extends BaseCmd { @Parameter(name=ApiConstants.URL, type=CommandType.STRING, required = true, description="URL of the traffic monitor Host") private String url; + + @Parameter(name=ApiConstants.INCL_ZONES, type=CommandType.STRING, description="Traffic going into the listed zones will be metered") + private String inclZones; + + @Parameter(name=ApiConstants.EXCL_ZONES, type=CommandType.STRING, description="Traffic going into the listed zones will not be metered") + private String exclZones; /////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// + public String getInclZones() { + return inclZones; + } + + public String getExclZones() { + return exclZones; + } + public Long getZoneId() { return zoneId; } diff --git a/server/src/com/cloud/api/doc/ApiXmlDocWriter.java b/server/src/com/cloud/api/doc/ApiXmlDocWriter.java index d31ef5a8a5f..ffdd5be237b 100644 --- a/server/src/com/cloud/api/doc/ApiXmlDocWriter.java +++ b/server/src/com/cloud/api/doc/ApiXmlDocWriter.java @@ -314,6 +314,11 @@ public class ApiXmlDocWriter { impl = clas.getSuperclass().getAnnotation(Implementation.class); } + if (impl == null) { + throw new IllegalStateException(String.format("An %1$s annotation is required for class %2$s.", + Implementation.class.getCanonicalName(), clas.getCanonicalName())); + } + if (impl.includeInApiDoc()) { String commandDescription = impl.description(); if (commandDescription != null && !commandDescription.isEmpty()) { diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 66ac2762de1..2eb269fc1e2 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -140,8 +140,8 @@ public enum Config { JobExpireMinutes("Advanced", ManagementServer.class, String.class, "job.expire.minutes", "1440", "Time (in minutes) for async-jobs to be kept in system", null), JobCancelThresholdMinutes("Advanced", ManagementServer.class, String.class, "job.cancel.threshold.minutes", "60", "Time (in minutes) for async-jobs to be forcely cancelled if it has been in process for long", null), SwiftEnable("Advanced", ManagementServer.class, Boolean.class, "swift.enable", "false", "enable swift ", null), - - EventPurgeInterval("Advanced", ManagementServer.class, Integer.class, "event.purge.interval", "86400", "The interval (in seconds) to wait before running the event purge thread", null), + S3Enable("Advanced", ManagementServer.class, Boolean.class, "s3.enable", "false", "enable s3 ", null), + EventPurgeInterval("Advanced", ManagementServer.class, Integer.class, "event.purge.interval", "86400", "The interval (in seconds) to wait before running the event purge thread", null), AccountCleanupInterval("Advanced", ManagementServer.class, Integer.class, "account.cleanup.interval", "86400", "The interval (in seconds) between cleanup for removed accounts", null), AllowPublicUserTemplates("Advanced", ManagementServer.class, Integer.class, "allow.public.user.templates", "true", "If false, users will not be able to create public templates.", null), InstanceName("Advanced", AgentManager.class, String.class, "instance.name", "VM", "Name of the deployment instance.", "instanceName"), @@ -274,6 +274,8 @@ public enum Config { DirectNetworkStatsInterval("Usage", ManagementServer.class, Integer.class, "direct.network.stats.interval", "86400", "Interval (in seconds) to collect stats from Traffic Monitor", null), UsageSanityCheckInterval("Usage", ManagementServer.class, Integer.class, "usage.sanity.check.interval", null, "Interval (in days) to check sanity of usage data", null), UsageAggregationTimezone("Usage", ManagementServer.class, String.class, "usage.aggregation.timezone", "GMT", "The timezone to use for usage stats aggregation", null), + TrafficSentinelIncludeZones("Usage", ManagementServer.class, Integer.class, "traffic.sentinel.include.zones", "EXTERNAL", "Traffic going into specified list of zones is metered. For metering all traffic leave this parameter empty", null), + TrafficSentinelExcludeZones("Usage", ManagementServer.class, Integer.class, "traffic.sentinel.exclude.zones", "", "Traffic going into specified list of zones is not metered.", null), // Hidden UseSecondaryStorageVm("Hidden", ManagementServer.class, Boolean.class, "secondary.storage.vm", "false", "Deploys a VM per zone to manage secondary storage if true, otherwise secondary storage is mounted on management server", null), diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index 33baaf12ca4..2d7dfe290a6 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -63,7 +63,9 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.SwiftVO; import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.S3Dao; import com.cloud.storage.dao.SwiftDao; +import com.cloud.storage.s3.S3Manager; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.swift.SwiftManager; import com.cloud.test.IPRangeConfig; @@ -118,6 +120,8 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura @Inject SwiftDao _swiftDao; @Inject + S3Dao _s3Dao; + @Inject ServiceOfferingDao _serviceOfferingDao; @Inject DiskOfferingDao _diskOfferingDao; @@ -158,6 +162,8 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura @Inject SwiftManager _swiftMgr; @Inject + S3Manager _s3Mgr; + @Inject PhysicalNetworkTrafficTypeDao _trafficTypeDao; @Inject NicDao _nicDao; @@ -419,6 +425,14 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura if (swift != null) { return " can not change " + Config.SwiftEnable.key() + " after you have added Swift"; } + if (this._s3Mgr.isS3Enabled()) { + return String.format("Swift is not supported when S3 is enabled."); + } + } + if (Config.S3Enable.key().equals(name)) { + if (this._swiftMgr.isSwiftEnabled()) { + return String.format("S3-backed Secondary Storage is not supported when Swift is enabled."); + } } return null; } @@ -1520,6 +1534,7 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura createDefaultSystemNetworks(zone.getId()); _swiftMgr.propagateSwiftTmplteOnZone(zone.getId()); + _s3Mgr.propagateTemplatesToZone(zone); txn.commit(); return zone; } catch (Exception ex) { diff --git a/server/src/com/cloud/configuration/DefaultComponentLibrary.java b/server/src/com/cloud/configuration/DefaultComponentLibrary.java index ef61044d06b..226ca49a56b 100755 --- a/server/src/com/cloud/configuration/DefaultComponentLibrary.java +++ b/server/src/com/cloud/configuration/DefaultComponentLibrary.java @@ -147,6 +147,7 @@ import com.cloud.storage.dao.DiskOfferingDaoImpl; import com.cloud.storage.dao.GuestOSCategoryDaoImpl; import com.cloud.storage.dao.GuestOSDaoImpl; import com.cloud.storage.dao.LaunchPermissionDaoImpl; +import com.cloud.storage.dao.S3DaoImpl; import com.cloud.storage.dao.SnapshotDaoImpl; import com.cloud.storage.dao.SnapshotPolicyDaoImpl; import com.cloud.storage.dao.SnapshotScheduleDaoImpl; @@ -159,11 +160,13 @@ import com.cloud.storage.dao.VMTemplateDaoImpl; import com.cloud.storage.dao.VMTemplateDetailsDaoImpl; import com.cloud.storage.dao.VMTemplateHostDaoImpl; import com.cloud.storage.dao.VMTemplatePoolDaoImpl; +import com.cloud.storage.dao.VMTemplateS3DaoImpl; import com.cloud.storage.dao.VMTemplateSwiftDaoImpl; import com.cloud.storage.dao.VMTemplateZoneDaoImpl; import com.cloud.storage.dao.VolumeDaoImpl; import com.cloud.storage.dao.VolumeHostDaoImpl; import com.cloud.storage.download.DownloadMonitorImpl; +import com.cloud.storage.s3.S3ManagerImpl; import com.cloud.storage.secondary.SecondaryStorageManagerImpl; import com.cloud.storage.snapshot.SnapshotManagerImpl; import com.cloud.storage.snapshot.SnapshotSchedulerImpl; @@ -270,6 +273,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com addDao("VMTemplateHostDao", VMTemplateHostDaoImpl.class); addDao("VolumeHostDao", VolumeHostDaoImpl.class); addDao("VMTemplateSwiftDao", VMTemplateSwiftDaoImpl.class); + addDao("VMTemplateS3Dao", VMTemplateS3DaoImpl.class); addDao("UploadDao", UploadDaoImpl.class); addDao("VMTemplatePoolDao", VMTemplatePoolDaoImpl.class); addDao("LaunchPermissionDao", LaunchPermissionDaoImpl.class); @@ -318,6 +322,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com addDao("KeystoreDao", KeystoreDaoImpl.class); addDao("DcDetailsDao", DcDetailsDaoImpl.class); addDao("SwiftDao", SwiftDaoImpl.class); + addDao("S3Dao", S3DaoImpl.class); addDao("AgentTransferMapDao", HostTransferMapDaoImpl.class); addDao("ProjectDao", ProjectDaoImpl.class); addDao("InlineLoadBalancerNicMapDao", InlineLoadBalancerNicMapDaoImpl.class); @@ -403,6 +408,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com info.addParameter("consoleproxy.sslEnabled", "true"); addManager("ProjectManager", ProjectManagerImpl.class); addManager("SwiftManager", SwiftManagerImpl.class); + addManager("S3Manager", S3ManagerImpl.class); addManager("StorageNetworkManager", StorageNetworkManagerImpl.class); addManager("ExternalLoadBalancerUsageManager", ExternalLoadBalancerUsageManagerImpl.class); addManager("HA Manager", HighAvailabilityManagerImpl.class); diff --git a/server/src/com/cloud/network/NetworkUsageManagerImpl.java b/server/src/com/cloud/network/NetworkUsageManagerImpl.java index 13e5e241855..42c72ad04bc 100755 --- a/server/src/com/cloud/network/NetworkUsageManagerImpl.java +++ b/server/src/com/cloud/network/NetworkUsageManagerImpl.java @@ -108,6 +108,8 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta @Inject ResourceManager _resourceMgr; ScheduledExecutorService _executor; int _networkStatsInterval; + String _TSinclZones; + String _TSexclZones; protected SearchBuilder AllocatedIpSearch; @Override @@ -148,8 +150,8 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta hostParams.put("zone", String.valueOf(zoneId)); hostParams.put("ipaddress", ipAddress); hostParams.put("url", cmd.getUrl()); - //hostParams("numRetries", numRetries); - //hostParams("timeout", timeout); + hostParams.put("inclZones", (cmd.getInclZones() != null) ? cmd.getInclZones() : _TSinclZones); + hostParams.put("exclZones", (cmd.getExclZones() != null) ? cmd.getExclZones() : _TSexclZones); hostParams.put("guid", guid); hostParams.put("name", guid); @@ -162,7 +164,14 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta Map hostDetails = new HashMap(); hostDetails.put("url", cmd.getUrl()); hostDetails.put("last_collection", ""+System.currentTimeMillis()); - + if(cmd.getInclZones() != null){ + hostDetails.put("inclZones", cmd.getInclZones()); + } + if(cmd.getExclZones() != null){ + hostDetails.put("exclZones", cmd.getExclZones()); + } + + Host trafficMonitor = _resourceMgr.addHost(zoneId, resource, Host.Type.TrafficMonitor, hostDetails); return trafficMonitor; } @@ -222,6 +231,8 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta AllocatedIpSearch.done(); _networkStatsInterval = NumbersUtil.parseInt(_configDao.getValue(Config.DirectNetworkStatsInterval.key()), 86400); + _TSinclZones = _configDao.getValue(Config.TrafficSentinelIncludeZones.key()); + _TSexclZones = _configDao.getValue(Config.TrafficSentinelExcludeZones.key()); _agentMgr.registerForHostEvents(new DirectNetworkStatsListener( _networkStatsInterval), true, false, false); _resourceMgr.registerResourceStateAdapter(this.getClass().getSimpleName(), this); return true; @@ -372,7 +383,7 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta //Get usage for Ips which were assigned for the entire duration if(fullDurationIpUsage.size() > 0){ - DirectNetworkUsageCommand cmd = new DirectNetworkUsageCommand(IpList, lastCollection, now); + DirectNetworkUsageCommand cmd = new DirectNetworkUsageCommand(IpList, lastCollection, now, _TSinclZones, _TSexclZones); DirectNetworkUsageAnswer answer = (DirectNetworkUsageAnswer) _agentMgr.easySend(host.getId(), cmd); if (answer == null || !answer.getResult()) { String details = (answer != null) ? answer.getDetails() : "details unavailable"; @@ -405,7 +416,7 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta for(UsageIPAddressVO usageIp : IpPartialUsage){ IpList = new ArrayList() ; IpList.add(usageIp.getAddress()); - DirectNetworkUsageCommand cmd = new DirectNetworkUsageCommand(IpList, usageIp.getAssigned(), usageIp.getReleased()); + DirectNetworkUsageCommand cmd = new DirectNetworkUsageCommand(IpList, usageIp.getAssigned(), usageIp.getReleased(), _TSinclZones, _TSexclZones); DirectNetworkUsageAnswer answer = (DirectNetworkUsageAnswer) _agentMgr.easySend(host.getId(), cmd); if (answer == null || !answer.getResult()) { String details = (answer != null) ? answer.getDetails() : "details unavailable"; @@ -532,8 +543,11 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta @Override public DeleteHostAnswer deleteHost(HostVO host, boolean isForced, boolean isForceDeleteStorage) throws UnableDeleteHostException { - // TODO Auto-generated method stub - return null; + if(host.getType() != Host.Type.TrafficMonitor){ + return null; + } + + return new DeleteHostAnswer(true); } } diff --git a/server/src/com/cloud/network/guru/GuestNetworkGuru.java b/server/src/com/cloud/network/guru/GuestNetworkGuru.java index cebfb08e952..2a8a092b835 100755 --- a/server/src/com/cloud/network/guru/GuestNetworkGuru.java +++ b/server/src/com/cloud/network/guru/GuestNetworkGuru.java @@ -442,15 +442,17 @@ public abstract class GuestNetworkGuru extends AdapterBase implements NetworkGur @Override public void shutdown(NetworkProfile profile, NetworkOffering offering) { - s_logger.debug("Releasing vnet for the network id=" + profile.getId()); - if (profile.getBroadcastUri() != null && !offering.getSpecifyVlan()) { + + if (profile.getBroadcastDomainType() == BroadcastDomainType.Vlan && + profile.getBroadcastUri() != null && !offering.getSpecifyVlan()) { + s_logger.debug("Releasing vnet for the network id=" + profile.getId()); _dcDao.releaseVnet(profile.getBroadcastUri().getHost(), profile.getDataCenterId(), profile.getPhysicalNetworkId(), profile.getAccountId(), profile.getReservationId()); EventUtils.saveEvent(UserContext.current().getCallerUserId(), profile.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_ZONE_VLAN_RELEASE, "Released Zone Vlan: " +profile.getBroadcastUri().getHost()+" for Network: "+profile.getId(), 0); - profile.setBroadcastUri(null); } + profile.setBroadcastUri(null); } @Override diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index ced601b179b..8d50886d13a 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -30,6 +30,11 @@ import java.util.Set; import javax.ejb.Local; import javax.naming.ConfigurationException; +import com.cloud.api.commands.AddS3Cmd; +import com.cloud.api.commands.ListS3sCmd; +import com.cloud.storage.S3; +import com.cloud.storage.S3VO; +import com.cloud.storage.s3.S3Manager; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; @@ -181,6 +186,8 @@ public class ResourceManagerImpl implements ResourceManager, ResourceService, Ma @Inject protected SwiftManager _swiftMgr; @Inject + protected S3Manager _s3Mgr; + @Inject protected HostDetailsDao _hostDetailsDao; @Inject protected ConfigurationDao _configDao; @@ -561,6 +568,16 @@ public class ResourceManagerImpl implements ResourceManager, ResourceService, Ma return _swiftMgr.listSwifts(cmd); } + @Override + public S3 discoverS3(final AddS3Cmd cmd) throws DiscoveryException { + return this._s3Mgr.addS3(cmd); + } + + @Override + public List listS3s(final ListS3sCmd cmd) { + return this._s3Mgr.listS3s(cmd); + } + private List discoverHostsFull(Long dcId, Long podId, Long clusterId, String clusterName, String url, String username, String password, String hypervisorType, List hostTags, Map params) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException { URI uri = null; diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index e252633f8e3..3e3a67a96c2 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -151,11 +151,13 @@ import com.cloud.storage.dao.StoragePoolWorkDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VMTemplateS3Dao; import com.cloud.storage.dao.VMTemplateSwiftDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeHostDao; import com.cloud.storage.download.DownloadMonitor; import com.cloud.storage.listener.StoragePoolMonitor; +import com.cloud.storage.s3.S3Manager; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.storage.snapshot.SnapshotScheduler; @@ -262,6 +264,10 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag @Inject protected VMTemplateSwiftDao _vmTemplateSwiftDao = null; @Inject + protected VMTemplateS3Dao _vmTemplateS3Dao; + @Inject + protected S3Manager _s3Mgr; + @Inject protected VMTemplateDao _vmTemplateDao = null; @Inject protected StoragePoolHostDao _poolHostDao = null; @@ -663,7 +669,6 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag String vdiUUID = null; Long snapshotId = snapshot.getId(); Long volumeId = snapshot.getVolumeId(); - String primaryStoragePoolNameLabel = pool.getUuid(); // pool's uuid is actually the namelabel. Long dcId = snapshot.getDataCenterId(); String secondaryStoragePoolUrl = _snapMgr.getSecondaryStorageURL(snapshot); long accountId = snapshot.getAccountId(); @@ -707,8 +712,10 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag try { if (snapshot.getSwiftId() != null && snapshot.getSwiftId() != 0) { _snapshotMgr.downloadSnapshotsFromSwift(snapshot); + } else if (snapshot.getS3Id() != null && snapshot.getS3Id() != 0) { + _snapshotMgr.downloadSnapshotsFromS3(snapshot); } - CreateVolumeFromSnapshotCommand createVolumeFromSnapshotCommand = new CreateVolumeFromSnapshotCommand(primaryStoragePoolNameLabel, secondaryStoragePoolUrl, dcId, accountId, volumeId, + CreateVolumeFromSnapshotCommand createVolumeFromSnapshotCommand = new CreateVolumeFromSnapshotCommand(pool, secondaryStoragePoolUrl, dcId, accountId, volumeId, backedUpSnapshotUuid, snapshot.getName(), _createVolumeFromSnapshotWait); CreateVolumeFromSnapshotAnswer answer; if (!_snapshotDao.lockInLockTable(snapshotId.toString(), 10)) { @@ -1281,7 +1288,7 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag } URI uri = null; try { - uri = new URI(cmd.getUrl()); + uri = new URI(UriUtils.encodeURIComponent(cmd.getUrl())); if (uri.getScheme() == null) { throw new InvalidParameterValueException("scheme is null " + cmd.getUrl() + ", add nfs:// as a prefix"); } else if (uri.getScheme().equalsIgnoreCase("nfs")) { @@ -2983,6 +2990,14 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag if (tsvs != null && tsvs.size() > 0) { size = tsvs.get(0).getSize(); } + + if (size == null && _s3Mgr.isS3Enabled()) { + VMTemplateS3VO vmTemplateS3VO = _vmTemplateS3Dao.findOneByTemplateId(template.getId()); + if (vmTemplateS3VO != null) { + size = vmTemplateS3VO.getSize(); + } + } + if (size == null) { List sss = _vmTemplateHostDao.search(sc, null); if (sss == null || sss.size() == 0) { diff --git a/server/src/com/cloud/storage/dao/S3Dao.java b/server/src/com/cloud/storage/dao/S3Dao.java new file mode 100644 index 00000000000..ebea3531339 --- /dev/null +++ b/server/src/com/cloud/storage/dao/S3Dao.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.dao; + +import com.cloud.agent.api.to.S3TO; +import com.cloud.storage.S3VO; +import com.cloud.utils.db.GenericDao; + +public interface S3Dao extends GenericDao { + + S3TO getS3TO(final Long id); + +} diff --git a/server/src/com/cloud/storage/dao/S3DaoImpl.java b/server/src/com/cloud/storage/dao/S3DaoImpl.java new file mode 100644 index 00000000000..6162e6ebf73 --- /dev/null +++ b/server/src/com/cloud/storage/dao/S3DaoImpl.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.dao; + +import com.cloud.agent.api.to.S3TO; +import com.cloud.storage.S3VO; +import com.cloud.utils.db.GenericDaoBase; + +import javax.ejb.Local; + +@Local(S3Dao.class) +public class S3DaoImpl extends GenericDaoBase implements S3Dao { + + @Override + public S3TO getS3TO(final Long id) { + + if (id != null) { + + final S3VO s3VO = findById(id); + if (s3VO != null) { + return s3VO.toS3TO(); + } + + } + + // NOTE: Excluded listAll / shuffle operation implemented in SwiftDaoImpl ... + + return null; + + } +} diff --git a/server/src/com/cloud/storage/dao/VMTemplateDao.java b/server/src/com/cloud/storage/dao/VMTemplateDao.java index f5b6913df8a..1284ba103cf 100755 --- a/server/src/com/cloud/storage/dao/VMTemplateDao.java +++ b/server/src/com/cloud/storage/dao/VMTemplateDao.java @@ -71,4 +71,6 @@ public interface VMTemplateDao extends GenericDao { List listPrivateTemplatesByHost(Long hostId); public Long countTemplatesForAccount(long accountId); + List findTemplatesToSyncToS3(); + } diff --git a/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java b/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java index 2a0dfc85559..5c71f1bdb0c 100755 --- a/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -92,6 +92,14 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem private final String SELECT_TEMPLATE_SWIFT_REF = "SELECT t.id, t.unique_name, t.name, t.public, t.featured, t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, " + "t.checksum, t.display_text, t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type FROM vm_template t"; + + private static final String SELECT_S3_CANDIDATE_TEMPLATES = "SELECT t.id, t.unique_name, t.name, t.public, t.featured, " + + "t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, t.checksum, t.display_text, " + + "t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type " + + "FROM vm_template t JOIN template_host_ref r ON t.id=r.template_id JOIN host h ON h.id=r.host_id " + + "WHERE t.hypervisor_type IN (SELECT hypervisor_type FROM host) AND r.download_state = 'DOWNLOADED' AND " + + "r.template_id NOT IN (SELECT template_id FROM template_s3_ref) AND r.destroyed = 0 AND t.type <> 'PERHOST'"; + protected SearchBuilder TemplateNameSearch; protected SearchBuilder UniqueNameSearch; protected SearchBuilder tmpltTypeSearch; @@ -917,5 +925,10 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem (accountType == Account.ACCOUNT_TYPE_DOMAIN_ADMIN) || (accountType == Account.ACCOUNT_TYPE_READ_ONLY_ADMIN)); } - + + @Override + public List findTemplatesToSyncToS3() { + return executeList(SELECT_S3_CANDIDATE_TEMPLATES, new Object[] {}); + } + } diff --git a/server/src/com/cloud/storage/dao/VMTemplateS3Dao.java b/server/src/com/cloud/storage/dao/VMTemplateS3Dao.java new file mode 100644 index 00000000000..d36fb3a2257 --- /dev/null +++ b/server/src/com/cloud/storage/dao/VMTemplateS3Dao.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.dao; + +import com.cloud.storage.VMTemplateS3VO; +import com.cloud.utils.db.GenericDao; + +import java.util.List; + +public interface VMTemplateS3Dao extends GenericDao { + + List listByS3Id(long id); + + VMTemplateS3VO findOneByTemplateId(long id); + + VMTemplateS3VO findOneByS3Template(long s3Id, long templateId); + + void expungeAllByTemplateId(long templateId); + +} diff --git a/server/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java b/server/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java new file mode 100644 index 00000000000..f23b803bf68 --- /dev/null +++ b/server/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.dao; + +import static com.cloud.utils.db.SearchCriteria.Op.*; +import static com.cloud.storage.VMTemplateS3VO.*; + +import com.cloud.storage.VMTemplateS3VO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +import javax.ejb.Local; +import java.util.List; + +@Local(VMTemplateS3Dao.class) +public class VMTemplateS3DaoImpl extends GenericDaoBase + implements VMTemplateS3Dao { + + private final SearchBuilder searchBuilder; + + public VMTemplateS3DaoImpl() { + + super(); + + this.searchBuilder = createSearchBuilder(); + this.searchBuilder + .and(S3_ID_COLUMN_NAME, this.searchBuilder.entity().getS3Id(), + EQ) + .and(TEMPLATE_ID_COLUMN_NAME, + this.searchBuilder.entity().getTemplateId(), EQ).done(); + + } + + @Override + public List listByS3Id(final long s3id) { + + final SearchCriteria criteria = this.searchBuilder + .create(); + + criteria.setParameters(S3_ID_COLUMN_NAME, s3id); + + return this.listBy(criteria); + + } + + @Override + public VMTemplateS3VO findOneByTemplateId(final long templateId) { + + final SearchCriteria criteria = this.searchBuilder + .create(); + + criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); + + return this.findOneBy(criteria); + + } + + @Override + public VMTemplateS3VO findOneByS3Template(final long s3Id, + final long templateId) { + + final SearchCriteria criteria = this.searchBuilder + .create(); + + criteria.setParameters(S3_ID_COLUMN_NAME, s3Id); + criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); + + return this.findOneBy(criteria); + + } + + @Override + public void expungeAllByTemplateId(long templateId) { + + final SearchCriteria criteria = this.searchBuilder + .create(); + + criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); + + this.expunge(criteria); + + } + +} diff --git a/server/src/com/cloud/storage/s3/S3Manager.java b/server/src/com/cloud/storage/s3/S3Manager.java new file mode 100644 index 00000000000..357f2aed463 --- /dev/null +++ b/server/src/com/cloud/storage/s3/S3Manager.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.s3; + +import java.util.List; + +import com.cloud.agent.api.to.S3TO; +import com.cloud.api.commands.AddS3Cmd; +import com.cloud.api.commands.ListS3sCmd; +import com.cloud.dc.DataCenterVO; +import com.cloud.exception.DiscoveryException; +import com.cloud.storage.S3; +import com.cloud.storage.S3VO; +import com.cloud.storage.VMTemplateS3VO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.utils.component.Manager; + +public interface S3Manager extends Manager { + + S3TO getS3TO(); + + S3TO getS3TO(Long s3Id); + + S3 addS3(AddS3Cmd addS3Cmd) throws DiscoveryException; + + Long chooseZoneForTemplateExtract(VMTemplateVO template); + + boolean isS3Enabled(); + + boolean isTemplateInstalled(Long templateId); + + void deleteTemplate(final Long accountId, final Long templateId); + + String downloadTemplateFromS3ToSecondaryStorage(final long dcId, + final long templateId, final int primaryStorageDownloadWait); + + List listS3s(ListS3sCmd listS3sCmd); + + VMTemplateS3VO findByTemplateId(Long templateId); + + void propagateTemplatesToZone(DataCenterVO zone); + + void propagateTemplateToAllZones(VMTemplateS3VO vmTemplateS3VO); + + void uploadTemplateToS3FromSecondaryStorage(final VMTemplateVO template); + +} diff --git a/server/src/com/cloud/storage/s3/S3ManagerImpl.java b/server/src/com/cloud/storage/s3/S3ManagerImpl.java new file mode 100644 index 00000000000..6b072540c66 --- /dev/null +++ b/server/src/com/cloud/storage/s3/S3ManagerImpl.java @@ -0,0 +1,669 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage.s3; + +import static com.cloud.storage.S3VO.ID_COLUMN_NAME; +import static com.cloud.utils.DateUtil.now; +import static com.cloud.utils.S3Utils.canConnect; +import static com.cloud.utils.S3Utils.canReadWriteBucket; +import static com.cloud.utils.S3Utils.checkBucketName; +import static com.cloud.utils.S3Utils.checkClientOptions; +import static com.cloud.utils.S3Utils.doesBucketExist; +import static com.cloud.utils.StringUtils.join; +import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock; +import static com.cloud.utils.db.SearchCriteria.Op.EQ; +import static java.lang.Boolean.TRUE; +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.shuffle; +import static java.util.Collections.singletonList; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; + +import javax.ejb.Local; +import javax.naming.ConfigurationException; + +import org.apache.log4j.Logger; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.DeleteTemplateFromS3Command; +import com.cloud.agent.api.DownloadTemplateFromS3ToSecondaryStorageCommand; +import com.cloud.agent.api.UploadTemplateToS3FromSecondaryStorageCommand; +import com.cloud.agent.api.to.S3TO; +import com.cloud.api.commands.AddS3Cmd; +import com.cloud.api.commands.ListS3sCmd; +import com.cloud.configuration.Config; +import com.cloud.configuration.dao.ConfigurationDao; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.exception.DiscoveryException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.storage.S3; +import com.cloud.storage.S3VO; +import com.cloud.storage.VMTemplateHostVO; +import com.cloud.storage.VMTemplateS3VO; +import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.dao.S3Dao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplateHostDao; +import com.cloud.storage.dao.VMTemplateS3Dao; +import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.storage.secondary.SecondaryStorageVmManager; +import com.cloud.utils.S3Utils.ClientOptions; +import com.cloud.utils.component.Inject; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.exception.CloudRuntimeException; + +@Local(value = { S3Manager.class }) +public class S3ManagerImpl implements S3Manager { + + private static final Logger LOGGER = Logger.getLogger(S3ManagerImpl.class); + + private String name; + + @Inject + private AgentManager agentManager; + + @Inject + private S3Dao s3Dao; + + @Inject + private VMTemplateZoneDao vmTemplateZoneDao; + + @Inject + private VMTemplateS3Dao vmTemplateS3Dao; + + @Inject + private VMTemplateHostDao vmTemplateHostDao; + + @Inject + private VMTemplateDao vmTemplateDao; + + @Inject + private ConfigurationDao configurationDao; + + @Inject + private DataCenterDao dataCenterDao; + + @Inject + private HostDao hostDao; + + @Inject + private SecondaryStorageVmManager secondaryStorageVMManager; + + protected S3ManagerImpl() { + super(); + } + + private void verifyConnection(final S3TO s3) throws DiscoveryException { + + if (!canConnect(s3)) { + throw new DiscoveryException(format("Unable to connect to S3 " + + "using access key %1$s, secret key %2$s, and endpoint, " + + "%3$S", s3.getAccessKey(), s3.getSecretKey(), + s3.getEndPoint() != null ? s3.getEndPoint() : "default")); + } + + } + + private void verifyBuckets(S3TO s3) throws DiscoveryException { + + final List errorMessages = new ArrayList(); + + errorMessages.addAll(verifyBucket(s3, s3.getBucketName())); + + throwDiscoveryExceptionFromErrorMessages(errorMessages); + + } + + private List verifyBucket(final ClientOptions clientOptions, + final String bucketName) { + + if (!doesBucketExist(clientOptions, bucketName)) { + return singletonList(format("Bucket %1$s does not exist.", + bucketName)); + } + + if (!canReadWriteBucket(clientOptions, bucketName)) { + return singletonList(format("Can read/write from bucket %1$s.", + bucketName)); + } + + return emptyList(); + } + + private void validateFields(final S3VO s3VO) { + + final List errorMessages = new ArrayList(); + + errorMessages.addAll(checkClientOptions(s3VO.toS3TO())); + + errorMessages.addAll(checkBucketName("template", s3VO.getBucketName())); + + throwDiscoveryExceptionFromErrorMessages(errorMessages); + + } + + private void enforceS3PreConditions() throws DiscoveryException { + + if (!this.isS3Enabled()) { + throw new DiscoveryException("S3 is not enabled."); + } + + if (this.getS3TO() != null) { + throw new DiscoveryException("Attempt to define multiple S3 " + + "instances. Only one instance definition is supported."); + } + + } + + private void throwDiscoveryExceptionFromErrorMessages( + final List errorMessages) { + + if (!errorMessages.isEmpty()) { + throw new CloudRuntimeException(join(errorMessages, " ")); + } + + } + + @SuppressWarnings("unchecked") + private String determineLockId(final long accountId, final long templateId) { + + // TBD The lock scope may be too coarse grained. Deletes need to lock + // the template across all zones where upload and download could + // probably safely scoped to the zone ... + return join(asList("S3_TEMPLATE", accountId, templateId), "_"); + + } + + @Override + public S3TO getS3TO(final Long s3Id) { + return this.s3Dao.getS3TO(s3Id); + } + + @Override + public S3TO getS3TO() { + + final List s3s = this.s3Dao.listAll(); + + if (s3s == null || (s3s != null && s3s.isEmpty())) { + return null; + } + + if (s3s.size() == 1) { + return s3s.get(0).toS3TO(); + } + + throw new CloudRuntimeException("Multiple S3 instances have been " + + "defined. Only one instance configuration is supported."); + + } + + @Override + public S3 addS3(final AddS3Cmd addS3Cmd) throws DiscoveryException { + + this.enforceS3PreConditions(); + + final S3VO s3VO = new S3VO(UUID.randomUUID().toString(), + addS3Cmd.getAccessKey(), addS3Cmd.getSecretKey(), + addS3Cmd.getEndPoint(), addS3Cmd.getBucketName(), + addS3Cmd.getHttpsFlag(), addS3Cmd.getConnectionTimeout(), + addS3Cmd.getMaxErrorRetry(), addS3Cmd.getSocketTimeout(), now()); + + this.validateFields(s3VO); + + final S3TO s3 = s3VO.toS3TO(); + this.verifyConnection(s3); + this.verifyBuckets(s3); + + return this.s3Dao.persist(s3VO); + + } + + @Override + public boolean isS3Enabled() { + return Boolean + .valueOf(configurationDao.getValue(Config.S3Enable.key())); + } + + @Override + public boolean isTemplateInstalled(final Long templateId) { + throw new UnsupportedOperationException( + "S3Manager#isTemplateInstalled (DeleteIsoCmd) has not yet " + + "been implemented"); + } + + @Override + public void deleteTemplate(final Long templateId, final Long accountId) { + + final S3TO s3 = getS3TO(); + + if (s3 == null) { + final String errorMessage = "Delete Template Failed: No S3 configuration defined."; + LOGGER.error(errorMessage); + throw new CloudRuntimeException(errorMessage); + } + + final VMTemplateS3VO vmTemplateS3VO = vmTemplateS3Dao + .findOneByS3Template(s3.getId(), templateId); + if (vmTemplateS3VO == null) { + final String errorMessage = format( + "Delete Template Failed: Unable to find Template %1$s in S3.", + templateId); + LOGGER.error(errorMessage); + throw new CloudRuntimeException(errorMessage); + } + + try { + + executeWithNoWaitLock(determineLockId(accountId, templateId), + new Callable() { + + @Override + public Void call() throws Exception { + + final Answer answer = agentManager.sendToSSVM(null, + new DeleteTemplateFromS3Command(s3, + accountId, templateId)); + if (answer == null || !answer.getResult()) { + final String errorMessage = format( + "Delete Template Failed: Unable to delete template id %1$s from S3 due to following error: %2$s", + templateId, + ((answer == null) ? "answer is null" + : answer.getDetails())); + LOGGER.error(errorMessage); + throw new CloudRuntimeException(errorMessage); + } + + vmTemplateS3Dao.remove(vmTemplateS3VO.getId()); + LOGGER.debug(format( + "Deleted template %1$s from S3.", + templateId)); + + return null; + + } + + }); + + } catch (Exception e) { + + final String errorMessage = format( + "Delete Template Failed: Unable to delete template id %1$s from S3 due to the following error: %2$s.", + templateId, e.getMessage()); + LOGGER.error(errorMessage); + throw new CloudRuntimeException(errorMessage, e); + + } + + } + + @SuppressWarnings("unchecked") + @Override + public String downloadTemplateFromS3ToSecondaryStorage( + final long dataCenterId, final long templateId, + final int primaryStorageDownloadWait) { + + if (!isS3Enabled()) { + return null; + } + + final VMTemplateVO template = vmTemplateDao.findById(templateId); + if (template == null) { + final String errorMessage = String + .format("Failed to download template id %1$s from S3 because the template definition was not found.", + templateId); + LOGGER.error(errorMessage); + return errorMessage; + } + + final VMTemplateS3VO templateS3VO = findByTemplateId(templateId); + if (templateS3VO == null) { + final String errorMessage = format( + "Failed to download template id %1$s from S3 because it does not exist in S3.", + templateId); + LOGGER.error(errorMessage); + return errorMessage; + } + + final S3TO s3 = getS3TO(templateS3VO.getS3Id()); + if (s3 == null) { + final String errorMessage = format( + "Failed to download template id %1$s from S3 because S3 id %2$s does not exist.", + templateId, templateS3VO); + LOGGER.error(errorMessage); + return errorMessage; + } + + final HostVO secondaryStorageHost = secondaryStorageVMManager + .findSecondaryStorageHost(dataCenterId); + if (secondaryStorageHost == null) { + final String errorMessage = format( + "Unable to find secondary storage host for zone id %1$s.", + dataCenterId); + LOGGER.error(errorMessage); + throw new CloudRuntimeException(errorMessage); + } + + final long accountId = template.getAccountId(); + final DownloadTemplateFromS3ToSecondaryStorageCommand cmd = new DownloadTemplateFromS3ToSecondaryStorageCommand( + s3, accountId, templateId, secondaryStorageHost.getName(), + primaryStorageDownloadWait); + + try { + + executeWithNoWaitLock(determineLockId(accountId, templateId), + new Callable() { + + @Override + public Void call() throws Exception { + + final Answer answer = agentManager.sendToSSVM( + dataCenterId, cmd); + + if (answer == null || !answer.getResult()) { + final String errMsg = String + .format("Failed to download template from S3 to secondary storage due to %1$s", + (answer == null ? "answer is null" + : answer.getDetails())); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + + final String installPath = join( + asList("template", "tmpl", accountId, + templateId), File.separator); + final VMTemplateHostVO tmpltHost = new VMTemplateHostVO( + secondaryStorageHost.getId(), templateId, + now(), 100, Status.DOWNLOADED, null, null, + null, installPath, template.getUrl()); + tmpltHost.setSize(templateS3VO.getSize()); + tmpltHost.setPhysicalSize(templateS3VO + .getPhysicalSize()); + vmTemplateHostDao.persist(tmpltHost); + + return null; + + } + + }); + + } catch (Exception e) { + final String errMsg = "Failed to download template from S3 to secondary storage due to " + + e.toString(); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + + return null; + + } + + @Override + public List listS3s(final ListS3sCmd cmd) { + + final Filter filter = new Filter(S3VO.class, ID_COLUMN_NAME, TRUE, + cmd.getStartIndex(), cmd.getPageSizeVal()); + final SearchCriteria criteria = this.s3Dao.createSearchCriteria(); + + if (cmd.getId() != null) { + criteria.addAnd(ID_COLUMN_NAME, EQ, cmd.getId()); + } + + return this.s3Dao.search(criteria, filter); + + } + + @Override + public VMTemplateS3VO findByTemplateId(final Long templateId) { + throw new UnsupportedOperationException( + "S3Manager#findByTemplateId(Long) has not yet " + + "been implemented"); + } + + @Override + public void propagateTemplatesToZone(final DataCenterVO zone) { + + if (!isS3Enabled()) { + return; + } + + final List s3VMTemplateRefs = this.vmTemplateS3Dao + .listAll(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(format("Propagating %1$s templates to zone %2$s.", + s3VMTemplateRefs.size(), zone.getName())); + } + + for (final VMTemplateS3VO templateS3VO : s3VMTemplateRefs) { + this.vmTemplateZoneDao.persist(new VMTemplateZoneVO(zone.getId(), + templateS3VO.getTemplateId(), now())); + } + + } + + @Override + public boolean configure(final String name, final Map params) + throws ConfigurationException { + + if (LOGGER.isInfoEnabled()) { + LOGGER.info(format("Configuring S3 Manager %1$s", name)); + } + + this.name = name; + + return true; + + } + + @Override + public boolean start() { + LOGGER.info("Starting S3 Manager"); + return true; + } + + @Override + public boolean stop() { + LOGGER.info("Stopping S3 Manager"); + return true; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public void propagateTemplateToAllZones(final VMTemplateS3VO vmTemplateS3VO) { + + final long templateId = vmTemplateS3VO.getId(); + + if (!isS3Enabled()) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(format( + "Attempt to propogate template id %1$s across all zones. However, S3 is not enabled.", + templateId)); + } + return; + + } + + final S3TO s3 = getS3TO(); + + if (s3 == null) { + LOGGER.warn(format( + "Unable to propagate template id %1$s across all zones because S3 is enabled, but not configured.", + templateId)); + return; + } + + if (vmTemplateS3VO != null) { + final List dataCenters = dataCenterDao.listAll(); + for (DataCenterVO dataCenter : dataCenters) { + final VMTemplateZoneVO tmpltZoneVO = new VMTemplateZoneVO( + dataCenter.getId(), templateId, now()); + vmTemplateZoneDao.persist(tmpltZoneVO); + } + } + + } + + @Override + public Long chooseZoneForTemplateExtract(VMTemplateVO template) { + + final S3TO s3 = getS3TO(); + + if (s3 == null) { + return null; + } + + final List templateHosts = vmTemplateHostDao + .listByOnlyTemplateId(template.getId()); + if (templateHosts != null) { + shuffle(templateHosts); + for (VMTemplateHostVO vmTemplateHostVO : templateHosts) { + final HostVO host = hostDao.findById(vmTemplateHostVO + .getHostId()); + if (host != null) { + return host.getDataCenterId(); + } + throw new CloudRuntimeException( + format("Unable to find secondary storage host for template id %1$s.", + template.getId())); + } + } + + final List dataCenters = dataCenterDao.listAll(); + shuffle(dataCenters); + return dataCenters.get(0).getId(); + + } + + @Override + public void uploadTemplateToS3FromSecondaryStorage( + final VMTemplateVO template) { + + final Long templateId = template.getId(); + + final List templateHostRefs = vmTemplateHostDao + .listByTemplateId(templateId); + + if (templateHostRefs == null + || (templateHostRefs != null && templateHostRefs.isEmpty())) { + throw new CloudRuntimeException( + format("Attempt to sync template id %1$s that is not attached to a host.", + templateId)); + } + + final VMTemplateHostVO templateHostRef = templateHostRefs.get(0); + + if (!isS3Enabled()) { + return; + } + + final S3TO s3 = getS3TO(); + if (s3 == null) { + LOGGER.warn("S3 Template Sync Failed: Attempt to sync templates with S3, but no S3 instance defined."); + return; + } + + final HostVO secondaryHost = this.hostDao.findById(templateHostRef + .getHostId()); + if (secondaryHost == null) { + throw new CloudRuntimeException(format( + "Unable to find secondary storage host id %1$s.", + templateHostRef.getHostId())); + } + + final Long dataCenterId = secondaryHost.getDataCenterId(); + final Long accountId = template.getAccountId(); + + try { + + executeWithNoWaitLock(determineLockId(accountId, templateId), + new Callable() { + + @Override + public Void call() throws Exception { + + final UploadTemplateToS3FromSecondaryStorageCommand cmd = new UploadTemplateToS3FromSecondaryStorageCommand( + s3, secondaryHost.getStorageUrl(), + dataCenterId, accountId, templateId); + + final Answer answer = agentManager.sendToSSVM( + dataCenterId, cmd); + if (answer == null || !answer.getResult()) { + + final String reason = answer != null ? answer + .getDetails() + : "S3 template sync failed due to an unspecified error."; + throw new CloudRuntimeException( + format("Failed to upload template id %1$s to S3 from secondary storage due to %2$s.", + templateId, reason)); + + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(format( + "Creating VMTemplateS3VO instance using template id %1s.", + templateId)); + } + + final VMTemplateS3VO vmTemplateS3VO = new VMTemplateS3VO( + s3.getId(), templateId, now(), + templateHostRef.getSize(), templateHostRef + .getPhysicalSize()); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(format("Persisting %1$s", + vmTemplateS3VO)); + } + + vmTemplateS3Dao.persist(vmTemplateS3VO); + propagateTemplateToAllZones(vmTemplateS3VO); + + return null; + + } + + }); + + } catch (Exception e) { + + final String errorMessage = format( + "Failed to upload template id %1$s for zone id %2$s to S3.", + templateId, dataCenterId); + LOGGER.error(errorMessage, e); + + } + + } + +} diff --git a/server/src/com/cloud/storage/snapshot/SnapshotManager.java b/server/src/com/cloud/storage/snapshot/SnapshotManager.java index a10298efb90..a7692de7107 100755 --- a/server/src/com/cloud/storage/snapshot/SnapshotManager.java +++ b/server/src/com/cloud/storage/snapshot/SnapshotManager.java @@ -128,6 +128,8 @@ public interface SnapshotManager { void downloadSnapshotsFromSwift(SnapshotVO ss); + void downloadSnapshotsFromS3(SnapshotVO snapshot); + HostVO getSecondaryStorageHost(SnapshotVO snapshot); String getSecondaryStorageURL(SnapshotVO snapshot); diff --git a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 32e37e63c0d..7ab464b2524 100755 --- a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.storage.snapshot; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; @@ -34,9 +35,11 @@ import com.cloud.agent.api.BackupSnapshotCommand; import com.cloud.agent.api.Command; import com.cloud.agent.api.DeleteSnapshotBackupCommand; import com.cloud.agent.api.DeleteSnapshotsDirCommand; +import com.cloud.agent.api.DownloadSnapshotFromS3Command; import com.cloud.agent.api.ManageSnapshotAnswer; import com.cloud.agent.api.ManageSnapshotCommand; import com.cloud.agent.api.downloadSnapshotFromSwiftCommand; +import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.alert.AlertManager; import com.cloud.api.commands.CreateSnapshotPolicyCmd; @@ -91,6 +94,7 @@ import com.cloud.storage.dao.SnapshotScheduleDao; import com.cloud.storage.dao.StoragePoolDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.s3.S3Manager; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.swift.SwiftManager; import com.cloud.tags.ResourceTagVO; @@ -171,6 +175,8 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma private ResourceLimitService _resourceLimitMgr; @Inject private SwiftManager _swiftMgr; + @Inject + private S3Manager _s3Mgr; @Inject private SecondaryStorageVmManager _ssvmMgr; @Inject @@ -477,11 +483,25 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma return createdSnapshot; } + private static void checkObjectStorageConfiguration(SwiftTO swift, S3TO s3) { + + if (swift != null && s3 != null) { + throw new CloudRuntimeException( + "Swift and S3 are not simultaneously supported for snapshot backup."); + } + + } @Override public void deleteSnapshotsForVolume (String secondaryStoragePoolUrl, Long dcId, Long accountId, Long volumeId ){ SwiftTO swift = _swiftMgr.getSwiftTO(); - DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand(swift, secondaryStoragePoolUrl, dcId, accountId, volumeId, null, true); + S3TO s3 = _s3Mgr.getS3TO(); + + checkObjectStorageConfiguration(swift, s3); + + DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand( + swift, s3, secondaryStoragePoolUrl, dcId, accountId, volumeId, + null, true); try { Answer ans = _agentMgr.sendToSSVM(dcId, cmd); if ( ans == null || !ans.getResult() ) { @@ -543,6 +563,54 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma } + private List determineBackupUuids(final SnapshotVO snapshot) { + + final List backupUuids = new ArrayList(); + backupUuids.add(0, snapshot.getBackupSnapshotId()); + + SnapshotVO tempSnapshot = snapshot; + while (tempSnapshot.getPrevSnapshotId() != 0) { + tempSnapshot = _snapshotDao.findById(tempSnapshot + .getPrevSnapshotId()); + backupUuids.add(0, tempSnapshot.getBackupSnapshotId()); + } + + return Collections.unmodifiableList(backupUuids); + } + + @Override + public void downloadSnapshotsFromS3(final SnapshotVO snapshot) { + + final VolumeVO volume = _volsDao.findById(snapshot.getVolumeId()); + final Long zoneId = volume.getDataCenterId(); + final HostVO secHost = _storageMgr.getSecondaryStorageHost(zoneId); + + final S3TO s3 = _s3Mgr.getS3TO(snapshot.getS3Id()); + final List backupUuids = determineBackupUuids(snapshot); + + try { + String parent = null; + for (final String backupUuid : backupUuids) { + final DownloadSnapshotFromS3Command cmd = new DownloadSnapshotFromS3Command( + s3, parent, secHost.getStorageUrl(), zoneId, + volume.getAccountId(), volume.getId(), backupUuid, + _backupsnapshotwait); + final Answer answer = _agentMgr.sendToSSVM(zoneId, cmd); + if ((answer == null) || !answer.getResult()) { + throw new CloudRuntimeException(String.format( + "S3 snapshot download failed due to %1$s.", + answer != null ? answer.getDetails() + : "unspecified error")); + } + parent = backupUuid; + } + } catch (Exception e) { + throw new CloudRuntimeException( + "Snapshot download from S3 failed due to " + e.toString(), + e); + } + + } @Override @DB @@ -560,7 +628,6 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma long volumeId = snapshot.getVolumeId(); VolumeVO volume = _volsDao.lockRow(volumeId, true); - String primaryStoragePoolNameLabel = _storageMgr.getPrimaryStorageNameLabel(volume); Long dcId = volume.getDataCenterId(); Long accountId = volume.getAccountId(); @@ -577,6 +644,9 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma SwiftTO swift = _swiftMgr.getSwiftTO(); + S3TO s3 = _s3Mgr.getS3TO(); + + checkObjectStorageConfiguration(swift, s3); long prevSnapshotId = snapshot.getPrevSnapshotId(); if (prevSnapshotId > 0) { @@ -586,7 +656,8 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma prevBackupUuid = prevSnapshot.getBackupSnapshotId(); prevSnapshotUuid = prevSnapshot.getPath(); } - } else if ( prevSnapshot.getSwiftId() != null && swift != null ) { + } else if ((prevSnapshot.getSwiftId() != null && swift != null) + || (prevSnapshot.getS3Id() != null && s3 != null)) { prevBackupUuid = prevSnapshot.getBackupSnapshotId(); prevSnapshotUuid = prevSnapshot.getPath(); } @@ -594,13 +665,15 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma boolean isVolumeInactive = _storageMgr.volumeInactive(volume); String vmName = _storageMgr.getVmNameOnVolume(volume); StoragePoolVO srcPool = _storagePoolDao.findById(volume.getPoolId()); - BackupSnapshotCommand backupSnapshotCommand = new BackupSnapshotCommand(primaryStoragePoolNameLabel, secondaryStoragePoolUrl, dcId, accountId, volumeId, snapshot.getId(), volume.getPath(), srcPool, snapshotUuid, + BackupSnapshotCommand backupSnapshotCommand = new BackupSnapshotCommand(secondaryStoragePoolUrl, dcId, accountId, volumeId, snapshot.getId(), volume.getPath(), srcPool, snapshotUuid, snapshot.getName(), prevSnapshotUuid, prevBackupUuid, isVolumeInactive, vmName, _backupsnapshotwait); if ( swift != null ) { backupSnapshotCommand.setSwift(swift); + } else if (s3 != null) { + backupSnapshotCommand.setS3(s3); } - + String backedUpSnapshotUuid = null; // By default, assume failed. boolean backedUp = false; @@ -621,6 +694,9 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma if (backupSnapshotCommand.getSwift() != null ) { snapshot.setSwiftId(swift.getId()); snapshot.setBackupSnapshotId(backedUpSnapshotUuid); + } else if (backupSnapshotCommand.getS3() != null) { + snapshot.setS3Id(s3.getId()); + snapshot.setBackupSnapshotId(backedUpSnapshotUuid); } else { snapshot.setSecHostId(secHost.getId()); snapshot.setBackupSnapshotId(backedUpSnapshotUuid); @@ -832,7 +908,13 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma return true; } SwiftTO swift = _swiftMgr.getSwiftTO(snapshot.getSwiftId()); - DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand(swift, secondaryStoragePoolUrl, dcId, accountId, volumeId, backupOfSnapshot, false); + S3TO s3 = _s3Mgr.getS3TO(); + + checkObjectStorageConfiguration(swift, s3); + + DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand( + swift, s3, secondaryStoragePoolUrl, dcId, accountId, volumeId, + backupOfSnapshot, false); Answer answer = _agentMgr.sendToSSVM(dcId, cmd); if ((answer != null) && answer.getResult()) { @@ -979,9 +1061,15 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma } List ssHosts = _ssvmMgr.listSecondaryStorageHostsInOneZone(dcId); SwiftTO swift = _swiftMgr.getSwiftTO(); - if (swift == null) { + S3TO s3 = _s3Mgr.getS3TO(); + + checkObjectStorageConfiguration(swift, s3); + + if (swift == null && s3 == null) { for (HostVO ssHost : ssHosts) { - DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand(null, ssHost.getStorageUrl(), dcId, accountId, volumeId, "", true); + DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand( + null, null, ssHost.getStorageUrl(), dcId, + accountId, volumeId, "", true); Answer answer = null; try { answer = _agentMgr.sendToSSVM(dcId, cmd); @@ -998,12 +1086,14 @@ public class SnapshotManagerImpl implements SnapshotManager, SnapshotService, Ma } } } else { - DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand(swift, "", dcId, accountId, volumeId, "", true); + DeleteSnapshotBackupCommand cmd = new DeleteSnapshotBackupCommand( + swift, s3, "", dcId, accountId, volumeId, "", true); Answer answer = null; try { answer = _agentMgr.sendToSSVM(dcId, cmd); } catch (Exception e) { - s_logger.warn("Failed to delete all snapshot for volume " + volumeId + " on swift"); + final String storeType = s3 != null ? "S3" : "swift"; + s_logger.warn("Failed to delete all snapshot for volume " + volumeId + " on " + storeType); } if ((answer != null) && answer.getResult()) { s_logger.debug("Deleted all snapshots for volume: " + volumeId + " under account: " + accountId); diff --git a/server/src/com/cloud/template/S3SyncTask.java b/server/src/com/cloud/template/S3SyncTask.java new file mode 100644 index 00000000000..ed179dc8961 --- /dev/null +++ b/server/src/com/cloud/template/S3SyncTask.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.template; + +import static java.lang.String.*; + +import java.util.List; + +import org.apache.log4j.Logger; + +import com.cloud.agent.api.to.S3TO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.s3.S3Manager; + +final class S3SyncTask implements Runnable { + + private static final Logger LOGGER = Logger.getLogger(S3SyncTask.class); + + private final VMTemplateDao vmTemplateDao; + private final S3Manager s3Mgr; + + S3SyncTask(final VMTemplateDao vmTemplateDao, final S3Manager s3Mgr) { + + super(); + + assert vmTemplateDao != null; + assert s3Mgr != null; + + this.vmTemplateDao = vmTemplateDao; + this.s3Mgr = s3Mgr; + + } + + @Override + public void run() { + + try { + + final S3TO s3 = s3Mgr.getS3TO(); + + if (s3 == null) { + LOGGER.warn("S3 sync skipped because no S3 instance is configured."); + return; + } + + final List candidateTemplates = vmTemplateDao + .findTemplatesToSyncToS3(); + + if (candidateTemplates.isEmpty()) { + LOGGER.debug("All templates are synced with S3."); + return; + } + + for (VMTemplateVO candidateTemplate : candidateTemplates) { + + if (LOGGER.isInfoEnabled()) { + LOGGER.info(format( + "Uploading template %1$s (id: %2$s) to S3.", + candidateTemplate.getName(), + candidateTemplate.getId())); + } + + s3Mgr.uploadTemplateToS3FromSecondaryStorage(candidateTemplate); + + } + + LOGGER.debug("Completed S3 template sync task."); + + } catch (Exception e) { + LOGGER.warn( + "S3 Sync Task ignored exception, and will continue to execute.", + e); + } + + } + +} diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 1e87de2fecb..805be3bf675 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -113,10 +113,12 @@ import com.cloud.storage.dao.UploadDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VMTemplateS3Dao; import com.cloud.storage.dao.VMTemplateSwiftDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.download.DownloadMonitor; +import com.cloud.storage.s3.S3Manager; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.swift.SwiftManager; import com.cloud.storage.upload.UploadMonitor; @@ -179,8 +181,12 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe @Inject SwiftManager _swiftMgr; @Inject + S3Manager _s3Mgr; + @Inject VMTemplateSwiftDao _tmpltSwiftDao; @Inject + VMTemplateS3Dao _vmS3TemplateDao; + @Inject ConfigurationDao _configDao; @Inject ClusterDao _clusterDao; @@ -207,6 +213,7 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe ExecutorService _preloadExecutor; ScheduledExecutorService _swiftTemplateSyncExecutor; + private ScheduledExecutorService _s3TemplateSyncExecutor = null; @Inject (adapter=TemplateAdapter.class) protected Adapters _adapters; @@ -344,10 +351,14 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe } } - if (zoneId == null) { + if (zoneId == null && _swiftMgr.isSwiftEnabled()) { zoneId = _swiftMgr.chooseZoneForTmpltExtract(templateId); } + if (zoneId == null && _s3Mgr.isS3Enabled()) { + zoneId = _s3Mgr.chooseZoneForTemplateExtract(template); + } + if (_dcDao.findById(zoneId) == null) { throw new IllegalArgumentException("Please specify a valid zone."); } @@ -380,7 +391,13 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe if (swift != null && sservers != null) { downloadTemplateFromSwiftToSecondaryStorage(zoneId, templateId); } + } else if (tmpltHostRef == null && _s3Mgr.isS3Enabled()) { + if (sservers != null) { + _s3Mgr.downloadTemplateFromS3ToSecondaryStorage(zoneId, + templateId, _primaryStorageDownloadWait); + } } + if (tmpltHostRef == null) { throw new InvalidParameterValueException("The " + desc + " has not been downloaded "); } @@ -594,6 +611,12 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); return null; } + result = _s3Mgr.downloadTemplateFromS3ToSecondaryStorage(dcId, + templateId, _primaryStorageDownloadWait); + if (result != null) { + s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); + return null; + } templateHostRef = _storageMgr.findVmTemplateHost(templateId, pool); if (templateHostRef == null || templateHostRef.getDownloadState() != Status.DOWNLOADED) { s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); @@ -644,7 +667,7 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe } String url = origUrl + "/" + templateHostRef.getInstallPath(); PrimaryStorageDownloadCommand dcmd = new PrimaryStorageDownloadCommand(template.getUniqueName(), url, template.getFormat(), - template.getAccountId(), pool.getId(), pool.getUuid(), _primaryStorageDownloadWait); + template.getAccountId(), pool, _primaryStorageDownloadWait); HostVO secondaryStorageHost = _hostDao.findById(templateHostRef.getHostId()); assert(secondaryStorageHost != null); dcmd.setSecondaryStorageUrl(secondaryStorageHost.getStorageUrl()); @@ -708,6 +731,12 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); return null; } + result = _s3Mgr.downloadTemplateFromS3ToSecondaryStorage(dcId, + templateId, _primaryStorageDownloadWait); + if (result != null) { + s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); + return null; + } templateHostRef = _storageMgr.findVmTemplateHost(templateId, pool); if (templateHostRef == null || templateHostRef.getDownloadState() != Status.DOWNLOADED) { s_logger.error("Unable to find a secondary storage host who has completely downloaded the template."); @@ -823,6 +852,12 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe if (_swiftMgr.isSwiftEnabled()) { throw new CloudRuntimeException("copytemplate API is disabled in Swift setup, templates in Swift can be accessed by all Zones"); } + + if (_s3Mgr.isS3Enabled()) { + throw new CloudRuntimeException( + "copytemplate API is disabled in S3 setup -- S3 templates are accessible in all zones."); + } + //Verify parameters if (sourceZoneId == destZoneId) { throw new InvalidParameterValueException("Please specify different source and destination zones."); @@ -1003,12 +1038,32 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe @Override public boolean start() { _swiftTemplateSyncExecutor.scheduleAtFixedRate(getSwiftTemplateSyncTask(), 60, 60, TimeUnit.SECONDS); + + if (_s3TemplateSyncExecutor != null) { + + final int initialDelay = 60; + final int period = 60; + + _s3TemplateSyncExecutor.scheduleAtFixedRate(new S3SyncTask( + this._tmpltDao, this._s3Mgr), initialDelay, period, + TimeUnit.SECONDS); + s_logger.info(String.format("Started S3 sync task to execute " + + "execute every %1$s after an initial delay of %2$s.", + period, initialDelay)); + + } + return true; } @Override public boolean stop() { _swiftTemplateSyncExecutor.shutdownNow(); + + if (_s3TemplateSyncExecutor != null) { + _s3TemplateSyncExecutor.shutdownNow(); + } + return true; } @@ -1041,7 +1096,16 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe _storagePoolMaxWaitSeconds = NumbersUtil.parseInt(_configDao.getValue(Config.StoragePoolMaxWaitSeconds.key()), 3600); _preloadExecutor = Executors.newFixedThreadPool(8, new NamedThreadFactory("Template-Preloader")); _swiftTemplateSyncExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("swift-template-sync-Executor")); - return false; + + if (_s3Mgr.isS3Enabled()) { + _s3TemplateSyncExecutor = Executors + .newSingleThreadScheduledExecutor(new NamedThreadFactory( + "s3-template-sync")); + } else { + s_logger.info("S3 secondary storage synchronization is disabled."); + } + + return false; } protected TemplateManagerImpl() { @@ -1195,13 +1259,19 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe if (cmd.getZoneId() == null && _swiftMgr.isSwiftEnabled()) { _swiftMgr.deleteTemplate(cmd); } + if (cmd.getZoneId() == null && _s3Mgr.isS3Enabled()) { + _s3Mgr.deleteTemplate(cmd.getId(), caller.getAccountId()); + } + TemplateAdapter adapter = getAdapter(template.getHypervisorType()); TemplateProfile profile = adapter.prepareDelete(cmd); boolean result = adapter.delete(profile); if (result){ - if (cmd.getZoneId() == null && _swiftMgr.isSwiftEnabled()) { - List templateZones = _tmpltZoneDao.listByZoneTemplate(null, templateId); + if (cmd.getZoneId() == null + && (_swiftMgr.isSwiftEnabled() || _s3Mgr.isS3Enabled())) { + List templateZones = _tmpltZoneDao + .listByZoneTemplate(null, templateId); if (templateZones != null) { for (VMTemplateZoneVO templateZone : templateZones) { _tmpltZoneDao.remove(templateZone.getId()); @@ -1234,6 +1304,10 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe if (cmd.getZoneId() == null && _swiftMgr.isSwiftEnabled()) { _swiftMgr.deleteIso(cmd); } + if (cmd.getZoneId() == null && _s3Mgr.isS3Enabled()) { + _s3Mgr.deleteTemplate(caller.getAccountId(), templateId); + } + if (zoneId != null && (_ssvmMgr.findSecondaryStorageHost(zoneId) == null)) { throw new InvalidParameterValueException("Failed to find a secondary storage host in the specified zone."); } @@ -1241,8 +1315,10 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe TemplateProfile profile = adapter.prepareDelete(cmd); boolean result = adapter.delete(profile); if (result) { - if (cmd.getZoneId() == null && _swiftMgr.isSwiftEnabled()) { - List templateZones = _tmpltZoneDao.listByZoneTemplate(null, templateId); + if (cmd.getZoneId() == null + && (_swiftMgr.isSwiftEnabled() || _s3Mgr.isS3Enabled())) { + List templateZones = _tmpltZoneDao + .listByZoneTemplate(null, templateId); if (templateZones != null) { for (VMTemplateZoneVO templateZone : templateZones) { _tmpltZoneDao.remove(templateZone.getId()); diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 02d630f31fd..9356048453a 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -1448,7 +1448,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager if( snapshot.getSwiftId() != null && snapshot.getSwiftId() != 0 ) { _snapshotMgr.downloadSnapshotsFromSwift(snapshot); } - cmd = new CreatePrivateTemplateFromSnapshotCommand(pool.getUuid(), secondaryStorageURL, dcId, accountId, snapshot.getVolumeId(), backupSnapshotUUID, snapshot.getName(), + cmd = new CreatePrivateTemplateFromSnapshotCommand(pool, secondaryStorageURL, dcId, accountId, snapshot.getVolumeId(), backupSnapshotUUID, snapshot.getName(), origTemplateInstallPath, templateId, name, _createprivatetemplatefromsnapshotwait); } else if (volumeId != null) { VolumeVO volume = _volsDao.findById(volumeId); @@ -1470,7 +1470,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager secondaryStorageURL = secondaryStorageHost.getStorageUrl(); pool = _storagePoolDao.findById(volume.getPoolId()); - cmd = new CreatePrivateTemplateFromVolumeCommand(pool.getUuid(), secondaryStorageURL, templateId, accountId, command.getTemplateName(), uniqueName, volume.getPath(), vmName, _createprivatetemplatefromvolumewait); + cmd = new CreatePrivateTemplateFromVolumeCommand(pool, secondaryStorageURL, templateId, accountId, command.getTemplateName(), uniqueName, volume.getPath(), vmName, _createprivatetemplatefromvolumewait); } else { throw new CloudRuntimeException("Creating private Template need to specify snapshotId or volumeId"); @@ -3161,10 +3161,6 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager if (zone != null) { sc.setParameters("dataCenterId", zone); - - if (state == null) { - sc.setParameters("stateNEQ", "Destroyed"); - } } if (pod != null) { sc.setParameters("podId", pod); diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index b0457d7ef3e..8010e3b88c5 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -146,6 +146,8 @@ DROP TABLE IF EXISTS `cloud`.`s2s_vpn_gateway`; DROP TABLE IF EXISTS `cloud`.`s2s_vpn_connection`; DROP TABLE IF EXISTS `cloud`,`external_nicira_nvp_devices`; DROP TABLE IF EXISTS `cloud`,`nicira_nvp_nic_map`; +DROP TABLE IF EXISTS `cloud`,`s3`; +DROP TABLE IF EXISTS `cloud`,`template_s3_ref`; DROP TABLE IF EXISTS `cloud`,`nicira_nvp_router_map`; DROP TABLE IF EXISTS `cloud`.`autoscale_vmgroup_policy_map`; DROP TABLE IF EXISTS `cloud`.`autoscale_policy_condition_map`; @@ -164,7 +166,7 @@ CREATE TABLE `cloud`.`version` ( INDEX `i_version__version`(`version`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -INSERT INTO `version` (`version`, `updated`, `step`) VALUES('@VERSION@', now(), 'Complete'); +INSERT INTO `version` (`version`, `updated`, `step`) VALUES('4.0.0.2012-09-12T14:47:37Z', now(), 'Complete'); CREATE TABLE `cloud`.`op_it_work` ( `id` char(40) COMMENT 'reservation id', @@ -480,12 +482,14 @@ CREATE TABLE `cloud`.`snapshots` ( `removed` datetime COMMENT 'Date removed. not null if removed', `backup_snap_id` varchar(255) COMMENT 'Back up uuid of the snapshot', `swift_id` bigint unsigned COMMENT 'which swift', + `s3_id` bigint unsigned COMMENT 'S3 to which this snapshot will be stored', `sechost_id` bigint unsigned COMMENT 'secondary storage host id', `prev_snap_id` bigint unsigned COMMENT 'Id of the most recent snapshot', `hypervisor_type` varchar(32) NOT NULL COMMENT 'hypervisor that the snapshot was taken under', `version` varchar(32) COMMENT 'snapshot version', PRIMARY KEY (`id`), CONSTRAINT `uc_snapshots__uuid` UNIQUE (`uuid`), + CONSTRAINT `fk_snapshots__s3_id` FOREIGN KEY `fk_snapshots__s3_id` (`s3_id`) REFERENCES `s3` (`id`), INDEX `i_snapshots__removed`(`removed`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -1850,6 +1854,36 @@ CREATE TABLE `cloud`.`swift` ( CONSTRAINT `uc_swift__uuid` UNIQUE (`uuid`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; +CREATE TABLE `cloud`.`s3` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `uuid` varchar(40), + `access_key` varchar(20) NOT NULL COMMENT ' The S3 access key', + `secret_key` varchar(40) NOT NULL COMMENT ' The S3 secret key', + `end_point` varchar(1024) COMMENT ' The S3 host', + `bucket` varchar(63) NOT NULL COMMENT ' The S3 host', + `https` tinyint unsigned DEFAULT NULL COMMENT ' Flag indicating whether or not to connect over HTTPS', + `connection_timeout` integer COMMENT ' The amount of time to wait (in milliseconds) when initially establishing a connection before giving up and timing out.', + `max_error_retry` integer COMMENT ' The maximum number of retry attempts for failed retryable requests (ex: 5xx error responses from services).', + `socket_timeout` integer COMMENT ' The amount of time to wait (in milliseconds) for data to be transfered over an established, open connection before the connection times out and is closed.', + `created` datetime COMMENT 'date the s3 first signed on', + PRIMARY KEY (`id`), + CONSTRAINT `uc_s3__uuid` UNIQUE (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `cloud`.`template_s3_ref` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `s3_id` bigint unsigned NOT NULL COMMENT ' Associated S3 instance id', + `template_id` bigint unsigned NOT NULL COMMENT ' Associated template id', + `created` DATETIME NOT NULL COMMENT ' The creation timestamp', + `size` bigint unsigned COMMENT ' The size of the object', + `physical_size` bigint unsigned DEFAULT 0 COMMENT ' The physical size of the object', + PRIMARY KEY (`id`), + CONSTRAINT `uc_template_s3_ref__template_id` UNIQUE (`template_id`), + CONSTRAINT `fk_template_s3_ref__s3_id` FOREIGN KEY `fk_template_s3_ref__s3_id` (`s3_id`) REFERENCES `s3` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_template_s3_ref__template_id` FOREIGN KEY `fk_template_s3_ref__template_id` (`template_id`) REFERENCES `vm_template` (`id`), + INDEX `i_template_s3_ref__s3_id`(`s3_id`), + INDEX `i_template_s3_ref__template_id`(`template_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`op_host_transfer` ( `id` bigint unsigned UNIQUE NOT NULL COMMENT 'Id of the host', diff --git a/setup/db/db/schema-40to410.sql b/setup/db/db/schema-40to410.sql new file mode 100644 index 00000000000..b0301d72538 --- /dev/null +++ b/setup/db/db/schema-40to410.sql @@ -0,0 +1,58 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.0.0 to 4.1.0; +--; + +CREATE TABLE `cloud`.`s3` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `uuid` varchar(40), + `access_key` varchar(20) NOT NULL COMMENT ' The S3 access key', + `secret_key` varchar(40) NOT NULL COMMENT ' The S3 secret key', + `end_point` varchar(1024) COMMENT ' The S3 host', + `bucket` varchar(63) NOT NULL COMMENT ' The S3 host', + `https` tinyint unsigned DEFAULT NULL COMMENT ' Flag indicating whether or not to connect over HTTPS', + `connection_timeout` integer COMMENT ' The amount of time to wait (in milliseconds) when initially establishing a connection before giving up and timing out.', + `max_error_retry` integer COMMENT ' The maximum number of retry attempts for failed retryable requests (ex: 5xx error responses from services).', + `socket_timeout` integer COMMENT ' The amount of time to wait (in milliseconds) for data to be transfered over an established, open connection before the connection times out and is closed.', + `created` datetime COMMENT 'date the s3 first signed on', + PRIMARY KEY (`id`), + CONSTRAINT `uc_s3__uuid` UNIQUE (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `cloud`.`template_s3_ref` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `s3_id` bigint unsigned NOT NULL COMMENT ' Associated S3 instance id', + `template_id` bigint unsigned NOT NULL COMMENT ' Associated template id', + `created` DATETIME NOT NULL COMMENT ' The creation timestamp', + `size` bigint unsigned COMMENT ' The size of the object', + `physical_size` bigint unsigned DEFAULT 0 COMMENT ' The physical size of the object', + PRIMARY KEY (`id`), + CONSTRAINT `uc_template_s3_ref__template_id` UNIQUE (`template_id`), + CONSTRAINT `fk_template_s3_ref__s3_id` FOREIGN KEY `fk_template_s3_ref__s3_id` (`s3_id`) REFERENCES `s3` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_template_s3_ref__template_id` FOREIGN KEY `fk_template_s3_ref__template_id` (`template_id`) REFERENCES `vm_template` (`id`), + INDEX `i_template_s3_ref__swift_id`(`s3_id`), + INDEX `i_template_s3_ref__template_id`(`template_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.enable', 'false', 'enable s3'); + +ALTER TABLE `cloud`.`snapshots` ADD COLUMN `s3_id` bigint unsigned COMMENT 'S3 to which this snapshot will be stored'; + +ALTER TABLE `cloud`.`snapshots` ADD CONSTRAINT `fk_snapshots__s3_id` FOREIGN KEY `fk_snapshots__s3_id` (`s3_id`) REFERENCES `s3` (`id`); + diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index eeaf2a29cc2..abff8d15980 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -116,6 +116,7 @@ known_categories = { 'LB': 'Load Balancer', 'ldap': 'LDAP', 'Swift': 'Swift', + 'S3' : 'S3', 'SecondaryStorage': 'Host', 'Project': 'Project', 'Lun': 'Storage', diff --git a/tools/cli/cloudmonkey/cloudmonkey.py b/tools/cli/cloudmonkey/cloudmonkey.py index 250ea68a5f1..e59a3b36164 100644 --- a/tools/cli/cloudmonkey/cloudmonkey.py +++ b/tools/cli/cloudmonkey/cloudmonkey.py @@ -312,7 +312,8 @@ class CloudStackShell(cmd.Cmd, object): if result is None: return try: - self.print_result(result.values()) + # Response is in the key "apiname+response" (lowercase) + self.print_result(result[api_name.lower()+'response']) print except Exception as e: self.print_shell("🙈 Error on parsing and printing", e) diff --git a/tools/devcloud/README.md b/tools/devcloud/README.md new file mode 100644 index 00000000000..410c36f5ea5 --- /dev/null +++ b/tools/devcloud/README.md @@ -0,0 +1,102 @@ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +=========================================================== + +NOTE - This folder is a work in progress. The project has not determined +how to best establish a nightly DevCloud build process, or how to distribute +the image. + +=========================================================== + +# Setting up Tools and Environment + + - Install [RVM](https://rvm.io/rvm/install) + - Setup paths: + export PATH=~/.rvm/bin:$PATH + - Install Ruby 1.9.3, if it installed some other version: + rvm install 1.9.3 + +All the dependencies will be fetched automatically. + +Vagrant: https://github.com/chipchilders/vagrant.git +Veewee: https://github.com/jedi4ever/veewee.git + +devcloudbase/Ubuntu: http://releases.ubuntu.com/12.04/ubuntu-12.04.1-server-i386.iso + +To save some time if you've downloaded iso of your distro, put the isos in: +tools/devcloud/deps/boxes/basebox-build/iso/ + +Note, gem would require gcc-4.2, make sure link exists: + + sudo ln -s /usr/bin/gcc /usr/bin/gcc-4.2 + +# How to build DevCloud + +DevCloud build scripts are in src/ +Move to src/deps/ to start the build process: + + cd src/deps/ + +Clean up any old stuff: + + ./boxer.sh -c all + +Build the dependent vms: + + ./boxer.sh -b all + +Now, start DevCloud: + + # Go back to the devcloud homedir + cd ../ + # Bring up the devcloud vm + vagrant up + +If you get a vagrant error, at that point, try: + + source .rvmrc + vagrant up + +# CloudStack Build Automation in DevCloud + +If you want to compile cloudstack in the devcloud vm: + + vim puppet/modules/devcloud/manifests/params.pp + +and set + + $build_cloudstack = true + +alternately, if you do not want to build cloudstack in the devcloud vm, set: + + $build_cloudstack = false + + +It will now bring up the devcloud vm for this first time. Note that it will +attempt to download the SSVM and CPVM templates so it will take a long time to +launch initially. It will also git clone the cloudstack repository and attempt +to build an launch it. + +You can optionally speed things up by packaging a successful devcloud instance +build. This will make subsequent launches must faster since it won't have to +re-downoad the SSVM and CPVM. Once it has successfully been built, you can run: + + #exports the devcloud vagrant instance and adds it as "devcloud" to vagrant boxlist + ./boxit.sh + #modifies the Vagrant file to use this newly added instance + sed -i 's,devcloudbase-xen,devcloud,g' Vagrantfile diff --git a/tools/devcloud/basebuild/Vagrantfile b/tools/devcloud/basebuild/Vagrantfile deleted file mode 100644 index a1f0740bc84..00000000000 --- a/tools/devcloud/basebuild/Vagrantfile +++ /dev/null @@ -1,51 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -Vagrant::Config.run do |config| - config.vm.box = "devcloudbase" - # TODO: Get a URL to host the base image - # config.vm.box_url = "http://domain.com/path/to/above.box" - config.vm.guest = :xen - config.ssh.username = "devcloud" - - # Uncomment this line to enable the console for debugging the - # build process. - #config.vm.boot_mode = :gui - - # Setup port forwarding - config.vm.forward_port 22, 2222 - config.vm.forward_port 8080, 8080 - config.vm.forward_port 8443, 8443 - config.vm.forward_port 5901, 5901 - config.vm.forward_port 8787, 8787 - config.vm.forward_port 8250, 8250 - - # Ensure the VM has the right virtual resources - #config.vm. - - config.vm.provision :puppet do |puppet| - puppet.manifests_path = "puppet-devcloudinitial" - puppet.manifest_file = "init.pp" - puppet.with_ssh = true - puppet.pp_path = "/etc/puppet" - puppet.module_path = "puppet-devcloudinitial" - end - -end diff --git a/tools/devcloud/basebuild/puppet-devcloudinitial/files/network.conf b/tools/devcloud/basebuild/puppet-devcloudinitial/files/network.conf deleted file mode 100644 index 7096907a5ff..00000000000 --- a/tools/devcloud/basebuild/puppet-devcloudinitial/files/network.conf +++ /dev/null @@ -1 +0,0 @@ -bridge diff --git a/tools/devcloud/basebuild/puppet-devcloudinitial/manifests/init.pp b/tools/devcloud/basebuild/puppet-devcloudinitial/manifests/init.pp deleted file mode 100644 index ded206759e2..00000000000 --- a/tools/devcloud/basebuild/puppet-devcloudinitial/manifests/init.pp +++ /dev/null @@ -1,106 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -class puppet-devcloudinitial { - - package { 'linux-headers-3.2.0-23-generic': - ensure => latest, - } - - package { 'xen-hypervisor-4.1-i386': - ensure => latest, - } - - package { 'xcp-xapi': - require => Package['xen-hypervisor-4.1-i386'], - ensure => latest, - } - - file { '/etc/xcp/network.conf': - require => Package['xcp-xapi'], - ensure => 'file', - source => 'puppet:///modules/puppet-devcloudinitial/network.conf', - group => '0', - mode => '644', - owner => '0', - } - - file { '/etc/init.d/xend': - require => Package['xcp-xapi'], - ensure => 'file', - source => 'puppet:///modules/puppet-devcloudinitial/xend', - group => '0', - owner => '0', - mode => '755', - } - - service { 'xendomains': - require => Package['xcp-xapi'], - ensure => 'stopped', - enable => 'false', - } - - file { '/etc/default/grub': - require => Package['xen-hypervisor-4.1-i386'], - ensure => 'file', - source => 'puppet:///modules/puppet-devcloudinitial/grub', - group => '0', - mode => '644', - owner => '0', - } - - exec { "/usr/sbin/update-grub": - subscribe => File['/etc/default/grub'], - refreshonly => true, - cwd => '/', - } - - file { '/usr/share/qemu': - require => Package['xen-hypervisor-4.1-i386'], - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - } - - file { '/usr/share/qemu/keymaps': - require => File['/usr/share/qemu'], - ensure => 'link', - group => '0', - mode => '777', - owner => '0', - target => '/usr/share/qemu-linaro/keymaps', - } - - file { '/etc/network/interfaces': - ensure => 'file', - source => 'puppet:///modules/puppet-devcloudinitial/interfaces', - group => '0', - mode => '644', - owner => '0', - } - - file { '/etc/default/xen': - require => Package['xen-hypervisor-4.1-i386'], - ensure => 'file', - source => 'puppet:///modules/puppet-devcloudinitial/xen-defaults', - group => '0', - mode => '644', - owner => '0', - } - -} diff --git a/tools/devcloud/build_vagrant_basebox.sh b/tools/devcloud/build_vagrant_basebox.sh deleted file mode 100755 index c90a6c9ba60..00000000000 --- a/tools/devcloud/build_vagrant_basebox.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Assumes that rvm is installed and you have ruby 1.9.2 installed -# Assumes that you have virtual box installed -# Assumes that you have wget installed -set -x -PROGNAME=$(basename $0) -function error_exit { - - # Display error message and exit - echo "${PROGNAME}: $*" 1>&2 - exit 1 -} - - -# Load RVM into a shell session *as a function* -if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then -# First try to load from a user install - source "$HOME/.rvm/scripts/rvm" - -elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then - -# Then try to load from a root install - source "/usr/local/rvm/scripts/rvm" - -else - - printf "ERROR: An RVM installation was not found.\n" - -fi - -mkdir ~/builddevcloud || error_exit -cd ~/builddevcloud || error_exit -git clone https://github.com/jedi4ever/veewee.git || error_exit -#TODO: We need to get this patched version of Vagrant to the upstream project -# or implement the desired changes to Vagrant as plugin modules and -# host it within the ASF git repo -git clone https://github.com/chipchilders/vagrant.git || error_exit -export rvm_trust_rvmrcs_flag=1 || error_exit -cd vagrant || error_exit -bundle install || error_exit "could not bundle install vagrant" -rake install || error_exit "could not rake vagrant" -cd ~/builddevcloud/veewee || error_exit -cp -R templates/ubuntu-12.04-server-i386 templates/ubuntu-12.04.1-server-i386 -cp -R templates/ubuntu-12.04-server-i386-packages templates/ubuntu-12.04.1-server-i386-packages -bundle install || error_exit -rake install || error_exit -bundle exec vagrant basebox define 'devcloudbase' 'ubuntu-12.04.1-server-i386' || error_exit "couldn't basebox define" -wget --no-check-certificate -O ./definitions/devcloudbase/definition.rb https://git-wip-us.apache.org/repos/asf\?p\=incubator-cloudstack.git\;a\=blob_plain\;f\=tools/devcloud/veewee/definition.rb\;hb\=HEAD || error_exit "couldn't get file" -wget --no-check-certificate -O ./definitions/devcloudbase/postinstall.sh https://git-wip-us.apache.org/repos/asf\?p\=incubator-cloudstack.git\;a\=blob_plain\;f\=tools/devcloud/veewee/postinstall.sh\;hb\=HEAD || error_exit "couldn't get file" -wget --no-check-certificate -O ./definitions/devcloudbase/preseed.cfg https://git-wip-us.apache.org/repos/asf\?p\=incubator-cloudstack.git\;a\=blob_plain\;f\=tools/devcloud/veewee/preseed.cfg\;hb\=HEAD || error_exit "couldn't get file" -bundle exec vagrant basebox build 'devcloudbase' -f -a -n || error_exit "couldn't basebox build" -# possibly use -r here too ^ -bundle exec vagrant basebox export 'devcloudbase' -f || error_exit "couldn't basebox export" -bundle exec vagrant basebox destroy 'devcloudbase' -f || error_exit "couldn't basebox destroy" -bundle exec vagrant box add 'devcloudbase' 'devcloudbase.box' -f || error_exit "couldn't basebox add" -rm -f devcloudbase.box || error_exit -cd ~/builddevcloud/vagrant || error_exit -mkdir devcloudbase || error_exit -cd devcloudbase || error_exit -mkdir puppet-devcloudinitial || error_exit -mkdir puppet-devcloudinitial/files || error_exit -mkdir puppet-devcloudinitial/manifests || error_exit -wget --no-check-certificate -O Vagrantfile "https://git-wip-us.apache.org/repos/asf?p=incubator-cloudstack.git;a=blob_plain;f=tools/devcloud/basebuild/Vagrantfile;hb=HEAD" || error_exit -wget --no-check-certificate -O puppet-devcloudinitial/init.pp "https://git-wip-us.apache.org/repos/asf?p=incubator-cloudstack.git;a=blob_plain;f=tools/devcloud/basebuild/puppet-devcloudinitial/init.pp;hb=HEAD" || error_exit -wget --no-check-certificate -O puppet-devcloudinitial/Modulefile "https://git-wip-us.apache.org/repos/asf?p=incubator-cloudstack.git;a=blob_plain;f=tools/devcloud/basebuild/puppet-devcloudinitial/Modulefile;hb=HEAD" || error_exit -wget --no-check-certificate -O puppet-devcloudinitial/files/grub "https://git-wip-us.apache.org/repos/asf?p=incubator-cloudstack.git;a=blob_plain;f=tools/devcloud/basebuild/puppet-devcloudinitial/files/grub;hb=HEAD" || error_exit -wget --no-check-certificate -O puppet-devcloudinitial/files/interfaces "https://git-wip-us.apache.org/repos/asf?p=incubator-cloudstack.git;a=blob_plain;f=tools/devcloud/basebuild/puppet-devcloudinitial/files/interfaces;hb=HEAD" || error_exit -wget --no-check-certificate -O puppet-devcloudinitial/files/network.conf "https://git-wip-us.apache.org/repos/asf?p=incubator-cloudstack.git;a=blob_plain;f=tools/devcloud/basebuild/puppet-devcloudinitial/files/network.conf;hb=HEAD" || error_exit -wget --no-check-certificate -O puppet-devcloudinitial/files/xen-defaults "https://git-wip-us.apache.org/repos/asf?p=incubator-cloudstack.git;a=blob_plain;f=tools/devcloud/basebuild/puppet-devcloudinitial/files/xen-defaults;hb=HEAD" || error_exit -wget --no-check-certificate -O puppet-devcloudinitial/files/xend "https://git-wip-us.apache.org/repos/asf?p=incubator-cloudstack.git;a=blob_plain;f=tools/devcloud/basebuild/puppet-devcloudinitial/files/xend;hb=HEAD" || error_exit -wget --no-check-certificate -O puppet-devcloudinitial/manifests/init.pp "https://git-wip-us.apache.org/repos/asf?p=incubator-cloudstack.git;a=blob_plain;f=tools/devcloud/basebuild/puppet-devcloudinitial/manifests/init.pp;hb=HEAD" || error_exit -cd ~/builddevcloud/vagrant/ -bundle install || error_exit -rake install || error_exit -cd ~/builddevcloud/vagrant/devcloudbase/ -bundle exec vagrant up || error_exit "vagrant up failed" -bundle exec vagrant halt || error_exit "vagrant halt failed" -bundle exec vagrant package default --output ~/devcloud.box || error_exit "vagrant package failed" -bundle exec vagrant destroy -f || error_exit "vagrant destroy failed" -bundle exec vagrant box remove devcloudbase virtualbox || error_exit "vagrant box remove failed" - -echo "Your new devcloud base box is stored in ~/devcloud.box" diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/manifests/init.pp b/tools/devcloud/devcloudbox/puppet-devcloud/manifests/init.pp deleted file mode 100644 index 3dc74a18206..00000000000 --- a/tools/devcloud/devcloudbox/puppet-devcloud/manifests/init.pp +++ /dev/null @@ -1,348 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -class puppet-devcloud { - - package { 'ebtables': - ensure => latest, - } - - service { 'ebtables': - require => Package['ebtables'], - ensure => 'running', - enable => 'true', - } - - package { 'iptables': - ensure => latest, - } - - file { '/etc/iptables.save': - require => Package['iptables'], - ensure => 'file', - source => 'puppet:///modules/puppet-devcloud/iptables.save', - group => '0', - mode => '644', - owner => '0', - } - - file { '/tmp/configebtables.sh': - ensure => 'file', - source => 'puppet:///modules/puppet-devcloud/configebtables.sh', - group => '0', - mode => '777', - owner => '0', - } - - exec { "/tmp/configebtables.sh": - require => [ - File['/tmp/configebtables.sh'], - Service['ebtables'] - ], - subscribe => Package['ebtables'], - refreshonly => true, - cwd => '/', - path => '/sbin/:/usr/bin/:/bin', - } - - package { 'nfs-server': - ensure => latest, - } - - file { '/opt/storage': - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - } - - file { '/opt/storage/secondary': - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - } - - file { '/opt/storage/secondary/template': - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - } - - file { '/opt/storage/secondary/template/tmpl': - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - } - - file { '/opt/storage/secondary/template/tmpl/1': - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - } - - file { '/opt/storage/secondary/template/tmpl/1/1': - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - } - - file { '/opt/storage/secondary/template/tmpl/1/5': - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - } - - file { '/etc/exports': - require => Package['nfs-server'], - ensure => 'file', - source => 'puppet:///modules/puppet-devcloud/exports', - mode => '644', - owner => '0', - group => '0', - } - - service { 'nfs-kernel-server': - require => Package['nfs-server'], - ensure => 'running', - enable => 'true', - } - -# TODO - it would be great to have an MD5 sum to check for each of these downloads, so that the files can be re-downloaded if they have been changed. - - exec { '/usr/bin/wget http://download.cloud.com/templates/devcloud/defaulttemplates/1/dc68eb4c-228c-4a78-84fa-b80ae178fbfd.vhd -P /opt/storage/secondary/template/tmpl/1/1/': - creates => '/opt/storage/secondary/template/tmpl/1/1/dc68eb4c-228c-4a78-84fa-b80ae178fbfd.vhd', - require => File['/opt/storage/secondary/template/tmpl/1/1/'], - timeout => '0', - } - - exec { '/usr/bin/wget http://download.cloud.com/templates/devcloud/defaulttemplates/1/template.properties -P /opt/storage/secondary/template/tmpl/1/1/': - creates => '/opt/storage/secondary/template/tmpl/1/1/template.properties', - require => File['/opt/storage/secondary/template/tmpl/1/1/'], - } - - exec { '/usr/bin/wget http://download.cloud.com/templates/devcloud/defaulttemplates/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd -P /opt/storage/secondary/template/tmpl/1/5/': - creates => '/opt/storage/secondary/template/tmpl/1/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd', - require => File['/opt/storage/secondary/template/tmpl/1/5/'], - timeout => '0', - } - - exec { '/usr/bin/wget http://download.cloud.com/templates/devcloud/defaulttemplates/5/template.properties -P /opt/storage/secondary/template/tmpl/1/5/': - creates => '/opt/storage/secondary/template/tmpl/1/5/template.properties', - require => File['/opt/storage/secondary/template/tmpl/1/5/'], - } - - exec { 'getecho': - command => '/usr/bin/wget http://download.cloud.com/templates/devcloud/echo -P /usr/lib/xcp/plugins/', - creates => '/usr/lib/xcp/plugins/echo', - } - - exec { '/bin/chmod -R 777 /usr/lib/xcp': - require => Exec['getecho'], - } - - file { '/opt/storage/primary': - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - } - - file { '/tmp/configlocalstorage.sh': - ensure => 'file', - group => '0', - mode => '777', - owner => '0', - source => 'puppet:///modules/puppet-devcloud/configlocalstorage.sh', - } - - exec { "configlocal": - require => [ - File['/opt/storage/primary'], - File['/tmp/configlocalstorage.sh'] - ], - command => '/tmp/configlocalstorage.sh', - cwd => '/', - } - - file { '/tmp/configvnc.sh': - ensure => 'file', - source => 'puppet:///modules/puppet-devcloud/configvnc.sh', - mode => '777', - group => '0', - owner => '0', - } - - exec { "configvnc": - require => File['/tmp/configvnc.sh'], - command => '/tmp/configvnc.sh', - cwd => '/', - } - - package { 'git': - ensure => latest, - } - - package { 'unzip': - ensure => latest, - } - - package { 'mysql-server': - ensure => latest, - } - - package { 'ant': - ensure => latest, - } - - package { 'openjdk-6-jdk': - ensure => latest, - } - - file { '/opt/cloudstack': - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - } - - file { '/tmp/updatecode.sh': - ensure => 'file', - source => 'puppet:///modules/puppet-devcloud/updatecode.sh', - mode => '777', - owner => '0', - group => '0', - } - - exec { 'get_code': - require => [ - Package['git'], - File['/opt/cloudstack/'], - File['/tmp/updatecode.sh'] - ], - command => '/tmp/updatecode.sh', - cwd => '/opt/cloudstack/', - timeout => '0', - } - - file { '/opt/cloudstack/incubator-cloudstack/target': - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - require => Exec['get_code'], - } - - file { '/opt/cloudstack/incubator-cloudstack/dist': - ensure => 'directory', - group => '0', - mode => '755', - owner => '0', - require => Exec['get_code'], - } - - exec { 'downloadtomcat': - command => '/usr/bin/wget http://archive.apache.org/dist/tomcat/tomcat-6/v6.0.32/bin/apache-tomcat-6.0.32.zip -P /opt/cloudstack/', - creates => '/opt/cloudstack/apache-tomcat-6.0.32.zip', - require => File['/opt/cloudstack/'], - timeout => '0', - } - - exec { "unziptomcat": - require => [ - Package['unzip'], - Exec["downloadtomcat"] - ], - creates => "/opt/cloudstack/apache-tomcat-6.0.32", - command => "/usr/bin/unzip apache-tomcat-6.0.32.zip", - cwd => "/opt/cloudstack", - timeout => '0', - } - - exec { "catalina_home": - require => Exec["unziptomcat"], - unless => '/bin/grep CATALINA_HOME /root/.bashrc', - command => '/bin/echo "export CATALINA_HOME=/opt/cloudstack/apache-tomcat-6.0.32" >> /root/.bashrc', - cwd => '/', - } - - package { 'mkisofs': - ensure => latest, - } - - file { '/opt/cloudstack/buildcloudstack.sh': - ensure => 'file', - source => 'puppet:///modules/puppet-devcloud/builddevcloud.sh', - mode => '777', - owner => '0', - group => '0', - } - - file { '/opt/cloudstack/installmaven.sh': - ensure => 'file', - source => 'puppet:///modules/puppet-devcloud/installmaven.sh', - mode => '777', - owner => '0', - group => '0', - } - - exec { "install_maven": - require => File['/opt/cloudstack/installmaven.sh'], - command => '/opt/cloudstack/installmaven.sh', - cwd => '/opt/cloudstack', - creates => '/opt/cloudstack/apache-maven-3.0.4/', - timeout => '0', - } - - exec { "build_cloudstack": - require => [ - Package['ant'], - Exec['install_maven'], - Exec["catalina_home"], - File['/opt/cloudstack/incubator-cloudstack/dist'], - File['/opt/cloudstack/incubator-cloudstack/target'], - Package['mkisofs'], - File['/opt/cloudstack/buildcloudstack.sh'] - ], - command => "/opt/cloudstack/buildcloudstack.sh", - cwd => "/opt/cloudstack/", - timeout => '0', - } - - file { '/opt/cloudstack/startdevcloud.sh': - ensure => 'file', - source => 'puppet:///modules/puppet-devcloud/startdevcloud.sh', - mode => '777', - owner => '0', - group => '0', - } - - exec { "start_cloudstack": - require => [ - Exec["build_cloudstack"], - File["/opt/cloudstack/startdevcloud.sh"] - ], - command => "/opt/cloudstack/startdevcloud.sh", - cwd => "/opt/cloudstack/", - } - -} diff --git a/tools/devcloud/devcloudsetup.sh b/tools/devcloud/devcloudsetup.sh deleted file mode 100644 index f8b69faa92e..00000000000 --- a/tools/devcloud/devcloudsetup.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -fileSvr="http://download.cloud.com/templates/devcloud/" -install_xen() { - aptitude update - echo "install xen" - aptitude -y install linux-headers-3.2.0-23-generic-pae - aptitude -y install xen-hypervisor-4.1-i386 xcp-xapi - echo "configure xen" - - sed -i -e 's/xend_start$/#xend_start/' -e 's/xend_stop$/#xend_stop/' /etc/init.d/xend - update-rc.d xendomains disable - - sed -i 's/GRUB_DEFAULT=.\+/GRUB_DEFAULT="Xen 4.1-i386"/' /etc/default/grub - echo 'GRUB_CMDLINE_XEN_DEFAULT="dom0_mem=512M,max:512M"' | cat /etc/default/grub - >> /etc/default/newgrub - mv /etc/default/newgrub /etc/default/grub - update-grub - - mkdir /usr/share/qemu - ln -s /usr/share/qemu-linaro/keymaps /usr/share/qemu/keymaps - -cat > /etc/network/interfaces << EOF -# The loopback network interface -auto lo -iface lo inet loopback - -# The primary network interface -auto xenbr0 -iface xenbr0 inet dhcp - gateway 10.0.2.2 - bridge_ports eth0 - - -auto eth0 -iface eth0 inet dhcp -pre-up iptables-save < /etc/iptables.save -pre-up /etc/init.d/ebtables load -EOF - - echo TOOLSTACK=xapi > /etc/default/xen - echo bridge > /etc/xcp/network.conf - - echo "set root password" - echo "root:password" | chpasswd - - echo "reboot" - reboot -} - -postsetup() { - #check xen dom0 is working - xe host-list > /dev/null - if [ $? -gt 0 ]; then - print "xen dom0 is not running, make sure dom0 is installed" - exit 1 - fi - - #disable virtualbox dhcp server for Vms created by cloudstack - apt-get install ebtables - iptables -A POSTROUTING -t mangle -p udp --dport bootpc -j CHECKSUM --checksum-fill - mac=`ifconfig xenbr0 |grep HWaddr |awk '{print $5}'` - ebtables -I FORWARD -d ! $mac -i eth0 -p IPV4 --ip-prot udp --ip-dport 67:68 -j DROP - iptables-save > /etc/iptables.save - /etc/init.d/ebtables save - - echo "configure NFS server" - aptitude -y install nfs-server - if [ ! -d /opt/storage/secondary ];then - mkdir -p /opt/storage/secondary - mkdir -p /opt/storage/secondary/template/tmpl/1/1 - mkdir -p /opt/storage/secondary/template/tmpl/1/5 - - echo "/opt/storage/secondary *(rw,no_subtree_check,no_root_squash,fsid=0)" > /etc/exports - wget $fileSvr/defaulttemplates/1/dc68eb4c-228c-4a78-84fa-b80ae178fbfd.vhd -P /opt/storage/secondary/template/tmpl/1/1/ - wget $fileSvr/defaulttemplates/1/template.properties -P /opt/storage/secondary/template/tmpl/1/1/ - wget $fileSvr/defaulttemplates/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd -P /opt/storage/secondary/template/tmpl/1/5/ - wget $fileSvr/defaulttemplates/5/template.properties -P /opt/storage/secondary/template/tmpl/1/5/ - /etc/init.d/nfs-kernel-server restart - fi - - echo "configure local storage" - if [ ! -d /opt/storage/primary ]; then - mkdir -p /opt/storage/primary - hostuuid=`xe host-list |grep uuid|awk '{print $5}'` - xe sr-create host-uuid=$hostuuid name-label=local-storage shared=false type=file device-config:location=/opt/storage/primary - fi - - echo "generate ssh key" - ssh-keygen -A -q - - echo "configure xcp" - wget $fileSvr/echo -P /usr/lib/xcp/plugins/ - chmod -R 777 /usr/lib/xcp - - sed -i 's/VNCTERM_LISTEN=.\+/VNCTERM_LISTEN="-v 0.0.0.0:1"/' /usr/lib/xcp/lib/vncterm-wrapper - - echo "install cloudstack " - - if [ ! -d /opt/cloudstack ];then - aptitude -y install git unzip openjdk-6-jdk mysql-server ant - mkdir /opt/cloudstack - cd /opt/cloudstack - git clone https://git-wip-us.apache.org/repos/asf/incubator-cloudstack.git - mkdir incubator-cloudstack/target - mkdir incubator-cloudstack/dist - wget http://archive.apache.org/dist/tomcat/tomcat-6/v6.0.32/bin/apache-tomcat-6.0.32.zip -P /opt/cloudstack/ - unzip apache-tomcat-6.0.32.zip - echo "export CATALINA_HOME=/opt/cloudstack/apache-tomcat-6.0.32" >> /root/.bashrc - cd ~ - fi - - echo "devCloud is ready to use" -} -usage() { - print "$0 -p: presetup enviroment, e.g. install xen, configure xcp etc" - print "$0 -P: postsetup, install cloudstack, prepare template etc" -} - -while getopts "pP" OPTION -do - case $OPTION in - p) - install_xen - exit 0 - ;; - P) - postsetup - exit 0 - ;; - ?) - usage - exit - ;; - esac -done diff --git a/tools/devcloud/src/.rvmrc b/tools/devcloud/src/.rvmrc new file mode 100644 index 00000000000..7e51d62af9f --- /dev/null +++ b/tools/devcloud/src/.rvmrc @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +rvm use ruby-1.9.3@vagrant-xen-cloudstack --create +export VAGRANT_HOME=$HOME/.vagrant.d-xen-cloudstack +bundle check > /dev/null 2>&1 +RETVAL=$? +if [ $RETVAL -ne 0 ]; then + bundle install +fi diff --git a/tools/devcloud/basebuild/puppet-devcloudinitial/files/interfaces b/tools/devcloud/src/Gemfile similarity index 73% rename from tools/devcloud/basebuild/puppet-devcloudinitial/files/interfaces rename to tools/devcloud/src/Gemfile index f65507911ab..2790c4a52a0 100644 --- a/tools/devcloud/basebuild/puppet-devcloudinitial/files/interfaces +++ b/tools/devcloud/src/Gemfile @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -15,19 +15,5 @@ # specific language governing permissions and limitations # under the License. -# The loopback network interface -auto lo -iface lo inet loopback - -# The primary network interface -auto xenbr0 -iface xenbr0 inet dhcp - gateway 10.0.2.2 - bridge_ports eth0 - - -auto eth0 -iface eth0 inet dhcp -pre-up iptables-save < /etc/iptables.save -pre-up /etc/init.d/ebtables load - +source 'https://rubygems.org' +gem 'vagrant', :git => 'https://github.com/chipchilders/vagrant' diff --git a/tools/devcloud/devcloudbox/Vagrantfile b/tools/devcloud/src/Vagrantfile similarity index 57% rename from tools/devcloud/devcloudbox/Vagrantfile rename to tools/devcloud/src/Vagrantfile index 121a9f58a94..3f0d9045bb6 100644 --- a/tools/devcloud/devcloudbox/Vagrantfile +++ b/tools/devcloud/src/Vagrantfile @@ -8,9 +8,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,33 +19,40 @@ # under the License. Vagrant::Config.run do |config| - config.vm.box = "devcloud" - # TODO: Get a URL to host the base image - # config.vm.box_url = "http://domain.com/path/to/above.box" + config.vm.network :hostonly, "192.168.56.10", :auto_config => false + #config.vm.box = "devcloud" + config.vm.box = "devcloudbase-xen" + #config.vm.box_url = "http://basho-cloudstack.s3.amazonaws.com/devcloud.box" + config.vm.guest = :xen config.ssh.username = "devcloud" - - # Uncomment this line to enable the console for debugging the + config.vm.host_name = "devcloud.local" + # Uncomment this line to enable the console for debugging the # build process. # config.vm.boot_mode = :gui # Setup port forwarding - config.vm.forward_port 22, 2222 + config.vm.forward_port 22, 7222 + config.vm.forward_port 3306, 3306 config.vm.forward_port 8080, 8080 config.vm.forward_port 8443, 8443 config.vm.forward_port 5901, 5901 config.vm.forward_port 8787, 8787 config.vm.forward_port 8250, 8250 - + config.vm.forward_port 8096, 8096 + config.vm.forward_port 7080, 7080 # Ensure the VM has the right virtual resources #config.vm. - + config.vm.customize ["modifyvm", :id, "--memory", 2048] + config.vm.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] + config.vm.provision :shell, :path => "waitforxe.sh" config.vm.provision :puppet do |puppet| - puppet.manifests_path = "puppet-devcloud" - puppet.manifest_file = "init.pp" - puppet.with_ssh = true - puppet.pp_path = "/etc/puppet" - puppet.module_path = "puppet-devcloud" + puppet.with_ssh = true + puppet.manifests_path = File.join 'puppet', 'manifests' + puppet.module_path = File.join 'puppet', 'modules', 'devcloud' + puppet.manifest_file = 'vagrant-devcloud.pp' + puppet.options = ['--pluginsync', '--trace', '--debug', '--verbose'] end + end diff --git a/tools/devcloud/src/boxit.sh b/tools/devcloud/src/boxit.sh new file mode 100755 index 00000000000..dae53cee6bc --- /dev/null +++ b/tools/devcloud/src/boxit.sh @@ -0,0 +1,21 @@ +#! /bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +vagrant halt +vagrant package default --output devcloud.box +vagrant box add devcloud devcloud.box -f diff --git a/tools/devcloud/src/deps/boxer.sh b/tools/devcloud/src/deps/boxer.sh new file mode 100755 index 00000000000..72d9cd1e6f3 --- /dev/null +++ b/tools/devcloud/src/deps/boxer.sh @@ -0,0 +1,184 @@ +#! /bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Load RVM into a shell session *as a function* +if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then +# First try to load from a user install + source "$HOME/.rvm/scripts/rvm" +elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then +# Then try to load from a root install + source "/usr/local/rvm/scripts/rvm" +else + printf "ERROR: An RVM installation was not found.\n" +fi + +BASEDIR=$PWD/boxes +DEVCLOUD_BASEBUILD_DIR=$BASEDIR/basebox-build +echo $DEVCLOUD_BASEBUILD_DIR +DEVCLOUD_XEN_BASEBUILD_DIR=$BASEDIR/xenbox-build +DEVCLOUD_BASE_NAME='devcloudbase' +DEVCLOUD_XEN_BASE_NAME='devcloudbase-xen' +OS='ubuntu-12.04.1-server-i386' + + +basebox() { + set +x + rvm rvmrc trust $DEVCLOUD_BASEBUILD_DIR/ + case "$1" in + build) + cd $DEVCLOUD_BASEBUILD_DIR/ + set -ex + vagrant basebox define $DEVCLOUD_BASE_NAME $OS + cp definition.rb postinstall.sh preseed.cfg definitions/$DEVCLOUD_BASE_NAME/ + vagrant basebox build $DEVCLOUD_BASE_NAME -f -a -n -r + vagrant basebox export $DEVCLOUD_BASE_NAME -f + set +ex + cd $DEVCLOUD_XEN_BASEBUILD_DIR + set -ex + vagrant box add $DEVCLOUD_BASE_NAME $DEVCLOUD_BASEBUILD_DIR/${DEVCLOUD_BASE_NAME}.box -f + ;; + clean) + cd $DEVCLOUD_BASEBUILD_DIR/ + set -x + rm -f iso/*.iso + vagrant -f basebox destroy $DEVCLOUD_BASE_NAME #-f + vagrant basebox undefine $DEVCLOUD_BASE_NAME + #hackery to inherit the proper rvmrc for the hacked vagrant + set +x + cd $BAS$DEVCLOUD_XEN_BASEBUILD_DIR + set -x + vagrant -f box remove $DEVCLOUD_BASE_NAME virtualbox + set +x + cd $DEVCLOUD_BASEBUILD_DIR + set -x + rm -f ${DEVCLOUD_BASE_NAME}.box + set +x + cd $BASEDIR + #rvm --force gemset delete vagrant-release-cloudstack + ;; + esac +} + +xenbox() { + + set +x + rvm rvmrc trust $DEVCLOUD_XEN_BASEBUILD_DIR/ + case "$1" in + build) + cd $DEVCLOUD_XEN_BASEBUILD_DIR + + #adding it here because it needs to be added into the $VAGRANT_HOME of + #the hacked vagrant + set -ex + vagrant up + vagrant halt + vagrant package default --output ${DEVCLOUD_XEN_BASE_NAME}.box + vagrant box add $DEVCLOUD_XEN_BASE_NAME ${DEVCLOUD_XEN_BASE_NAME}.box -f + ;; + clean) + cd $DEVCLOUD_XEN_BASEBUILD_DIR + set -x + vagrant -f box remove $DEVCLOUD_XEN_BASE_NAME virtualbox + vagrant destroy -f + rm -f ${DEVCLOUD_XEN_BASE_NAME}.box + set +x + #rvm --force gemset delete vagrant-xen-cloudstack + set -x + ;; + esac +} +usage() { + +cat <&2 + echo "$usage" >&2 + exit 1 + ;; + esac +done +shift $((OPTIND - 1)) + +posargs=$@ + +#removes duplicate positionals + +posargs=$(echo "$posargs" | tr ' ' '\n' | nl | sort -u -k2 | sort -n | cut -f2-) + + + +for arg in $posargs; do + + case "$arg" in + basebox) + true + ;; + xenbox) + true + ;; + all) + true + ;; + *) + usage + exit 1 + ;; + esac + +done + +cd $BASEDIR + +for arg in $posargs; do + case "$1" in + "all") + case "$action" in + clean) + xenbox $action + basebox $action + ;; + build) + basebox $action + xenbox $action + ;; + esac + ;; + $arg) + $arg $action + ;; + esac +done diff --git a/tools/devcloud/src/deps/boxes/basebox-build/.rvmrc b/tools/devcloud/src/deps/boxes/basebox-build/.rvmrc new file mode 100644 index 00000000000..3c8c66c97f9 --- /dev/null +++ b/tools/devcloud/src/deps/boxes/basebox-build/.rvmrc @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +rvm use ruby-1.9.3@vagrant-release-cloudstack --create +export VAGRANT_HOME=$HOME/.vagrant.d-release-cloudstack +bundle check > /dev/null 2>&1 +RETVAL=$? +if [ $RETVAL -ne 0 ]; then + bundle install +fi diff --git a/tools/devcloud/src/deps/boxes/basebox-build/Gemfile b/tools/devcloud/src/deps/boxes/basebox-build/Gemfile new file mode 100644 index 00000000000..dae53be6dd0 --- /dev/null +++ b/tools/devcloud/src/deps/boxes/basebox-build/Gemfile @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source 'https://rubygems.org' +gem 'veewee', :git => 'https://github.com/jedi4ever/veewee.git' +gem 'em-winrm' diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/init.pp b/tools/devcloud/src/deps/boxes/basebox-build/Vagrantfile similarity index 96% rename from tools/devcloud/devcloudbox/puppet-devcloud/init.pp rename to tools/devcloud/src/deps/boxes/basebox-build/Vagrantfile index 2574392251a..245692337bc 100644 --- a/tools/devcloud/devcloudbox/puppet-devcloud/init.pp +++ b/tools/devcloud/src/deps/boxes/basebox-build/Vagrantfile @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -15,4 +15,3 @@ # specific language governing permissions and limitations # under the License. -include puppet-devcloud diff --git a/tools/devcloud/veewee/definition.rb b/tools/devcloud/src/deps/boxes/basebox-build/definition.rb similarity index 93% rename from tools/devcloud/veewee/definition.rb rename to tools/devcloud/src/deps/boxes/basebox-build/definition.rb index cb0a9079520..24668421c5d 100644 --- a/tools/devcloud/veewee/definition.rb +++ b/tools/devcloud/src/deps/boxes/basebox-build/definition.rb @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. - Veewee::Session.declare({ :cpu_count => '1', :memory_size=> '2048', @@ -28,9 +27,13 @@ Veewee::Session.declare({ :iso_md5 => '32184a83c8b5e6031e1264e5c499bc03', :iso_download_timeout => "1000", :boot_wait => "4", - :ioapic => "on", - :nestedpaging => "on", - :hwvirtex => "on", + :virtualbox => { + :vm_options => [ + "ioapic" => "on", + "hwvirtex" => "on", + "nestedpaging" => "on" + ] + }, :boot_cmd_sequence => [ '', '/install/vmlinuz noapic preseed/url=http://%IP%:%PORT%/preseed.cfg ', diff --git a/tools/devcloud/veewee/postinstall.sh b/tools/devcloud/src/deps/boxes/basebox-build/postinstall.sh similarity index 99% rename from tools/devcloud/veewee/postinstall.sh rename to tools/devcloud/src/deps/boxes/basebox-build/postinstall.sh index aeafafef17d..217d23024aa 100644 --- a/tools/devcloud/veewee/postinstall.sh +++ b/tools/devcloud/src/deps/boxes/basebox-build/postinstall.sh @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -40,4 +40,3 @@ chown -R devcloud /home/devcloud/.ssh # Zero out the free space to save space in the final image: dd if=/dev/zero of=/EMPTY bs=1M rm -f /EMPTY - diff --git a/tools/devcloud/veewee/preseed.cfg b/tools/devcloud/src/deps/boxes/basebox-build/preseed.cfg similarity index 99% rename from tools/devcloud/veewee/preseed.cfg rename to tools/devcloud/src/deps/boxes/basebox-build/preseed.cfg index 4a97171e76f..00bae613647 100644 --- a/tools/devcloud/veewee/preseed.cfg +++ b/tools/devcloud/src/deps/boxes/basebox-build/preseed.cfg @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/tools/devcloud/src/deps/boxes/xenbox-build/.rvmrc b/tools/devcloud/src/deps/boxes/xenbox-build/.rvmrc new file mode 100644 index 00000000000..7e51d62af9f --- /dev/null +++ b/tools/devcloud/src/deps/boxes/xenbox-build/.rvmrc @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +rvm use ruby-1.9.3@vagrant-xen-cloudstack --create +export VAGRANT_HOME=$HOME/.vagrant.d-xen-cloudstack +bundle check > /dev/null 2>&1 +RETVAL=$? +if [ $RETVAL -ne 0 ]; then + bundle install +fi diff --git a/tools/devcloud/src/deps/boxes/xenbox-build/Gemfile b/tools/devcloud/src/deps/boxes/xenbox-build/Gemfile new file mode 100644 index 00000000000..2790c4a52a0 --- /dev/null +++ b/tools/devcloud/src/deps/boxes/xenbox-build/Gemfile @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source 'https://rubygems.org' +gem 'vagrant', :git => 'https://github.com/chipchilders/vagrant' diff --git a/tools/devcloud/src/deps/boxes/xenbox-build/Vagrantfile b/tools/devcloud/src/deps/boxes/xenbox-build/Vagrantfile new file mode 100644 index 00000000000..58e9e64a1d0 --- /dev/null +++ b/tools/devcloud/src/deps/boxes/xenbox-build/Vagrantfile @@ -0,0 +1,52 @@ + # -*- mode: ruby -*- +# vi: set ft=ruby : + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +Vagrant::Config.run do |config| + config.vm.box = "devcloudbase" + # TODO: Get a URL to host the base image + # config.vm.box_url = "http://domain.com/path/to/above.box" + config.vm.guest = :xen + config.ssh.username = "devcloud" + config.vm.host_name = "devcloud.local" + + # Uncomment this line to enable the console for debugging the + # build process. + #config.vm.boot_mode = :gui + + # Setup port forwarding + # config.vm.forward_port 22, 2222 + # config.vm.forward_port 8080, 8080 + # config.vm.forward_port 8443, 8443 + # config.vm.forward_port 5901, 5901 + # config.vm.forward_port 8787, 8787 + # config.vm.forward_port 8250, 8250 + + # Ensure the VM has the right virtual resources + #config.vm. + + config.vm.provision :puppet do |puppet| + puppet.with_ssh = true + puppet.manifests_path = File.join 'puppet', 'manifests' + puppet.module_path = File.join 'puppet', 'modules', 'devcloudinitial' + puppet.manifest_file = 'vagrant-devcloudinitial.pp' + puppet.options = ['--trace', '--debug', '--verbose'] + end + +end diff --git a/tools/devcloud/src/deps/boxes/xenbox-build/puppet/manifests/site.pp b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/manifests/site.pp new file mode 100644 index 00000000000..13a83393a91 --- /dev/null +++ b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/manifests/site.pp @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tools/devcloud/src/deps/boxes/xenbox-build/puppet/manifests/vagrant-devcloudinitial.pp b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/manifests/vagrant-devcloudinitial.pp new file mode 100644 index 00000000000..0d48071536a --- /dev/null +++ b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/manifests/vagrant-devcloudinitial.pp @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permission s and limitations +# under the License. + +node default { include devcloudinitial } diff --git a/tools/devcloud/basebuild/puppet-devcloudinitial/Modulefile b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/Modulefile similarity index 100% rename from tools/devcloud/basebuild/puppet-devcloudinitial/Modulefile rename to tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/Modulefile diff --git a/tools/devcloud/basebuild/puppet-devcloudinitial/files/grub b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/grub similarity index 99% rename from tools/devcloud/basebuild/puppet-devcloudinitial/files/grub rename to tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/grub index be14e82f7c5..d5de16c536b 100644 --- a/tools/devcloud/basebuild/puppet-devcloudinitial/files/grub +++ b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/grub @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/interfaces b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/interfaces new file mode 100644 index 00000000000..0c197209d10 --- /dev/null +++ b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/interfaces @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet manual + +allow-hotplug eth1 +iface eth1 inet manual + +auto xenbr0 +iface xenbr0 inet dhcp + bridge_ports eth0 + dns_nameservers 8.8.8.8 8.8.4.4 + post-up route add default gw 10.0.2.2 + +auto xenbr1 +iface xenbr1 inet static + bridge_ports eth1 + address 192.168.56.10 + netmask 255.255.255.0 + network 192.168.56.0 + broadcast 192.168.56.255 + gateway 192.168.56.1 + dns_nameservers 8.8.8.8 8.8.4.4 + post-up route del default gw 192.168.56.1; route add default gw 192.168.56.1 metric 100; + + +pre-up iptables-save < /etc/iptables.save +pre-up /etc/init.d/ebtables load diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/files/iptables.save b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/iptables.save similarity index 99% rename from tools/devcloud/devcloudbox/puppet-devcloud/files/iptables.save rename to tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/iptables.save index 333932723dc..07647f83bad 100644 --- a/tools/devcloud/devcloudbox/puppet-devcloud/files/iptables.save +++ b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/iptables.save @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/tools/devcloud/basebuild/puppet-devcloudinitial/files/xen-defaults b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/xen-defaults similarity index 99% rename from tools/devcloud/basebuild/puppet-devcloudinitial/files/xen-defaults rename to tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/xen-defaults index bc3dc67b522..908396812fb 100644 --- a/tools/devcloud/basebuild/puppet-devcloudinitial/files/xen-defaults +++ b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/xen-defaults @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/tools/devcloud/basebuild/puppet-devcloudinitial/files/xend b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/xend similarity index 99% rename from tools/devcloud/basebuild/puppet-devcloudinitial/files/xend rename to tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/xend index 19ebd49e226..4a532992f94 100644 --- a/tools/devcloud/basebuild/puppet-devcloudinitial/files/xend +++ b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/files/xend @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/manifests/init.pp b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/manifests/init.pp new file mode 100644 index 00000000000..e1f4c5ac271 --- /dev/null +++ b/tools/devcloud/src/deps/boxes/xenbox-build/puppet/modules/devcloudinitial/manifests/init.pp @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class devcloudinitial { + + if $::architecture == 'x86_64'{ + $debarch='amd64' + } + else { + $debarch='i386' + } + + exec { "apt-update": + command => "/usr/bin/apt-get update" + } + + Exec["apt-update"] -> Package <| |> + + package { + "linux-headers-${::kernelrelease}": + ensure => latest; + "xen-hypervisor-4.1-${debarch}": + ensure => latest, + require => Package["linux-headers-${::kernelrelease}"]; + 'xcp-xapi': + ensure => latest, + require => Package["xen-hypervisor-4.1-${debarch}"]; + 'iptables': + ensure => latest; + 'ebtables': + ensure => latest; + } + + file { + '/etc/iptables.save': + ensure => 'file', + require => Package['iptables'], + source => 'puppet:///modules/devcloudinitial/iptables.save', + group => '0', + mode => '0644', + owner => '0'; + '/etc/xcp/network.conf': + ensure => 'file', + require => Package['xcp-xapi'], + content => "bridge", + group => '0', + mode => '0644', + owner => '0'; + '/etc/init.d/xend': + ensure => 'file', + require => Package['xcp-xapi'], + source => 'puppet:///modules/devcloudinitial/xend', + group => '0', + owner => '0', + mode => '0755'; + '/etc/default/grub': + ensure => 'file', + require => Package["xen-hypervisor-4.1-${debarch}"], + source => 'puppet:///modules/devcloudinitial/grub', + group => '0', + mode => '0644', + owner => '0'; + '/usr/share/qemu': + ensure => 'directory', + require => Package["xen-hypervisor-4.1-${debarch}"], + group => '0', + mode => '0755', + owner => '0'; + '/usr/share/qemu/keymaps': + ensure => 'link', + require => File['/usr/share/qemu'], + group => '0', + mode => '0777', + owner => '0', + target => '/usr/share/qemu-linaro/keymaps'; + '/etc/network/interfaces': + ensure => 'file', + source => 'puppet:///modules/devcloudinitial/interfaces', + group => '0', + mode => '0644', + owner => '0'; + '/etc/default/xen': + ensure => 'file', + require => Package["xen-hypervisor-4.1-${debarch}"], + source => 'puppet:///modules/devcloudinitial/xen-defaults', + group => '0', + mode => '0644', + owner => '0'; + } + + service { + 'xendomains': + ensure => 'stopped', + require => Package['xcp-xapi'], + enable => false; + } + + exec { '/usr/sbin/update-grub': + subscribe => File['/etc/default/grub'], + refreshonly => true, + cwd => '/', + } + +} diff --git a/tools/devcloud/basebuild/puppet-devcloudinitial/init.pp b/tools/devcloud/src/puppet/manifests/vagrant-devcloud.pp similarity index 95% rename from tools/devcloud/basebuild/puppet-devcloudinitial/init.pp rename to tools/devcloud/src/puppet/manifests/vagrant-devcloud.pp index 213ef597008..47b07dc6dd1 100644 --- a/tools/devcloud/basebuild/puppet-devcloudinitial/init.pp +++ b/tools/devcloud/src/puppet/manifests/vagrant-devcloud.pp @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -15,4 +15,4 @@ # specific language governing permissions and limitations # under the License. -include puppet-devcloudinitial +include devcloud diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/Modulefile b/tools/devcloud/src/puppet/modules/devcloud/Modulefile similarity index 100% rename from tools/devcloud/devcloudbox/puppet-devcloud/Modulefile rename to tools/devcloud/src/puppet/modules/devcloud/Modulefile diff --git a/tools/devcloud/src/puppet/modules/devcloud/files/compare.sh b/tools/devcloud/src/puppet/modules/devcloud/files/compare.sh new file mode 100644 index 00000000000..e0ee95ca077 --- /dev/null +++ b/tools/devcloud/src/puppet/modules/devcloud/files/compare.sh @@ -0,0 +1,22 @@ +#! /bin/bash -eux +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FILE=$1 +WORKING_DIR=$2 +cd $WORKING_DIR +test `grep $FILE ${WORKING_DIR}/md5sum.txt | awk '{print $1}'` == `md5sum $FILE |awk '{print $1}'` diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/files/configebtables.sh b/tools/devcloud/src/puppet/modules/devcloud/files/configebtables.sh similarity index 99% rename from tools/devcloud/devcloudbox/puppet-devcloud/files/configebtables.sh rename to tools/devcloud/src/puppet/modules/devcloud/files/configebtables.sh index 741884ee665..83293336640 100644 --- a/tools/devcloud/devcloudbox/puppet-devcloud/files/configebtables.sh +++ b/tools/devcloud/src/puppet/modules/devcloud/files/configebtables.sh @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/files/configlocalstorage.sh b/tools/devcloud/src/puppet/modules/devcloud/files/configlocalstorage.sh similarity index 94% rename from tools/devcloud/devcloudbox/puppet-devcloud/files/configlocalstorage.sh rename to tools/devcloud/src/puppet/modules/devcloud/files/configlocalstorage.sh index ab47d2dad4f..3ed1a39dcd7 100644 --- a/tools/devcloud/devcloudbox/puppet-devcloud/files/configlocalstorage.sh +++ b/tools/devcloud/src/puppet/modules/devcloud/files/configlocalstorage.sh @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,7 +19,7 @@ /usr/bin/xe sr-list | /bin/grep local-storage rc=$? if [[ $rc != 0 ]] ; then - hostuuid=`xe host-list |grep uuid|awk '{print $5}'`; + hostuuid=`xe host-list |grep uuid|awk '{print $5}'`; xe sr-create host-uuid=$hostuuid name-label=local-storage shared=false type=file device-config:location=/opt/storage/primary fi exit 0 diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/files/configvnc.sh b/tools/devcloud/src/puppet/modules/devcloud/files/configvnc.sh similarity index 99% rename from tools/devcloud/devcloudbox/puppet-devcloud/files/configvnc.sh rename to tools/devcloud/src/puppet/modules/devcloud/files/configvnc.sh index b13a7a307ab..b739dc6acb0 100644 --- a/tools/devcloud/devcloudbox/puppet-devcloud/files/configvnc.sh +++ b/tools/devcloud/src/puppet/modules/devcloud/files/configvnc.sh @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/files/exports b/tools/devcloud/src/puppet/modules/devcloud/files/exports similarity index 92% rename from tools/devcloud/devcloudbox/puppet-devcloud/files/exports rename to tools/devcloud/src/puppet/modules/devcloud/files/exports index c270c6306bd..1f9165011ac 100644 --- a/tools/devcloud/devcloudbox/puppet-devcloud/files/exports +++ b/tools/devcloud/src/puppet/modules/devcloud/files/exports @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -15,4 +15,5 @@ # specific language governing permissions and limitations # under the License. +/opt/storage/primary *(rw,no_subtree_check,no_root_squash,fsid=0) /opt/storage/secondary *(rw,no_subtree_check,no_root_squash,fsid=0) diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/files/installmaven.sh b/tools/devcloud/src/puppet/modules/devcloud/files/installmaven.sh similarity index 99% rename from tools/devcloud/devcloudbox/puppet-devcloud/files/installmaven.sh rename to tools/devcloud/src/puppet/modules/devcloud/files/installmaven.sh index 48ffdfe8de7..8cd3df01c89 100644 --- a/tools/devcloud/devcloudbox/puppet-devcloud/files/installmaven.sh +++ b/tools/devcloud/src/puppet/modules/devcloud/files/installmaven.sh @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/files/startdevcloud.sh b/tools/devcloud/src/puppet/modules/devcloud/files/startdevcloud.sh similarity index 95% rename from tools/devcloud/devcloudbox/puppet-devcloud/files/startdevcloud.sh rename to tools/devcloud/src/puppet/modules/devcloud/files/startdevcloud.sh index f496891d40f..27a7a044851 100644 --- a/tools/devcloud/devcloudbox/puppet-devcloud/files/startdevcloud.sh +++ b/tools/devcloud/src/puppet/modules/devcloud/files/startdevcloud.sh @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,5 +19,5 @@ export CATALINA_HOME=/opt/cloudstack/apache-tomcat-6.0.32 cd /opt/cloudstack/incubator-cloudstack/ -nohup ant run > /dev/null 2>&1 & +nohup ant run > /dev/null 2>&1 & exit 0 diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/files/updatecode.sh b/tools/devcloud/src/puppet/modules/devcloud/files/updatecode.sh similarity index 99% rename from tools/devcloud/devcloudbox/puppet-devcloud/files/updatecode.sh rename to tools/devcloud/src/puppet/modules/devcloud/files/updatecode.sh index 15e191aacb9..55259208c40 100644 --- a/tools/devcloud/devcloudbox/puppet-devcloud/files/updatecode.sh +++ b/tools/devcloud/src/puppet/modules/devcloud/files/updatecode.sh @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/tools/devcloud/src/puppet/modules/devcloud/lib/facter/xeninfo.rb b/tools/devcloud/src/puppet/modules/devcloud/lib/facter/xeninfo.rb new file mode 100644 index 00000000000..a0b8748577a --- /dev/null +++ b/tools/devcloud/src/puppet/modules/devcloud/lib/facter/xeninfo.rb @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permission s and limitations +# under the License. + +Facter.add(:xen_hostuuid) do + setcode do + uuid=Facter::Util::Resolution.exec('xe host-list |grep uuid|awk \'{print $5}\'') + end +end diff --git a/tools/devcloud/src/puppet/modules/devcloud/manifests/functions/httpdownload.pp b/tools/devcloud/src/puppet/modules/devcloud/manifests/functions/httpdownload.pp new file mode 100644 index 00000000000..2a88771beee --- /dev/null +++ b/tools/devcloud/src/puppet/modules/devcloud/manifests/functions/httpdownload.pp @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permission s and limitations +# under the License. + +define devcloud::functions::httpdownload () { + $file="${name['basedir']}/${name['basefile']}" + + exec { + "getfileifnotexist${name}": + command => "/usr/bin/wget ${name['url']}/${file} -O ${name['local_dir']}/${file}", + timeout => 0, + unless => "test -f ${name['local_dir']}/${file}", + require => [ File["${name['local_dir']}/${name['base_dir']}/"], + Exec["get_md5sums"] ]; + + + "getfileifnotmatch${name}": + command => "/usr/bin/wget ${name['url']}/${file} -O ${name['local_dir']}/${file}", + timeout => 0, + unless => "/usr/local/bin/compare.sh ${file} ${name['working_dir']} ", + require => [ Exec["getfileifnotexist${name}"], File["/usr/local/bin/compare.sh"] ] + } + +} diff --git a/tools/devcloud/src/puppet/modules/devcloud/manifests/init.pp b/tools/devcloud/src/puppet/modules/devcloud/manifests/init.pp new file mode 100644 index 00000000000..260e2542e01 --- /dev/null +++ b/tools/devcloud/src/puppet/modules/devcloud/manifests/init.pp @@ -0,0 +1,255 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permission s and limitations +# under the License. + + +class devcloud ( + + $cs_dir = $devcloud::params::cs_dir , + $devcloud_path = $devcloud::params::devcloud_path, + $gitrepo = $devcloud::params::gitrepo, + $storage_dir = $devcloud::params::storage_dir, + $tomcat_version = $devcloud::params::tomcat_version, + $tomcat_url = $devcloud::params::tomcat_url, + $tomcat_home = $devcloud::params::tomcat_home, + $maven_version = $devcloud::params::maven_version, + $maven_url = $devcloud::params::maven_url, + $maven_home = $devcloud::params::maven_home, + $downloads = $devcloud::params::downloads, + $md5sum_local = $devcloud::params::md5sum_local, + $md5sum_remote = $devcloud::params::md5sum_remote, + $hostuuid = $::xen_hostuuid, + $bridge_device_mac = $::macaddress_xenbr0, + $build_cloudstack = $devcloud::params::build_cloudstack + +) inherits devcloud::params { + + Exec { path => [ '/bin/', '/sbin/' , '/usr/bin/', '/usr/sbin/' ] } + + Exec["apt-update"] -> Package <| |> + + service { + 'ebtables': + ensure => 'running', + enable => true; + + 'nfs-kernel-server': + ensure => 'running', + require => Package['nfs-server'], + enable => true, + subscribe => File['/etc/exports']; + } + + package + { [ 'ant', + 'git', + 'python-mysql.connector', + 'mkisofs', + 'mysql-server', + 'nfs-server', + 'openjdk-6-jdk', + 'unzip' ]: + ensure => latest, + } + + exec { + + 'apt-update': + command => '/usr/bin/apt-get update'; + + 'get_md5sums': + command => "/usr/bin/wget -N ${md5sum_remote} -O ${md5sum_local}", + require => File["${storage_dir}/secondary/template/tmpl/1/"], + timeout => '0'; + + 'getecho': + command => "/usr/bin/wget ${devcloud_path}/echo -O /usr/lib/xcp/plugins/echo", + creates => '/usr/lib/xcp/plugins/echo'; + + 'setxcpperms': + command => '/bin/chmod -R 755 /usr/lib/xcp', + require => Exec['getecho']; + + 'get_code': + command => "git clone ${gitrepo}", + cwd => $cs_dir, + require => File[$cs_dir], + timeout => '7200', + creates => "${cs_dir}/incubator-cloudstack"; + + 'update_code': + command => 'git pull origin master', + cwd => "${cs_dir}/incubator-cloudstack", + timeout => '7200', + require => [ Exec['get_code']]; + + 'configlocal': + command => "xe sr-create host-uuid=${hostuuid} name-label=local-storage shared=false type=file device-config:location=${storage_dir}/primary", + cwd => '/', + unless => '/usr/bin/xe sr-list | /bin/egrep \'local-storage|Cloud Stack Local EXT Storage Pool\'', + require => [ + File["${storage_dir}/primary"], + ]; + + 'configvnc': + command => 'sed -i \'s/VNCTERM_LISTEN=.\+/VNCTERM_LISTEN="-v 0.0.0.0:1"/\' /usr/lib/xcp/lib/vncterm-wrapper', + onlyif => '/bin/grep "0.0.0.0:1" /usr/lib/xcp/lib/vncterm-wrapper'; + + 'downloadtomcat': + command => "/usr/bin/wget ${tomcat_url} -P ${cs_dir}/", + creates => "${cs_dir}/apache-tomcat-${tomcat_version}.zip", + require => File[$cs_dir], + timeout => '0'; + + 'unziptomcat': + require => [ + Package['unzip'], + Exec['downloadtomcat'] + ], + creates => $tomcat_home, + command => "/usr/bin/unzip apache-tomcat-${tomcat_version}.zip", + cwd => $cs_dir, + timeout => '0'; + + 'downloadmaven': + command => "/usr/bin/wget ${maven_url} -P ${cs_dir}/", + creates => "${cs_dir}/apache-maven-${maven_version}-bin.tar.gz", + require => Exec['unziptomcat'], + timeout => '0'; + + 'install_maven': + require => Exec['downloadmaven'], + creates => $maven_home, + command => "/bin/tar xzvf ${cs_dir}/apache-maven-${maven_version}-bin.tar.gz", + cwd => $cs_dir, + timeout => '0'; + + 'tomcatperms': + command => "chmod +x ${tomcat_home}/bin/*.sh", + require => Exec['unziptomcat']; + + 'catalina_home': + require => Exec['unziptomcat'], + unless => '/bin/grep CATALINA_HOME /root/.bashrc', + command => "/bin/echo \"export CATALINA_HOME=${tomcat_home}\" >> /root/.bashrc", + cwd => '/'; + + 'configebtables': + require => Service['ebtables'], + command => "/sbin/ebtables -I FORWARD -d ! ${bridge_device_mac} -i eth1 -p IPV4 --ip-prot udp --ip-dport 67:68 -j DROP", + unless => "/sbin/ebtables -L | grep \"-I FORWARD -d ! ${bridge_device_mac} -i eth1 -p IPV4 --ip-prot udp --ip-dport 67:68 -j DROP\"", + refreshonly => true, + cwd => '/', + path => '/sbin/:/usr/bin/:/bin' +} + + if $build_cloudstack { + + exec { + 'build_cloudstack': + require => [ + Package['ant','mkisofs'], + Exec['install_maven'], + File[ + "${cs_dir}/incubator-cloudstack/dist", + "${cs_dir}/incubator-cloudstack/target", + "${cs_dir}/buildcloudstack.sh"] + ], + command => "${cs_dir}/buildcloudstack.sh", + cwd => $cs_dir, + timeout => '0'; + } + } + + + file { + + [ $cs_dir, + $storage_dir, + "${storage_dir}/primary", + "${storage_dir}/secondary", + "${storage_dir}/secondary/template", + "${storage_dir}/secondary/template/tmpl", + "${storage_dir}/secondary/template/tmpl/1", + "${storage_dir}/secondary/template/tmpl/1/1", + "${storage_dir}/secondary/template/tmpl/1/5" ]: + ensure => 'directory', + group => '0', + mode => '0755', + owner => '0'; + + [ "${cs_dir}/incubator-cloudstack/dist", + "${cs_dir}/incubator-cloudstack/target" ] : + ensure => 'directory', + group => '0', + mode => '0755', + owner => '0', + require => [ Exec['update_code']]; + + "${cs_dir}/buildcloudstack.sh": + ensure => 'file', + mode => '0755', + owner => '0', + group => '0', + content => template('devcloud/buildcloudstack.sh.erb'); + + "${cs_dir}/startcloudstack.sh": + ensure => 'file', + mode => '0755', + owner => '0', + group => '0', + content => template('devcloud/startcloudstack.sh.erb'); + + '/root/.ssh' : + ensure => 'directory', + group => 'root', + mode => '0700', + owner => 'root'; + + "${cs_dir}/startdevcloud.sh": + ensure => 'file', + source => 'puppet:///modules/devcloud/startdevcloud.sh', + mode => '0755', + owner => '0', + group => '0'; + + '/usr/local/bin/compare.sh': + ensure => 'file', + source => 'puppet:///modules/devcloud/compare.sh', + mode => '0755', + owner => 'root', + group => 'root'; + + '/etc/exports': + ensure => 'file', + require => Package['nfs-server'], + source => 'puppet:///modules/devcloud/exports', + mode => '0644', + owner => '0', + group => '0'; + + } + + + devcloud::functions::httpdownload{ + $downloads: + require => + [ File["${storage_dir}/secondary/template/tmpl/1/1"], + File["${storage_dir}/secondary/template/tmpl/1/5"], + Exec['get_md5sums'] ] + } + +} diff --git a/tools/devcloud/src/puppet/modules/devcloud/manifests/params.pp b/tools/devcloud/src/puppet/modules/devcloud/manifests/params.pp new file mode 100644 index 00000000000..ff625c176f3 --- /dev/null +++ b/tools/devcloud/src/puppet/modules/devcloud/manifests/params.pp @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permission s and limitations +# under the License. + +# == Class: devcloud::params +# +# This class implements the module params pattern, but it's loaded using hiera +# as opposed to the 'default' usage of coding the parameter values in your +# manifest. +# +# == Usage +# +# Don't use this class directly; it's being used where it is needed +# +class devcloud::params { + + $cs_dir = '/opt/cloudstack' + $storage_dir = '/opt/storage' + $tomcat_version = '6.0.32' + $tomcat_url = "http://archive.apache.org/dist/tomcat/tomcat-6/v${tomcat_version}/bin/apache-tomcat-${tomcat_version}.zip" + $tomcat_home = "${cs_dir}/apache-tomcat-${tomcat_version}" + $maven_version = '3.0.4' + $maven_url = "http://apache.mirrors.pair.com/maven/maven-3/${maven_version}/binaries/apache-maven-${maven_version}-bin.tar.gz" + $maven_home = "${cs_dir}/apache-maven-${maven_version}" + $devcloud_path = 'http://download.cloud.com/templates/devcloud' + $template_path = "${devcloud_path}/defaulttemplates" + $md5sum_remote = "${template_path}/md5sum.txt" + $md5sum_local = "${storage_dir}/secondary/template/tmpl/1/md5sum.txt" + $template_dir = "${storage_dir}/secondary/template/tmpl/1" + $gitrepo = 'https://github.com/apache/incubator-cloudstack.git' + $build_cloudstack = false + + + $downloads = [ + + { + 'basefile' => 'template.properties', + 'basedir' => '1', + 'url' => $template_path, + 'local_dir' => $template_dir, + 'working_dir' => $template_dir + }, + { + 'basefile' => 'template.properties', + 'basedir' => '5', + 'url' => $template_path, + 'local_dir' => $template_dir, + 'working_dir' => $template_dir + }, + { + 'basefile' => 'dc68eb4c-228c-4a78-84fa-b80ae178fbfd.vhd', + 'basedir' => '1', + 'url' => $template_path, + 'local_dir' => $template_dir, + 'working_dir' => $template_dir + }, + { + 'basefile' => 'ce5b212e-215a-3461-94fb-814a635b2215.vhd', + 'basedir' => '5', + 'url' => $template_path, + 'local_dir' => $template_dir, + 'working_dir' => $template_dir + } + ] +} diff --git a/tools/devcloud/src/puppet/modules/devcloud/templates/buildcloudstack.sh.erb b/tools/devcloud/src/puppet/modules/devcloud/templates/buildcloudstack.sh.erb new file mode 100644 index 00000000000..acf9b0b3b95 --- /dev/null +++ b/tools/devcloud/src/puppet/modules/devcloud/templates/buildcloudstack.sh.erb @@ -0,0 +1,28 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export CATALINA_HOME=<%= @tomcat_home %> +export M2_HOME=<%= @maven_home %> +export M2=$M2_HOME/bin +MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=800m -Xdebug -Xrunjdwp:transport=dt_socket,address=8787,server=y,suspend=n" +PATH=$M2:$PATH +cd <%= @cs_dir %>/incubator-cloudstack/ +<%= @maven_home %>/bin/mvn clean install -P developer,systemvm +<%= @maven_home %>/bin/mvn -pl developer,tools/devcloud -Ddeploydb -P developer +<%= @maven_home %>/bin/mvn -P developer -pl tools/devcloud -Ddeploysvr diff --git a/tools/devcloud/devcloudbox/puppet-devcloud/files/builddevcloud.sh b/tools/devcloud/src/puppet/modules/devcloud/templates/startcloudstack.sh.erb similarity index 74% rename from tools/devcloud/devcloudbox/puppet-devcloud/files/builddevcloud.sh rename to tools/devcloud/src/puppet/modules/devcloud/templates/startcloudstack.sh.erb index 03bd854e015..4a07711591b 100644 --- a/tools/devcloud/devcloudbox/puppet-devcloud/files/builddevcloud.sh +++ b/tools/devcloud/src/puppet/modules/devcloud/templates/startcloudstack.sh.erb @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -17,12 +17,10 @@ # specific language governing permissions and limitations # under the License. -export CATALINA_HOME=/opt/cloudstack/apache-tomcat-6.0.32 -export M2_HOME=/opt/cloudstack/apache-maven-3.0.4 +export CATALINA_HOME=<%= @tomcat_home %> +export M2_HOME=<%= @maven_home %> export M2=$M2_HOME/bin -MAVEN_OPTS="-Xms256m -Xmx512m" +MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=800m -Xdebug -Xrunjdwp:transport=dt_socket,address=8787,server=y,suspend=n" PATH=$M2:$PATH -cd /opt/cloudstack/incubator-cloudstack/ -/usr/bin/mvn -P deps -/usr/bin/mvn clean -/usr/bin/ant clean-all build-all deploy-server deploydb +cd <%= @cs_dir %>/incubator-cloudstack/ +<%= @maven_home %>/bin/mvn -pl client jetty:run & diff --git a/tools/devcloud/src/waitforxe.sh b/tools/devcloud/src/waitforxe.sh new file mode 100755 index 00000000000..637b9fc92c9 --- /dev/null +++ b/tools/devcloud/src/waitforxe.sh @@ -0,0 +1,39 @@ +#! /bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +date +interval=20 +timeout=300 +command="xe host-list" + +count=0 +maxcount=$(($timeout/$interval)) + +until [ $count -gt $maxcount ]; do + if $command > /dev/null 2>&1; then + echo "\"$command\" executed successfully." + date + exit 0 + fi + let count=count+1 + echo "Waiting for \"$command\" to run successfully." + sleep $interval +done + +echo "\"$command\" failed to complete." +date diff --git a/tools/marvin/marvin/cloudstackConnection.py b/tools/marvin/marvin/cloudstackConnection.py index c8052130fe6..8c4e3251690 100644 --- a/tools/marvin/marvin/cloudstackConnection.py +++ b/tools/marvin/marvin/cloudstackConnection.py @@ -42,7 +42,7 @@ class cloudConnection(object): else: self.protocol=protocol self.path = path - if port == 8096: + if port == 8096 or (self.apiKey == None and self.securityKey == None): self.auth = False else: self.auth = True diff --git a/tools/marvin/marvin/deployDataCenter.py b/tools/marvin/marvin/deployDataCenter.py index bdf08cc2d2d..7bdd9751c16 100644 --- a/tools/marvin/marvin/deployDataCenter.py +++ b/tools/marvin/marvin/deployDataCenter.py @@ -258,9 +258,9 @@ class deployDataCenters(): traffic_type = addTrafficType.addTrafficTypeCmd() traffic_type.physicalnetworkid = physical_network_id traffic_type.traffictype = traffictype.typ - if traffictype.labeldict: - traffic_type.kvmnetworklabel = traffictype.labeldict.xen - traffic_type.xennetworklabel = traffictype.labeldict.kvm + if traffictype.labeldict is not None: + traffic_type.kvmnetworklabel = traffictype.labeldict.kvm + traffic_type.xennetworklabel = traffictype.labeldict.xen traffic_type.vmwarenetworklabel = traffictype.labeldict.vmware return self.apiClient.addTrafficType(traffic_type) @@ -399,9 +399,9 @@ class deployDataCenters(): logging=self.testClientLogger) """config database""" - dbSvr = self.config.dbSvr - self.testClient.dbConfigure(dbSvr.dbSvr, dbSvr.port, dbSvr.user, \ - dbSvr.passwd, dbSvr.db) + #dbSvr = self.config.dbSvr + #self.testClient.dbConfigure(dbSvr.dbSvr, dbSvr.port, dbSvr.user, \ + # dbSvr.passwd, dbSvr.db) self.apiClient = self.testClient.getApiClient() def updateConfiguration(self, globalCfg): @@ -419,7 +419,6 @@ class deployDataCenters(): self.createZones(self.config.zones) self.updateConfiguration(self.config.globalConfig) - if __name__ == "__main__": parser = OptionParser() diff --git a/tools/marvin/marvin/integration/lib/base.py b/tools/marvin/marvin/integration/lib/base.py index e8c3b25bd89..0daea5dd579 100644 --- a/tools/marvin/marvin/integration/lib/base.py +++ b/tools/marvin/marvin/integration/lib/base.py @@ -205,10 +205,16 @@ class VirtualMachine: self.__dict__.update(items) if "username" in services: self.username = services["username"] + else: + self.username = 'root' if "password" in services: self.password = services["password"] + else: + self.password = 'password' if "ssh_port" in services: self.ssh_port = services["ssh_port"] + else: + self.ssh_port = 22 self.ssh_client = None #extract out the ipaddress self.ipaddress = self.nic[0].ipaddress diff --git a/tools/marvin/marvin/integration/lib/common.py b/tools/marvin/marvin/integration/lib/common.py index 5f90110c367..69aa733420e 100644 --- a/tools/marvin/marvin/integration/lib/common.py +++ b/tools/marvin/marvin/integration/lib/common.py @@ -81,7 +81,7 @@ def get_zone(apiclient, services=None): zones = apiclient.listZones(cmd) if isinstance(zones, list): - assert len(zones) > 0 + assert len(zones) > 0, "There are no available zones in the deployment" return zones[0] else: raise Exception("Failed to find specified zone.") @@ -100,7 +100,7 @@ def get_pod(apiclient, zoneid, services=None): pods = apiclient.listPods(cmd) if isinstance(pods, list): - assert len(pods) > 0 + assert len(pods) > 0, "No pods found for zone %s"%zoneid return pods[0] else: raise Exception("Exception: Failed to find specified pod.") @@ -129,10 +129,11 @@ def get_template(apiclient, zoneid, ostype, services=None): list_templates = apiclient.listTemplates(cmd) - assert len(list_templates) > 0 - for template in list_templates: - if template.ostypeid == ostypeid: - return template + if isinstance(list_templates, list): + assert len(list_templates) > 0, "received empty response on template of type %s"%ostype + for template in list_templates: + if template.ostypeid == ostypeid: + return template raise Exception("Exception: Failed to find template with OSTypeID: %s" % ostypeid) diff --git a/tools/marvin/marvin/remoteSSHClient.py b/tools/marvin/marvin/remoteSSHClient.py index 036e1c381e1..95a9adab53f 100644 --- a/tools/marvin/marvin/remoteSSHClient.py +++ b/tools/marvin/marvin/remoteSSHClient.py @@ -19,6 +19,7 @@ import paramiko import time import cloudstackException import contextlib +import logging from contextlib import closing class remoteSSHClient(object): @@ -29,11 +30,16 @@ class remoteSSHClient(object): self.passwd = passwd self.ssh = paramiko.SSHClient() self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.logger = logging.getLogger('sshClient') + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + self.logger.addHandler(ch) retry_count = retries while True: try: self.ssh.connect(str(host),int(port), user, passwd) + self.logger.debug("connecting to server %s with user %s passwd %s"%(str(host), user, passwd)) except paramiko.SSHException, sshex: if retry_count == 0: raise cloudstackException.InvalidParameterException(repr(sshex)) @@ -46,6 +52,7 @@ class remoteSSHClient(object): def execute(self, command): stdin, stdout, stderr = self.ssh.exec_command(command) + self.logger.debug("sending command %s to host %s"%(command, str(self.host))) output = stdout.readlines() errors = stderr.readlines() results = [] @@ -57,7 +64,7 @@ class remoteSSHClient(object): else: for strOut in output: results.append(strOut.rstrip()) - + self.logger.debug("command %s returned %s"%(command, results)) return results def scp(self, srcFile, destPath): diff --git a/ui/css/cloudstack3.css b/ui/css/cloudstack3.css index f233095a6bc..a1a68cb774a 100644 --- a/ui/css/cloudstack3.css +++ b/ui/css/cloudstack3.css @@ -656,44 +656,12 @@ div.list-view div.toolbar div.section-switcher div.section-select label { position: absolute; } -/*** EULA*/ -.install-wizard .eula-copy { - width: 688px; - height: 480px; - background: #FFFFFF; - border: 1px solid #BCBCBC; - /*+box-shadow:0px 4px 10px #B9B9B9;*/ - -moz-box-shadow: 0px 4px 10px #B9B9B9; - -webkit-box-shadow: 0px 4px 10px #B9B9B9; - -o-box-shadow: 0px 4px 10px #B9B9B9; - box-shadow: 0px 4px 10px #B9B9B9; - padding: 5px; - /*+border-radius:4px;*/ - -moz-border-radius: 4px; - -webkit-border-radius: 4px; - -khtml-border-radius: 4px; - border-radius: 4px; - border-radius: 4px 4px 4px 4px; - overflow: auto; - overflow-x: hidden; -} - -.install-wizard .eula-copy p { - font-size: 12px; - margin-top: 21px; -} - /*** Intro*/ .install-wizard .step.intro.what-is-cloudstack p { background: url(../images/bg-what-is-cloudstack.png) no-repeat 50% 237px; height: 540px; } -.install-wizard .step.intro.what-is-cloudplatform p { - background: url(../images/bg-what-is-cloudplatform.png) no-repeat 50% 237px; - height: 540px; -} - /*** Diagram*/ .install-wizard .diagram { width: 910px; @@ -2072,15 +2040,6 @@ div.detail-group.actions td { cursor: pointer; } -#header div.logo { - width: 170px; - height: 47px; - position: relative; - float: left; - margin: 4px 0 0 19px; - background: url(../images/logo-cloudplatform.png) no-repeat 0 center; -} - #header.nologo div.logo { width: 170px; height: 47px; diff --git a/ui/dictionary.jsp b/ui/dictionary.jsp index e72481eeefc..203c9daa52e 100644 --- a/ui/dictionary.jsp +++ b/ui/dictionary.jsp @@ -25,6 +25,17 @@ under the License. <% long now = System.currentTimeMillis(); %>