From 6079e0f18d0d8c7e90837f9a2ea46db200828d9f Mon Sep 17 00:00:00 2001 From: vishesh92 Date: Mon, 5 Jan 2026 17:42:54 +0530 Subject: [PATCH] rotate keys wrapped with older versions --- .../org/apache/cloudstack/kms/KMSManager.java | 124 ++--- .../cloudstack/kms/dao/KMSKekVersionDao.java | 6 + .../kms/dao/KMSKekVersionDaoImpl.java | 7 + .../cloudstack/kms/dao/KMSWrappedKeyDao.java | 10 + .../kms/dao/KMSWrappedKeyDaoImpl.java | 9 + .../META-INF/db/schema-42210to42300.sql | 4 +- .../com/cloud/user/AccountManagerImpl.java | 13 + .../apache/cloudstack/kms/KMSManagerImpl.java | 444 ++++++++++-------- 8 files changed, 320 insertions(+), 297 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/kms/KMSManager.java b/api/src/main/java/org/apache/cloudstack/kms/KMSManager.java index 569d76ae336..44e70805c12 100644 --- a/api/src/main/java/org/apache/cloudstack/kms/KMSManager.java +++ b/api/src/main/java/org/apache/cloudstack/kms/KMSManager.java @@ -127,6 +127,32 @@ public interface KMSManager extends Manager, Configurable { ConfigKey.Scope.Global ); + /** + * Global: batch size for background rewrap operations + */ + ConfigKey KMSRewrapBatchSize = new ConfigKey<>( + "Advanced", + Integer.class, + "kms.rewrap.batch.size", + "50", + "Number of wrapped keys to rewrap per batch in background job", + true, + ConfigKey.Scope.Global + ); + + /** + * Global: interval for background rewrap job + */ + ConfigKey KMSRewrapIntervalMs = new ConfigKey<>( + "Advanced", + Long.class, + "kms.rewrap.interval.ms", + "300000", + "Interval in milliseconds between background rewrap job executions (default: 5 minutes)", + true, + ConfigKey.Scope.Global + ); + // ==================== Provider Management ==================== /** @@ -161,63 +187,6 @@ public interface KMSManager extends Manager, Configurable { */ boolean isKmsEnabled(Long zoneId); - // ==================== KEK Management ==================== - - /** - * Create a new KEK for a zone and purpose - * - * @param zoneId the zone ID - * @param purpose the key purpose - * @param label optional custom label (null for auto-generated) - * @param keyBits key size in bits - * @return the KEK identifier - * @throws KMSException if creation fails - */ - String createKek(Long zoneId, KeyPurpose purpose, String label, int keyBits) throws KMSException; - - /** - * Delete a KEK (WARNING: makes all DEKs wrapped by it unrecoverable) - * - * @param zoneId the zone ID - * @param kekId the KEK identifier - * @throws KMSException if deletion fails - */ - void deleteKek(Long zoneId, String kekId) throws KMSException; - - /** - * List KEKs for a zone and purpose - * - * @param zoneId the zone ID - * @param purpose the purpose filter (null for all) - * @return list of KEK identifiers - * @throws KMSException if listing fails - */ - List listKeks(Long zoneId, KeyPurpose purpose) throws KMSException; - - /** - * Check if a KEK is available - * - * @param zoneId the zone ID - * @param kekId the KEK identifier - * @return true if available - * @throws KMSException if check fails - */ - boolean isKekAvailable(Long zoneId, String kekId) throws KMSException; - - /** - * Rotate a KEK (create new one and rewrap all DEKs) - * - * @param zoneId the zone ID - * @param purpose the purpose - * @param oldKekLabel the old KEK label (must be specified) - * @param newKekLabel the new KEK label (null for auto-generated) - * @param keyBits the new KEK size - * @return the new KEK identifier - * @throws KMSException if rotation fails - */ - String rotateKek(Long zoneId, KeyPurpose purpose, String oldKekLabel, - String newKekLabel, int keyBits) throws KMSException; - // ==================== DEK Operations ==================== /** @@ -233,15 +202,6 @@ public interface KMSManager extends Manager, Configurable { // ==================== Health & Status ==================== - /** - * Check KMS provider health for a zone - * - * @param zoneId the zone ID (null for global) - * @return true if healthy - * @throws KMSException if health check fails critically - */ - boolean healthCheck(Long zoneId) throws KMSException; - // ==================== User KEK Management ==================== /** @@ -274,20 +234,11 @@ public interface KMSManager extends Manager, Configurable { List listUserKMSKeys(Long accountId, Long domainId, Long zoneId, KeyPurpose purpose, KMSKey.State state); - /** - * Get a KMS key by UUID (with permission check) - * - * @param uuid the key UUID - * @param callerAccountId the caller's account ID - * @return the KMS key, or null if not found or no permission - */ - KMSKey getUserKMSKey(String uuid, Long callerAccountId); - /** * Check if caller has permission to use a KMS key * * @param callerAccountId the caller's account ID - * @param keyUuid the key UUID + * @param key the KMS key * @return true if caller has permission */ boolean hasPermission(Long callerAccountId, KMSKey key); @@ -305,7 +256,7 @@ public interface KMSManager extends Manager, Configurable { /** * Generate and wrap a DEK using a specific KMS key UUID * - * @param kekUuid the KMS key UUID + * @param kmsKey the KMS key * @param callerAccountId the caller's account ID * @return wrapped key ready for database storage * @throws KMSException if operation fails @@ -364,17 +315,6 @@ public interface KMSManager extends Manager, Configurable { */ String rotateKMSKey(RotateKMSKeyCmd cmd) throws KMSException; - /** - * Gradually rewrap all wrapped keys for a KMS key to use new KEK version - * - * @param kmsKeyId KMS key ID - * @param newKekVersionId New active KEK version ID - * @param batchSize Number of keys to process per batch - * @return Number of keys successfully rewrapped - * @throws KMSException if rewrap fails - */ - int rewrapWrappedKeysForKMSKey(Long kmsKeyId, Long newKekVersionId, int batchSize) throws KMSException; - /** * Migrate passphrase-based volumes to KMS encryption * @@ -383,4 +323,12 @@ public interface KMSManager extends Manager, Configurable { * @throws KMSException if migration fails */ int migrateVolumesToKMS(MigrateVolumesToKMSCmd cmd) throws KMSException; + + /** + * Delete all KMS keys owned by an account (called during account cleanup) + * + * @param accountId the account ID + * @return true if all keys were successfully deleted + */ + boolean deleteKMSKeysByAccountId(Long accountId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSKekVersionDao.java b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSKekVersionDao.java index 5e61f081b92..8bda982ed00 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSKekVersionDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSKekVersionDao.java @@ -47,4 +47,10 @@ public interface KMSKekVersionDao extends GenericDao { * Find a KEK version by KEK label */ KMSKekVersionVO findByKekLabel(String kekLabel); + + /** + * Find all KEK versions with a specific status + * (useful for background jobs to find versions needing processing) + */ + List findByStatus(KMSKekVersionVO.Status status); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSKekVersionDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSKekVersionDaoImpl.java index 619400f70b4..b34c6f020c7 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSKekVersionDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSKekVersionDaoImpl.java @@ -76,4 +76,11 @@ public class KMSKekVersionDaoImpl extends GenericDaoBase sc.setParameters("kekLabel", kekLabel); return findOneBy(sc); } + + @Override + public List findByStatus(KMSKekVersionVO.Status status) { + SearchCriteria sc = allFieldSearch.create(); + sc.setParameters("status", status); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDao.java b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDao.java index 401c7382f11..2daab72f4ef 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDao.java @@ -62,6 +62,16 @@ public interface KMSWrappedKeyDao extends GenericDao { */ List listByKekVersionId(Long kekVersionId); + /** + * List wrapped keys using a specific KEK version with pagination limit + * (useful for batch processing in background jobs) + * + * @param kekVersionId the KEK version ID (FK to kms_kek_versions) + * @param limit maximum number of keys to return + * @return list of wrapped keys (limited to specified count) + */ + List listByKekVersionId(Long kekVersionId, int limit); + /** * List wrapped keys for a KMS key that need re-encryption (not using specified version) * diff --git a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDaoImpl.java index 97db64e054a..ad924ba59ee 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDaoImpl.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.kms.dao; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -81,6 +82,14 @@ public class KMSWrappedKeyDaoImpl extends GenericDaoBase return listBy(sc); } + @Override + public List listByKekVersionId(Long kekVersionId, int limit) { + SearchCriteria sc = allFieldSearch.create(); + sc.setParameters("kekVersionId", kekVersionId); + Filter filter = new Filter(limit); + return listBy(sc, filter); + } + @Override public List listWrappedKeysForRewrap(long kmsKeyId, long excludeKekVersionId) { SearchCriteria sc = rewrapExcludeVersionSearch.create(); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index e7dde87c99f..c207006b5af 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -180,8 +180,8 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kms_wrapped_key` ( INDEX `idx_kms_key_id` (`kms_key_id`, `removed`), INDEX `idx_kek_version_id` (`kek_version_id`, `removed`), INDEX `idx_zone_id` (`zone_id`, `removed`), - CONSTRAINT `fk_kms_wrapped_key__kms_key_id` FOREIGN KEY (`kms_key_id`) REFERENCES `kms_keys`(`id`) ON DELETE RESTRICT, - CONSTRAINT `fk_kms_wrapped_key__kek_version_id` FOREIGN KEY (`kek_version_id`) REFERENCES `kms_kek_versions`(`id`) ON DELETE RESTRICT, + CONSTRAINT `fk_kms_wrapped_key__kms_key_id` FOREIGN KEY (`kms_key_id`) REFERENCES `kms_keys`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_kms_wrapped_key__kek_version_id` FOREIGN KEY (`kek_version_id`) REFERENCES `kms_kek_versions`(`id`) ON DELETE CASCADE, CONSTRAINT `fk_kms_wrapped_key__zone_id` FOREIGN KEY (`zone_id`) REFERENCES `data_center`(`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='KMS wrapped encryption keys (DEKs) - references kms_keys for KEK metadata and kek_versions for specific version'; diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index d9af7af6c33..538addd334d 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -347,6 +347,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M private NetworkPermissionDao networkPermissionDao; @Inject private SslCertDao sslCertDao; + @Inject + private org.apache.cloudstack.kms.KMSManager kmsManager; private List _querySelectors; @@ -1247,6 +1249,17 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M // Delete Webhooks deleteWebhooksForAccount(accountId); + // Delete KMS keys + try { + if (!kmsManager.deleteKMSKeysByAccountId(accountId)) { + logger.warn("Failed to delete all KMS keys for account {}", account); + accountCleanupNeeded = true; + } + } catch (Exception e) { + logger.error("Error deleting KMS keys for account {}: {}", account, e.getMessage(), e); + accountCleanupNeeded = true; + } + return true; } catch (Exception ex) { logger.warn("Failed to cleanup account " + account + " due to ", ex); diff --git a/server/src/main/java/org/apache/cloudstack/kms/KMSManagerImpl.java b/server/src/main/java/org/apache/cloudstack/kms/KMSManagerImpl.java index 4727c34ce75..fb6e1a286b4 100644 --- a/server/src/main/java/org/apache/cloudstack/kms/KMSManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/kms/KMSManagerImpl.java @@ -56,6 +56,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; +import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import javax.inject.Inject; import java.util.ArrayList; @@ -64,6 +65,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Timer; import java.util.UUID; public class KMSManagerImpl extends ManagerBase implements KMSManager, PluggableService { @@ -115,7 +117,7 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable @Override public KMSProvider getKMSProviderForZone(Long zoneId) throws KMSException { // For now, use global provider - // In future, could support zone-specific providers via zone-scoped config + // In the future, could support zone-specific providers via zone-scoped config return getConfiguredKmsProvider(); } @@ -127,84 +129,10 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable return KMSEnabled.valueIn(zoneId); } - @Override - @ActionEvent(eventType = EventTypes.EVENT_KMS_KEK_CREATE, eventDescription = "creating KEK", async = false) - public String createKek(Long zoneId, KeyPurpose purpose, String label, int keyBits) throws KMSException { - validateKmsEnabled(zoneId); - - KMSProvider provider = getKMSProviderForZone(zoneId); - - try { - logger.info("Creating KEK for zone {} with purpose {} and {} bits", zoneId, purpose, keyBits); - return retryOperation(() -> provider.createKek(purpose, label, keyBits)); - } catch (Exception e) { - logger.error("Failed to create KEK for zone {}: {}", zoneId, e.getMessage()); - throw handleKmsException(e); - } - } - - // ==================== KEK Management ==================== - - @Override - @ActionEvent(eventType = EventTypes.EVENT_KMS_KEK_DELETE, eventDescription = "deleting KEK", async = false) - public void deleteKek(Long zoneId, String kekId) throws KMSException { - validateKmsEnabled(zoneId); - - KMSProvider provider = getKMSProviderForZone(zoneId); - - // Check if any wrapped keys use this KEK - KMSKeyVO key = kmsKeyDao.findByKekLabel(kekId, provider.getProviderName()); - if (key != null) { - long wrappedKeyCount = kmsKeyDao.countWrappedKeysByKmsKey(key.getId()); - if (wrappedKeyCount > 0) { - throw KMSException.invalidParameter("Cannot delete KEK: " + wrappedKeyCount + - " wrapped key(s) still reference the corresponding KMS key"); - } - } - - try { - logger.warn("Deleting KEK {} for zone {}", kekId, zoneId); - retryOperation(() -> { - provider.deleteKek(kekId); - return null; - }); - } catch (Exception e) { - logger.error("Failed to delete KEK {} for zone {}: {}", kekId, zoneId, e.getMessage()); - throw handleKmsException(e); - } - } - - @Override - public List listKeks(Long zoneId, KeyPurpose purpose) throws KMSException { - validateKmsEnabled(zoneId); - - KMSProvider provider = getKMSProviderForZone(zoneId); - - try { - return retryOperation(() -> provider.listKeks(purpose)); - } catch (Exception e) { - logger.error("Failed to list KEKs for zone {}: {}", zoneId, e.getMessage()); - throw handleKmsException(e); - } - } - - @Override - public boolean isKekAvailable(Long zoneId, String kekId) throws KMSException { - if (!isKmsEnabled(zoneId)) { - return false; - } - - try { - KMSProvider provider = getKMSProviderForZone(zoneId); - return provider.isKekAvailable(kekId); - } catch (Exception e) { - logger.warn("Error checking KEK availability: {}", e.getMessage()); - return false; - } - } - - @Override - public String rotateKek(Long zoneId, KeyPurpose purpose, String oldKekLabel, + /** + * Internal method to rotate a KEK (create new version and update KMS key state) + */ + private String rotateKek(Long zoneId, KeyPurpose purpose, String oldKekLabel, String newKekLabel, int keyBits) throws KMSException { validateKmsEnabled(zoneId); @@ -230,10 +158,11 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable } // Create new KEK in provider - String newKekId = provider.createKek(purpose, newKekLabel, keyBits); + String finalNewKekLabel = newKekLabel; + String newKekId = retryOperation(() -> provider.createKek(purpose, finalNewKekLabel, keyBits)); // Create new KEK version (marks old as Previous, new as Active) - KMSKekVersionVO newVersion = createKekVersion(kmsKey.getId(), newKekId, keyBits); + KMSKekVersionVO newVersion = createKekVersion(kmsKey.getId(), newKekId); logger.info("KEK rotation: KMS key {} now has {} versions (active: v{}, previous: v{})", kmsKey, newVersion.getVersionNumber(), newVersion.getVersionNumber(), @@ -252,7 +181,7 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable // ==================== DEK Operations ==================== @Override - @ActionEvent(eventType = EventTypes.EVENT_KMS_KEY_UNWRAP, eventDescription = "unwrapping volume key", async = false) + @ActionEvent(eventType = EventTypes.EVENT_KMS_KEY_UNWRAP, eventDescription = "unwrapping volume key") public byte[] unwrapVolumeKey(WrappedKey wrappedKey, Long zoneId) throws KMSException { validateKmsEnabled(zoneId); @@ -270,24 +199,7 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable } @Override - @ActionEvent(eventType = EventTypes.EVENT_KMS_HEALTH_CHECK, eventDescription = "KMS health check", async = false) - public boolean healthCheck(Long zoneId) throws KMSException { - if (!isKmsEnabled(zoneId)) { - logger.debug("KMS is not enabled for zone {}", zoneId); - return false; - } - - try { - KMSProvider provider = getKMSProviderForZone(zoneId); - return provider.healthCheck(); - } catch (Exception e) { - logger.error("Health check failed for zone {}: {}", zoneId, e.getMessage()); - throw handleKmsException(e); - } - } - - @Override - @ActionEvent(eventType = EventTypes.EVENT_KMS_KEK_CREATE, eventDescription = "creating user KMS key", async = false) + @ActionEvent(eventType = EventTypes.EVENT_KMS_KEK_CREATE, eventDescription = "creating user KMS key") public KMSKey createUserKMSKey(Long accountId, Long domainId, Long zoneId, String name, String description, KeyPurpose purpose, Integer keyBits) throws KMSException { @@ -328,19 +240,6 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable return kmsKeyDao.listAccessibleKeys(accountId, domainId, zoneId, purpose, state); } - @Override - public KMSKey getUserKMSKey(String uuid, Long callerAccountId) { - KMSKeyVO key = kmsKeyDao.findByUuid(uuid); - if (key == null || key.getState() == KMSKey.State.Deleted) { - return null; - } - - if (!hasPermission(callerAccountId, key)) { - return null; - } - return key; - } - @Override public boolean hasPermission(Long callerAccountId, KMSKey key) { if (key == null || key.getState() == KMSKey.State.Deleted) { @@ -478,7 +377,7 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable @Override @ActionEvent(eventType = EventTypes.EVENT_KMS_KEY_WRAP, - eventDescription = "generating volume key with specified KEK", async = false) + eventDescription = "generating volume key with specified KEK") public WrappedKey generateVolumeKeyWithKek(KMSKey kmsKey, Long callerAccountId) throws KMSException { // Get and validate KMS key if (kmsKey == null) { @@ -700,7 +599,7 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable /** * Create a new KEK version for a KMS key */ - private KMSKekVersionVO createKekVersion(Long kmsKeyId, String kekLabel, int keyBits) throws KMSException { + private KMSKekVersionVO createKekVersion(Long kmsKeyId, String kekLabel) throws KMSException { // Get existing versions to determine next version number List existingVersions = kmsKekVersionDao.listByKmsKeyId(kmsKeyId); int nextVersion = existingVersions.stream() @@ -754,104 +653,49 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable KMSKekVersionVO newVersion = getActiveKekVersion(kmsKey.getId()); - logger.info("KMS key rotation completed: {} -> new KEK version {} (UUID: {})", - kmsKey, newVersion.getVersionNumber(), newVersion.getUuid()); + logger.info("KMS key rotation initiated: {} -> new KEK version {} (UUID: {}). " + + "Background job will gradually rewrap {} wrapped key(s)", + kmsKey, newVersion.getVersionNumber(), newVersion.getUuid(), + kmsWrappedKeyDao.countByKmsKeyId(kmsKey.getId())); - // Perform rewrapping of existing wrapped keys - // This runs within the async job context - rewrapWrappedKeysForKMSKey(kmsKey.getId(), newVersion.getId(), 50); + // Background KMSRewrapWorker will automatically detect Previous versions + // and gradually rewrap wrapped keys in batches return newVersion.getUuid(); } - @Override - public int rewrapWrappedKeysForKMSKey(Long kmsKeyId, Long newKekVersionId, int batchSize) throws KMSException { - if (kmsKeyId == null || newKekVersionId == null) { - throw KMSException.invalidParameter("kmsKeyId and newKekVersionId must be specified"); - } + /** + * Helper method to rewrap a single wrapped key with a new KEK version. + * Unwraps the key, re-wraps it with the new KEK, and updates the database. + * + * @param wrappedKeyVO the wrapped key to rewrap + * @param kmsKey the KMS key + * @param newVersion the new KEK version to wrap with + * @param provider the KMS provider + */ + private void rewrapSingleKey(KMSWrappedKeyVO wrappedKeyVO, KMSKeyVO kmsKey, + KMSKekVersionVO newVersion, KMSProvider provider) { + byte[] dek = null; + try { + // Unwrap with current/old version + dek = unwrapKey(wrappedKeyVO.getId()); - if (batchSize <= 0) { - batchSize = 50; // Default batch size - } + // Wrap the existing DEK with new KEK version + WrappedKey newWrapped = provider.wrapKey( + dek, + kmsKey.getPurpose(), + newVersion.getKekLabel() + ); - // Get KMS key and new version - KMSKeyVO kmsKey = kmsKeyDao.findById(kmsKeyId); - if (kmsKey == null) { - throw KMSException.kekNotFound("KMS key not found: " + kmsKeyId); - } - - KMSKekVersionVO newVersion = kmsKekVersionDao.findById(newKekVersionId); - if (newVersion == null || !newVersion.getKmsKeyId().equals(kmsKeyId)) { - throw KMSException.kekNotFound("KEK version not found or doesn't belong to KMS key: " + newKekVersionId); - } - - KMSProvider provider = getKMSProviderForZone(kmsKey.getZoneId()); - - // Get all wrapped keys that need rewrap - List wrappedKeys = kmsWrappedKeyDao.listWrappedKeysForRewrap(kmsKeyId, newKekVersionId); - int totalKeys = wrappedKeys.size(); - int successCount = 0; - int failureCount = 0; - - logger.info("Starting rewrap operation for {} wrapped keys (KMS key: {}, new version: {})", - totalKeys, kmsKey, newKekVersionId); - - for (int i = 0; i < wrappedKeys.size(); i += batchSize) { - int endIndex = Math.min(i + batchSize, wrappedKeys.size()); - List batch = wrappedKeys.subList(i, endIndex); - - for (KMSWrappedKeyVO wrappedKeyVO : batch) { - byte[] dek = null; - try { - // Unwrap with old version - dek = unwrapKey(wrappedKeyVO.getId()); - - // Wrap the existing DEK with new active version - WrappedKey newWrapped = provider.wrapKey( - dek, - kmsKey.getPurpose(), - newVersion.getKekLabel() - ); - - wrappedKeyVO.setKekVersionId(newKekVersionId); - wrappedKeyVO.setWrappedBlob(newWrapped.getWrappedKeyMaterial()); - kmsWrappedKeyDao.update(wrappedKeyVO.getId(), wrappedKeyVO); - - successCount++; - logger.debug("Rewrapped key {} (batch {}/{})", wrappedKeyVO.getId(), - (i / batchSize) + 1, (totalKeys + batchSize - 1) / batchSize); - } catch (Exception e) { - failureCount++; - logger.warn("Failed to rewrap key {}: {}", wrappedKeyVO.getId(), e.getMessage()); - } finally { - // Zeroize DEK - if (dek != null) { - Arrays.fill(dek, (byte) 0); - } - } - } - - logger.info("Processed batch {}/{}: {} success, {} failures", - (i / batchSize) + 1, (totalKeys + batchSize - 1) / batchSize, successCount, failureCount); - } - - // Archive old versions if no wrapped keys reference them - List oldVersions = kmsKekVersionDao.getVersionsForDecryption(kmsKeyId); - for (KMSKekVersionVO oldVersion : oldVersions) { - if (oldVersion.getStatus() == KMSKekVersionVO.Status.Previous) { - List keysUsingVersion = kmsWrappedKeyDao.listByKekVersionId(oldVersion.getId()); - if (keysUsingVersion.isEmpty()) { - oldVersion.setStatus(KMSKekVersionVO.Status.Archived); - kmsKekVersionDao.update(oldVersion.getId(), oldVersion); - logger.info("Archived KEK version {} (no wrapped keys using it)", oldVersion.getVersionNumber()); - } + wrappedKeyVO.setKekVersionId(newVersion.getId()); + wrappedKeyVO.setWrappedBlob(newWrapped.getWrappedKeyMaterial()); + kmsWrappedKeyDao.update(wrappedKeyVO.getId(), wrappedKeyVO); + } finally { + // Always zeroize DEK from memory + if (dek != null) { + Arrays.fill(dek, (byte) 0); } } - - logger.info("Rewrap operation completed: {} success, {} failures out of {} total", - successCount, failureCount, totalKeys); - - return successCount; } @Override @@ -909,7 +753,7 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable byte[] passphraseBytes = passphrase.getPassphrase(); // Get or create KMS key for account - KMSKeyVO kmsKey = null; + KMSKeyVO kmsKey; List accountKeys = listUserKMSKeys( volume.getAccountId(), volume.getDomainId(), @@ -960,7 +804,7 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable volume.setPassphraseId(null); // Clear passphrase reference volumeDao.update(volume.getId(), volume); - // Zeroize passphrase bytes + // zeroize passphrase bytes if (passphraseBytes != null) { Arrays.fill(passphraseBytes, (byte) 0); } @@ -1122,9 +966,193 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable logger.warn("KMS provider health check error: {}", e.getMessage()); } + // Schedule background rewrap worker + scheduleRewrapWorker(); + return true; } + /** + * Schedule the background KEK rewrap worker + */ + private void scheduleRewrapWorker() { + final ManagedContextTimerTask rewrapTask = new ManagedContextTimerTask() { + @Override + protected void runInContext() { + try { + processRewrapBatch(); + } catch (final Exception e) { + logger.error("Error while running KMS rewrap worker", e); + } + } + }; + + long intervalMs = KMSRewrapIntervalMs.value(); + Timer rewrapTimer = new Timer("KMSRewrapWorker"); + rewrapTimer.schedule(rewrapTask, 10000L, intervalMs); // Start after 10 seconds, run at configured interval + logger.info("KMS rewrap worker scheduled with interval: {} ms", intervalMs); + } + + /** + * Background worker method that processes KEK rewrap batches. + * Finds KEK versions marked as Previous and gradually rewraps wrapped keys + * using the active version. + */ + private void processRewrapBatch() { + try { + // Find all KEK versions marked as Previous (rotation in progress) + List previousVersions = kmsKekVersionDao.findByStatus(KMSKekVersionVO.Status.Previous); + + if (previousVersions.isEmpty()) { + logger.trace("No KEK versions pending rewrap"); + return; + } + + logger.debug("Found {} KEK version(s) with status Previous - processing rewrap batches", previousVersions.size()); + + int batchSize = KMSRewrapBatchSize.value(); + + for (KMSKekVersionVO oldVersion : previousVersions) { + try { + processVersionRewrap(oldVersion, batchSize); + } catch (Exception e) { + logger.error("Error processing rewrap for KEK version {}: {}", oldVersion, e.getMessage(), e); + // Continue with next version + } + } + } catch (Exception e) { + logger.error("Error in rewrap worker: {}", e.getMessage(), e); + } + } + + /** + * Process rewrap for a single KEK version (used by background worker) + */ + private void processVersionRewrap(KMSKekVersionVO oldVersion, int batchSize) throws KMSException { + KMSKeyVO kmsKey = kmsKeyDao.findById(oldVersion.getKmsKeyId()); + if (kmsKey == null) { + logger.warn("KMS key not found for KEK version {}, skipping", oldVersion); + return; + } + + // Get active version for this KMS key + KMSKekVersionVO activeVersion = kmsKekVersionDao.getActiveVersion(oldVersion.getKmsKeyId()); + if (activeVersion == null) { + logger.warn("No active KEK version found for KMS key {}, skipping", kmsKey); + return; + } + + // Query wrapped keys still using the old version (limited to batch size) + List keysToRewrap = kmsWrappedKeyDao.listByKekVersionId(oldVersion.getId(), batchSize); + + if (keysToRewrap.isEmpty()) { + // All keys rewrapped - archive the old version + logger.info("All wrapped keys rewrapped for KEK version {} (v{}) - archiving", + oldVersion.getUuid(), oldVersion.getVersionNumber()); + + oldVersion.setStatus(KMSKekVersionVO.Status.Archived); + kmsKekVersionDao.update(oldVersion.getId(), oldVersion); + + return; + } + + // Get provider + KMSProvider provider = getKMSProviderForZone(kmsKey.getZoneId()); + + // Rewrap this batch using the common helper + int successCount = 0; + int failureCount = 0; + + for (KMSWrappedKeyVO wrappedKeyVO : keysToRewrap) { + try { + rewrapSingleKey(wrappedKeyVO, kmsKey, activeVersion, provider); + successCount++; + } catch (Exception e) { + failureCount++; + logger.warn("Failed to rewrap key {} for KMS key {}: {}", + wrappedKeyVO.getId(), kmsKey, e.getMessage()); + // Continue with next key - will retry in next run + } + } + + logger.info("Rewrapped batch for KMS key {} (KEK v{} -> v{}): {} success, {} failures", + kmsKey, oldVersion.getVersionNumber(), activeVersion.getVersionNumber(), + successCount, failureCount); + } + + @Override + public boolean deleteKMSKeysByAccountId(Long accountId) { + if (accountId == null) { + logger.warn("Cannot delete KMS keys: account ID is null"); + return false; + } + + try { + // List all KMS keys owned by this account + List accountKeys = kmsKeyDao.listByAccount(accountId, null, null); + + if (accountKeys == null || accountKeys.isEmpty()) { + logger.debug("No KMS keys found for account {}", accountId); + return true; + } + + logger.info("Deleting {} KMS key(s) for account {}", accountKeys.size(), accountId); + + boolean allDeleted = true; + for (KMSKeyVO key : accountKeys) { + try { + KMSProvider provider = getKMSProviderForZone(key.getZoneId()); + + // Step 1: Delete all KEKs from the provider first + List kekVersions = kmsKekVersionDao.listByKmsKeyId(key.getId()); + if (kekVersions != null && !kekVersions.isEmpty()) { + logger.debug("Deleting {} KEK version(s) from provider for KMS key {}", + kekVersions.size(), key.getUuid()); + for (KMSKekVersionVO kekVersion : kekVersions) { + try { + provider.deleteKek(kekVersion.getKekLabel()); + logger.debug("Deleted KEK {} (v{}) from provider", + kekVersion.getKekLabel(), kekVersion.getVersionNumber()); + } catch (Exception e) { + logger.warn("Failed to delete KEK {} from provider: {}", + kekVersion.getKekLabel(), e.getMessage()); + // Continue - still delete from database even if provider deletion fails + } + } + } + + // Step 2: Delete the KMS key from database + // This will CASCADE delete: + // - KEK versions (kms_kek_versions) + // - Wrapped keys (kms_wrapped_key) + boolean deleted = kmsKeyDao.remove(key.getId()); + if (deleted) { + logger.debug("Deleted KMS key {} as part of account {} cleanup", key.getUuid(), accountId); + } else { + logger.warn("Failed to delete KMS key {} as part of account {} cleanup", + key.getUuid(), accountId); + allDeleted = false; + } + } catch (Exception e) { + logger.error("Error deleting KMS key {} for account {}: {}", + key.getUuid(), accountId, e.getMessage(), e); + allDeleted = false; + } + } + + if (allDeleted) { + logger.info("Successfully deleted all KMS keys for account {}", accountId); + } else { + logger.warn("Some KMS keys for account {} could not be deleted", accountId); + } + + return allDeleted; + } catch (Exception e) { + logger.error("Error during KMS key cleanup for account {}: {}", accountId, e.getMessage(), e); + return false; + } + } + @Override public String getConfigComponentName() { return KMSManager.class.getSimpleName(); @@ -1138,7 +1166,9 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable KMSDekSizeBits, KMSRetryCount, KMSRetryDelayMs, - KMSOperationTimeoutSec + KMSOperationTimeoutSec, + KMSRewrapBatchSize, + KMSRewrapIntervalMs }; }