mirror of https://github.com/apache/cloudstack.git
NE: VIF binding hooks for OVS-backed extensions
CloudStack's existing OvsVifDriver already binds NICs correctly when the
NicProfile's BroadcastDomainType is Lswitch: it emits libvirt
<virtualport type='openvswitch' interfaceid='<nic.getUuid()>'/> and
libvirt sets external_ids:iface-id atomically with tap creation. No
agent patch is required for OVS-backed extensions to consume this path
-- they just need (a) a way to opt in, and (b) nic.getUuid() carried in
per-NIC script commands so the SDN-side port identifier can match.
Add the framework hooks to enable this without any KVM agent change:
* ExtensionHelper.VIF_BINDING_DETAIL_KEY ("vif.binding") -- new
top-level extension detail. Currently supported value: "lswitch".
* NetworkExtensionElement.prepare(...) -- when the extension owning the
NIC's network declares vif.binding=lswitch in its extension_details,
override nic.setBroadcastType(Networks.BroadcastDomainType.Lswitch).
OvsVifDriver on the KVM agent then picks the existing Lswitch path
unchanged. Without the opt-in, the previous default (typically Vlan)
is preserved -- existing reference extensions like network-namespace
are unaffected.
* NetworkExtensionElement.getNicUuidArgs(network, nic) -- helper that
returns ["--nic-uuid", "<uuid>"] only when vif.binding=lswitch is
declared. Wired into add-dhcp-entry, remove-dhcp-entry,
add-dns-entry, save-vm-data, save-password, save-userdata,
save-sshkey, and save-hypervisor-hostname. Extensions that do not
declare the hint never see --nic-uuid, so backwards-compatible.
* README -- new section "VIF Binding for OVS-backed Extensions"
documenting the contract end-to-end: cmk createExtension snippet,
what prepare() does, how --nic-uuid flows, why the extension never
writes iface-id remotely on the boot path. Also notes the new
argument in the add-dhcp-entry table.
Result: an OVN extension (or any future OVS-backed extension) gets
correct VIF binding by adding a single detail key at extension creation
time. No host-side agent patch, no libvirt patch, no OVS schema
change.
This commit is contained in:
Marco Sinhoreli
Wei Zhou
parent
7f9d3e350f
commit
0edce199a0
|
|
@ -43,6 +43,30 @@ public interface ExtensionHelper {
|
|||
*/
|
||||
String NETWORK_SERVICE_CAPABILITIES_DETAIL_KEY = "network.service.capabilities";
|
||||
|
||||
/**
|
||||
* Detail key used by an OVS-backed NetworkOrchestrator extension to declare
|
||||
* how its Logical Switch Port name should be matched against the OVS
|
||||
* {@code external_ids:iface-id} written by libvirt on the hypervisor.
|
||||
*
|
||||
* <p>Currently supported value:</p>
|
||||
* <ul>
|
||||
* <li>{@code "lswitch"} — the framework sets {@code BroadcastDomainType.Lswitch}
|
||||
* on the {@link com.cloud.vm.NicProfile} during {@code prepare(...)} and
|
||||
* propagates {@code nic.getUuid()} to per-NIC script commands as
|
||||
* {@code --nic-uuid}. The extension is then expected to use that UUID as
|
||||
* the LSP name, so it matches the {@code interfaceid} that
|
||||
* {@code OvsVifDriver} emits in the libvirt {@code <virtualport>} for
|
||||
* {@code Lswitch} broadcast type.</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>If absent, the framework keeps the network's broadcast type unchanged
|
||||
* (typically {@code Vlan}) and does not propagate {@code --nic-uuid}.</p>
|
||||
*/
|
||||
String VIF_BINDING_DETAIL_KEY = "vif.binding";
|
||||
|
||||
/** Value of {@link #VIF_BINDING_DETAIL_KEY} that selects the Lswitch path. */
|
||||
String VIF_BINDING_LSWITCH = "lswitch";
|
||||
|
||||
String getExtensionScriptPath(Extension extension);
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package org.apache.cloudstack.framework.extensions.network;
|
|||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -118,6 +119,7 @@ import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationSe
|
|||
import org.apache.cloudstack.extension.Extension;
|
||||
import org.apache.cloudstack.extension.ExtensionHelper;
|
||||
import org.apache.cloudstack.extension.NetworkCustomActionProvider;
|
||||
import org.apache.cloudstack.framework.extensions.dao.ExtensionDetailsDao;
|
||||
import org.apache.cloudstack.resourcedetail.dao.VpcDetailsDao;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
|
|
@ -241,6 +243,8 @@ public class NetworkExtensionElement extends AdapterBase implements
|
|||
@Inject
|
||||
private PhysicalNetworkDao physicalNetworkDao;
|
||||
@Inject
|
||||
private ExtensionDetailsDao extensionDetailsDao;
|
||||
@Inject
|
||||
private DataCenterDao dataCenterDao;
|
||||
@Inject
|
||||
private VlanDao vlanDao;
|
||||
|
|
@ -306,6 +310,7 @@ public class NetworkExtensionElement extends AdapterBase implements
|
|||
copy.networkDetailsDao = this.networkDetailsDao;
|
||||
copy.ipAddressManager = this.ipAddressManager;
|
||||
copy.physicalNetworkDao = this.physicalNetworkDao;
|
||||
copy.extensionDetailsDao = this.extensionDetailsDao;
|
||||
copy.dataCenterDao = this.dataCenterDao;
|
||||
copy.vlanDao = this.vlanDao;
|
||||
copy.guestOSCategoryDao = this.guestOSCategoryDao;
|
||||
|
|
@ -506,12 +511,68 @@ public class NetworkExtensionElement extends AdapterBase implements
|
|||
return false;
|
||||
}
|
||||
|
||||
// VIF binding hint -- when the extension declares vif.binding=lswitch,
|
||||
// override the NicProfile's broadcast type so OvsVifDriver picks the
|
||||
// Lswitch path on the KVM agent. That path already emits libvirt
|
||||
// <virtualport type='openvswitch' interfaceid='<nic.getUuid()>'/> and
|
||||
// libvirt sets external_ids:iface-id atomically with tap creation.
|
||||
// No agent patch is required for this binding mode.
|
||||
if (isLswitchVifBinding(network)) {
|
||||
nic.setBroadcastType(Networks.BroadcastDomainType.Lswitch);
|
||||
logger.debug("prepare: applied Lswitch broadcast type to NIC {} (uuid={}) on network {} per extension vif.binding hint",
|
||||
nic.getId(), nic.getUuid(), network.getId());
|
||||
}
|
||||
|
||||
final NetworkOfferingVO offering = networkOfferingDao.findById(network.getNetworkOfferingId());
|
||||
implement(network, offering, dest, context);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns {@code true} when the extension that owns the given network
|
||||
* declares {@code vif.binding=lswitch} in its {@code extension_details}.
|
||||
* Used by {@link #prepare(Network, NicProfile, VirtualMachineProfile,
|
||||
* DeployDestination, ReservationContext)} to switch the NIC's
|
||||
* {@link Networks.BroadcastDomainType} to {@code Lswitch} so the KVM
|
||||
* agent's existing {@code OvsVifDriver} Lswitch path is exercised --
|
||||
* see the framework README for the full contract.
|
||||
*/
|
||||
private boolean isLswitchVifBinding(Network network) {
|
||||
try {
|
||||
Extension extension = resolveExtension(network);
|
||||
if (extension == null) {
|
||||
return false;
|
||||
}
|
||||
Map<String, String> details = extensionDetailsDao.listDetailsKeyPairs(extension.getId());
|
||||
if (details == null) {
|
||||
return false;
|
||||
}
|
||||
String vifBinding = details.get(ExtensionHelper.VIF_BINDING_DETAIL_KEY);
|
||||
return ExtensionHelper.VIF_BINDING_LSWITCH.equalsIgnoreCase(vifBinding);
|
||||
} catch (Exception e) {
|
||||
logger.debug("Failed to resolve vif.binding for network {}: {}", network.getId(), e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns {@code ["--nic-uuid", "<uuid>"]} when the extension prefers the
|
||||
* Lswitch VIF binding path so the script can use the same UUID as the LSP
|
||||
* name (matching the {@code interfaceid} that {@code OvsVifDriver} emits).
|
||||
* Returns an empty list when the extension does not opt in -- existing
|
||||
* extensions that derive identifiers from the MAC keep working unchanged.
|
||||
*/
|
||||
private List<String> getNicUuidArgs(Network network, NicProfile nic) {
|
||||
if (nic == null || nic.getUuid() == null || nic.getUuid().isBlank()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
if (!isLswitchVifBinding(network)) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
return List.of("--nic-uuid", nic.getUuid());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean release(Network network, NicProfile nic, VirtualMachineProfile vm,
|
||||
ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException {
|
||||
|
|
@ -1346,6 +1407,7 @@ public class NetworkExtensionElement extends AdapterBase implements
|
|||
args.add("--default-nic"); args.add(String.valueOf(nic.isDefaultNic()));
|
||||
args.add("--domain"); args.add(safeStr(network.getNetworkDomain()));
|
||||
args.add("--extension-ip"); args.add(safeStr(extensionIp));
|
||||
args.addAll(getNicUuidArgs(network, nic));
|
||||
args.addAll(getVpcIdArgs(network));
|
||||
return executeScript(network, "add-dhcp-entry", args.toArray(new String[0]));
|
||||
}
|
||||
|
|
@ -1434,6 +1496,7 @@ public class NetworkExtensionElement extends AdapterBase implements
|
|||
args.add("--mac"); args.add(safeStr(nic.getMacAddress()));
|
||||
args.add("--ip"); args.add(safeStr(nic.getIPv4Address()));
|
||||
args.add("--extension-ip"); args.add(safeStr(extensionIp));
|
||||
args.addAll(getNicUuidArgs(network, nic));
|
||||
args.addAll(getVpcIdArgs(network));
|
||||
return executeScript(network, "remove-dhcp-entry", args.toArray(new String[0]));
|
||||
}
|
||||
|
|
@ -1456,6 +1519,7 @@ public class NetworkExtensionElement extends AdapterBase implements
|
|||
args.add("--ip"); args.add(safeStr(nic.getIPv4Address()));
|
||||
args.add("--hostname"); args.add(safeStr(hostname));
|
||||
args.add("--extension-ip"); args.add(safeStr(extensionIp));
|
||||
args.addAll(getNicUuidArgs(network, nic));
|
||||
args.addAll(getVpcIdArgs(network));
|
||||
return executeScript(network, "add-dns-entry", args.toArray(new String[0]));
|
||||
}
|
||||
|
|
@ -1632,6 +1696,7 @@ public class NetworkExtensionElement extends AdapterBase implements
|
|||
args.add("--ip"); args.add(safeStr(nicIpAddress));
|
||||
args.add("--gateway"); args.add(safeStr(nic.getIPv4Gateway()));
|
||||
args.add("--extension-ip"); args.add(safeStr(ensureExtensionIp(network)));
|
||||
args.addAll(getNicUuidArgs(network, nic));
|
||||
args.addAll(getVpcIdArgs(network));
|
||||
return executeScriptWithFilePayload(network, "save-vm-data", "--vm-data-file",
|
||||
vmDataArg, args.toArray(new String[0]));
|
||||
|
|
@ -1655,6 +1720,7 @@ public class NetworkExtensionElement extends AdapterBase implements
|
|||
args.add("--gateway"); args.add(safeStr(nic.getIPv4Gateway()));
|
||||
args.add("--password"); args.add(password);
|
||||
args.add("--extension-ip"); args.add(safeStr(extensionIp));
|
||||
args.addAll(getNicUuidArgs(network, nic));
|
||||
args.addAll(getVpcIdArgs(network));
|
||||
return executeScript(network, "save-password", args.toArray(new String[0]));
|
||||
}
|
||||
|
|
@ -1681,6 +1747,7 @@ public class NetworkExtensionElement extends AdapterBase implements
|
|||
args.add("--gateway"); args.add(safeStr(nic.getIPv4Gateway()));
|
||||
args.add("--userdata"); args.add(userData);
|
||||
args.add("--extension-ip"); args.add(safeStr(extensionIp));
|
||||
args.addAll(getNicUuidArgs(network, nic));
|
||||
args.addAll(getVpcIdArgs(network));
|
||||
return executeScript(network, "save-userdata", args.toArray(new String[0]));
|
||||
}
|
||||
|
|
@ -1704,6 +1771,7 @@ public class NetworkExtensionElement extends AdapterBase implements
|
|||
args.add("--gateway"); args.add(safeStr(nic.getIPv4Gateway()));
|
||||
args.add("--sshkey"); args.add(sshKeyBase64);
|
||||
args.add("--extension-ip"); args.add(safeStr(extensionIp));
|
||||
args.addAll(getNicUuidArgs(network, nic));
|
||||
args.addAll(getVpcIdArgs(network));
|
||||
return executeScript(network, "save-sshkey", args.toArray(new String[0]));
|
||||
}
|
||||
|
|
@ -1727,6 +1795,7 @@ public class NetworkExtensionElement extends AdapterBase implements
|
|||
args.add("--gateway"); args.add(safeStr(nic.getIPv4Gateway()));
|
||||
args.add("--hypervisor-hostname"); args.add(hostname);
|
||||
args.add("--extension-ip"); args.add(safeStr(extensionIp));
|
||||
args.addAll(getNicUuidArgs(network, nic));
|
||||
args.addAll(getVpcIdArgs(network));
|
||||
return executeScript(network, "save-hypervisor-hostname", args.toArray(new String[0]));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -71,8 +71,9 @@ hosts. Use it as a working example.
|
|||
8. [Capabilities Configuration](#capabilities-configuration)
|
||||
9. [VPC Networks](#vpc-networks)
|
||||
10. [Extension IP](#extension-ip)
|
||||
11. [Exit Codes](#exit-codes)
|
||||
12. [Minimal Script Skeleton](#minimal-script-skeleton)
|
||||
11. [VIF Binding for OVS-backed Extensions](#vif-binding-for-ovs-backed-extensions)
|
||||
12. [Exit Codes](#exit-codes)
|
||||
13. [Minimal Script Skeleton](#minimal-script-skeleton)
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -636,6 +637,7 @@ network whose DHCP service is provided by this extension.
|
|||
| `--default-nic <bool>` | `true` if this is the VM's default NIC. |
|
||||
| `--domain <name>` | Network domain suffix (e.g. `cs.example.com`). |
|
||||
| `--extension-ip <ip>` | |
|
||||
| `--nic-uuid <uuid>` | (optional) Present only when the extension declared `vif.binding=lswitch`. Carries `nic.getUuid()` so the extension can use it as the SDN-side port identifier (matches `external_ids:iface-id` set by libvirt on the OVS tap). See [VIF Binding for OVS-backed Extensions](#vif-binding-for-ovs-backed-extensions). |
|
||||
| `--vpc-id <N>` | (optional) |
|
||||
|
||||
**`remove-dhcp-entry` arguments:**
|
||||
|
|
@ -1106,6 +1108,77 @@ To use this extension as a VPC provider:
|
|||
|
||||
---
|
||||
|
||||
## VIF Binding for OVS-backed Extensions
|
||||
|
||||
Extensions that drive OVS-based fabrics (OVN, NSX-OVS, …) need the OVS
|
||||
tap interface that libvirt creates for each VM NIC to carry the
|
||||
`external_ids:iface-id` value that the SDN controller expects. CloudStack
|
||||
already does the right thing for `BroadcastDomainType.Lswitch` networks:
|
||||
its `OvsVifDriver` emits
|
||||
|
||||
```xml
|
||||
<virtualport type='openvswitch'>
|
||||
<parameters interfaceid='<nic.getUuid()>'/>
|
||||
</virtualport>
|
||||
```
|
||||
|
||||
and libvirt sets `external_ids:iface-id=<nic-uuid>` on the tap atomically
|
||||
with port creation. No agent patch is required.
|
||||
|
||||
To opt into this binding mode an extension declares it as a top-level
|
||||
capability hint in its `extension_details`:
|
||||
|
||||
```bash
|
||||
cmk createExtension \
|
||||
name=my-ovs-sdn \
|
||||
type=NetworkOrchestrator \
|
||||
"details[0].key=network.services" \
|
||||
"details[0].value=Dhcp,Dns,UserData,SourceNat,…" \
|
||||
"details[1].key=network.service.capabilities" \
|
||||
"details[1].value=$(cat my-caps.json)" \
|
||||
"details[2].key=vif.binding" \
|
||||
"details[2].value=lswitch"
|
||||
```
|
||||
|
||||
When `vif.binding=lswitch` is present:
|
||||
|
||||
1. **`prepare()` overrides the NIC broadcast type.**
|
||||
`NetworkExtensionElement.prepare(...)` calls
|
||||
`nic.setBroadcastType(Networks.BroadcastDomainType.Lswitch)` so
|
||||
`OvsVifDriver` on the KVM agent picks the existing Lswitch path and
|
||||
emits the libvirt `<virtualport>` shown above.
|
||||
|
||||
2. **Per-NIC commands receive `--nic-uuid <uuid>`.**
|
||||
`add-dhcp-entry`, `remove-dhcp-entry`, `add-dns-entry`, `save-vm-data`,
|
||||
`save-password`, `save-userdata`, `save-sshkey`, and
|
||||
`save-hypervisor-hostname` all gain a `--nic-uuid <uuid>` argument
|
||||
carrying `nic.getUuid()`.
|
||||
|
||||
3. **The script must use `--nic-uuid` as the SDN-side port identifier.**
|
||||
Whatever object the extension creates on its controller (OVN
|
||||
Logical_Switch_Port, NSX logical port, …) **must be named exactly the
|
||||
value of `--nic-uuid`**. That is the string libvirt will write to
|
||||
`external_ids:iface-id` on the tap, so the SDN controller's local
|
||||
agent (e.g. `ovn-controller`) finds a match and binds the port.
|
||||
|
||||
When the extension does not declare `vif.binding`, the framework keeps
|
||||
the default `BroadcastDomainType.Vlan` and does not propagate
|
||||
`--nic-uuid` -- existing reference extensions (e.g.
|
||||
`network-namespace`) are unaffected.
|
||||
|
||||
### Why not the extension setting `iface-id` remotely?
|
||||
|
||||
The OVS tap only exists *after* libvirt creates the VM, so any remote
|
||||
write from the extension would race `ovn-controller` on the host. By
|
||||
letting libvirt do the write atomically with tap creation, the binding
|
||||
is ready by the time the controller scans the bridge.
|
||||
|
||||
The extension may still talk OVSDB to the host (read-only checks,
|
||||
`bridge-mappings` setup, post-incident repair) -- but never for the
|
||||
boot path.
|
||||
|
||||
---
|
||||
|
||||
## Exit Codes
|
||||
|
||||
| Exit code | Meaning |
|
||||
|
|
|
|||
Loading…
Reference in New Issue