Multisite

Diagram

Classes

vxlan

Name	Type	Constraint	Mandatory	Default Value
multisite	Class	[multisite]	No

multisite (vxlan)

Name	Type	Constraint	Mandatory	Default Value
child_fabrics	List	[child_fabrics]	No
overlay	Class	[overlay]	No
anycast_gateway_mac	Any	String[Regex: ^[a-f0-9]{1}.[a-f0-9]{1}.[a-f0-9]{1}$] or String[Regex: ^[a-f0-9]{4}.[a-f0-9]{4}.[a-f0-9]{4}$] or String[Regex: ^[a-f0-9]{2}:[a-f0-9]{2}:[a-f0-9]{2}:[a-f0-9]{2}:[a-f0-9]{2}:[a-f0-9]{2}$] or String[Regex: ^[a-f0-9]{2}-[a-f0-9]{2}-[a-f0-9]{2}-[a-f0-9]{2}-[a-f0-9]{2}-[a-f0-9]{2}$]	No	20:20:00:00:00:aa
vtep_loopback_id	Integer	min: 0, max: 1023	No	100
bgw_ip_tag	String		No	54321
overlay_dci	Class	[overlay_dci]	No
ipv4_vtep_loopback_range	IP		No	10.10.0.0/24
isn	Class	[isn]	No

child_fabrics (vxlan.multisite)

Name	Type	Constraint	Mandatory	Default Value
name	String		Yes
cluster	String		No

overlay (vxlan.multisite)

Name	Type	Constraint	Mandatory	Default Value
vrfs	List	[vrfs]	No
networks	List	[networks]	No
vrf_attach_groups	List	[vrf_attach_groups]	No
network_attach_groups	List	[network_attach_groups]	No

overlay_dci (vxlan.multisite)

Name	Type	Constraint	Mandatory	Default Value
deployment_method	Choice	Manual, Centralized_To_Route_Server, Direct_To_BGWS	No	Direct_To_BGWS
ipv4_dci_subnet_range	IP		No	10.10.1.0/24
ipv4_dci_subnet_mask	Integer	min: 8, max: 31	No	30
route_server	Class	[route_server]	No
underlay_autoconfig	Boolean	true, false	No	true
enable_bgp_send_community	Boolean	true, false	No	false
enable_bgp_log_neighbor_change	Boolean	true, false	No	false
enable_bgp_bfd	Boolean	true, false	No	false
delay_restore	Integer	min: 30, max: 1000	No	300
enable_ebgp_password	Boolean	true, false	No	false
ebgp_password	String		No
ebgp_password_encryption_type	Choice	3, 7	No
enable_trm	Boolean	true, false	No	false

isn (vxlan.multisite)

Name	Type	Constraint	Mandatory	Default Value
bgp_asn	String	Regex: ^(?:\d{1,10}|\d{1,5}\.\d{1,5})$	Yes
sub_int_range	String		No	2-511
enable_nxapi_http	Boolean	true, false	No
nxapi_http_port	Integer		No
enable_nxapi_https	Boolean	true, false	No
nxapi_https_port	Integer		No
bootstrap	Class	[bootstrap]	No

vrfs (vxlan.multisite.overlay)

Name	Type	Constraint	Mandatory	Default Value
name	String		Yes
vrf_id	Integer	min: 4096, max: 1677215	No
vlan_id	Integer	min: 1, max: 4094	No
vrf_vlan_name	String		No
vrf_intf_desc	String		No
vrf_description	String		No
vrf_int_mtu	Integer	min: 1500, max: 9216	No
loopback_route_tag	Integer	min: 0, max: 4294967295	No
max_bgp_paths	Integer	min: 1, max: 128	No
max_ibgp_paths	Integer	min: 1, max: 128	No
ipv6_linklocal_enable	Boolean	true, false	No
adv_host_routes	Boolean	true, false	No
adv_default_routes	Boolean	true, false	No
static_default_route	Boolean	true, false	No
bgp_password	String		No
bgp_password_encryption_type	Choice	3, 7	No
disable_rt_auto	Boolean	true, false	No
export_evpn_rt	String		No
export_mvpn_rt	String		No
export_vpn_rt	String		No
import_evpn_rt	String		No
import_mvpn_rt	String		No
import_vpn_rt	String		No
netflow_enable	Boolean	true, false	No
netflow_monitor	String		No
no_rp	Boolean	true, false	No
overlay_multicast_group	String		No
redist_direct_routemap	String		No
rp_address	IP		No
rp_external	Boolean	true, false	No
rp_loopback_id	Integer	min: 0, max: 1023	No
trm_enable	Boolean	true, false	No
trm_bgw_msite	Boolean	true, false	No
underlay_mcast_ip	IP		No
child_fabrics	List	[child_fabrics]	No
vrf_attach_group	String		No

networks (vxlan.multisite.overlay)

Name	Type	Constraint	Mandatory	Default Value
name	String		Yes
is_l2_only	Boolean	true, false	No
vrf_name	String		No
net_id	Integer	min: 4096, max: 1677215	No
vlan_id	Integer	min: 1, max: 4094	No
vlan_name	String		No
gw_ip_address	IP		No
arp_suppress	Boolean	true, false	No
dhcp_loopback_id	Integer	min: 0, max: 1023	No
dhcp_servers	List	[dhcp_servers]	No
gw_ipv6_address	String		No
int_desc	String		No
l3gw_on_border	Boolean	true, false	No
mtu_l3intf	Integer		No
multicast_group_address	IP		No
netflow_enable	Boolean	true, false	No
route_target_both	Boolean	true, false	No
route_tag	Integer	min: 0, max: 4294967295	No
secondary_ip_addresses	List	[secondary_ip_addresses]	No
trm_enable	Boolean	true, false	No
vlan_netflow_monitor	String		No
child_fabrics	List	[child_fabrics]	No
network_attach_group	String		No

vrf_attach_groups (vxlan.multisite.overlay)

Name	Type	Constraint	Mandatory	Default Value
name	String		Yes
switches	List	[switches]	No

network_attach_groups (vxlan.multisite.overlay)

Name	Type	Constraint	Mandatory	Default Value
name	String		Yes
switches	List	[switches]	No

route_server (vxlan.multisite.overlay_dci)

Name	Type	Constraint	Mandatory	Default Value
peers	List	[peers]	No
redistribute_direct	Boolean	true, false	No	false
ip_tag	Integer	min: 0, max: 4294967295	No	54321

bootstrap (vxlan.multisite.isn)

Name	Type	Constraint	Mandatory	Default Value
enable_bootstrap	Boolean	true, false	No
enable_local_dhcp_server	Boolean	true, false	No
dhcp_version	Choice	DHCPv4, DHCPv6	No
dhcp_v4	Class	[dhcp_v4]	No
dhcp_v6	Class	[dhcp_v6]	No
enable_cdp_mgmt	Boolean	true, false	No

child_fabrics (vxlan.multisite.overlay.vrfs)

Name	Type	Constraint	Mandatory	Default Value
name	String		Yes
adv_host_routes	Boolean	true, false	No
adv_default_routes	Boolean	true, false	No
static_default_route	Boolean	true, false	No
bgp_password	String		No
bgp_password_encryption_type	Choice	3, 7	No
netflow_enable	Boolean	true, false	No
netflow_monitor	String		No
trm_enable	Boolean	true, false	No
trm_bgw_msite	Boolean	true, false	No
no_rp	Boolean	true, false	No
rp_external	Boolean	true, false	No
rp_address	IP		No
rp_loopback_id	Integer	min: 0, max: 1023	No
underlay_mcast_ip	IP		No
overlay_multicast_group	String		No
import_mvpn_rt	String		No
export_mvpn_rt	String		No

dhcp_servers (vxlan.multisite.overlay.networks)

Name	Type	Constraint	Mandatory	Default Value
ip_address	IP		Yes
vrf	String		Yes

secondary_ip_addresses (vxlan.multisite.overlay.networks)

Name	Type	Constraint	Mandatory	Default Value
ip_address	IP		Yes
route_tag	Integer	min: 0, max: 4294967295	No

child_fabrics (vxlan.multisite.overlay.networks)

Name	Type	Constraint	Mandatory	Default Value
name	String		Yes
dhcp_loopback_id	Integer	min: 0, max: 1023	No
dhcp_servers	List	[dhcp_servers]	No
multicast_group_address	IP		No
trm_enable	Boolean	true, false	No
netflow_enable	Boolean	true, false	No
vlan_netflow_monitor	String		No
l3gw_on_border	Boolean	true, false	No

switches (vxlan.multisite.overlay.vrf_attach_groups)

Name	Type	Constraint	Mandatory	Default Value
hostname	String		Yes
loopback_id	Integer	min: 0, max: 1023	No
loopback_ipv4	IP		No
loopback_ipv6	IP		No

switches (vxlan.multisite.overlay.network_attach_groups)

Name	Type	Constraint	Mandatory	Default Value
hostname	String		Yes
ports	List	String[Regex: (?i)^(?:e|eth(?:ernet)?)((\d)/\d{1,3})$|^(?:po|port-channel)([1-9]|[1-9][0-9]{1,2}|[1-3][0-9]{3}|40[0-8][0-9]|409[0-6])$]	No
tors	List	[tors]	No

peers (vxlan.multisite.overlay_dci.route_server)

Name	Type	Constraint	Mandatory	Default Value
ip_address	IP		Yes
bgp_asn	String	Regex: ^(?:\d{1,10}|\d{1,5}\.\d{1,5})$	Yes

dhcp_v4 (vxlan.multisite.isn.bootstrap)

Name	Type	Constraint	Mandatory	Default Value
scope_start_address	IP		Yes
scope_end_address	IP		Yes
switch_mgmt_default_gw	IP		Yes
mgmt_prefix	Integer	min: 8, max: 30	Yes
multi_subnet_scope	String		No
domain_name	String		No

dhcp_v6 (vxlan.multisite.isn.bootstrap)

Name	Type	Constraint	Mandatory	Default Value
scope_start_address	IP		Yes
scope_end_address	IP		Yes
switch_mgmt_default_gw	IP		Yes
mgmt_prefix	Integer	min: 64, max: 126	Yes
multi_subnet_scope	String		No
domain_name	String		No

tors (vxlan.multisite.overlay.network_attach_groups.switches)

Name	Type	Constraint	Mandatory	Default Value
hostname	String		Yes
ports	List	String[Regex: (?i)^(?:e|eth(?:ernet)?)((\d)/\d{1,3})$|^(?:po|port-channel)([1-9]|[1-9][0-9]{1,2}|[1-3][0-9]{3}|40[0-8][0-9]|409[0-6])$]	No

Workflow

To manage an MSD Fabric, you need to follow this process:

Step 1 - Create your child fabrics. In this example, we have four (4) childs fabrics:

• nac-ndfc1
• nac-ndfc2
• nac-ndfc3
• nac-isn

Each fabric has their own data source under host_vars.

host_vars directories

❯ tree -L1  host_vars
host_vars
├── nac-isn
├── nac-msd
├── nac-ndfc1
├── nac-ndfc2
└── nac-ndfc3

inventory.yaml

❯ cat inventory.yaml
---
all:
  children:
    ndfc:
      hosts:
        nac-isn:
          ansible_host: 10.x.x.x
        nac-msd:
          ansible_host: 10.x.x.x
        nac-ndfc1:
          ansible_host: 10.x.x.x
        nac-ndfc2:
          ansible_host: 10.x.x.x
        nac-ndfc3:
          ansible_host: 10.x.x.x

tip

If your fabric uses a different controller, you can provide a different IP address. Each folder will contain their own YAML data files to describe each respective fabric.

Step 2 - Create the MSD fabric, which includes child fabrics.

MSD Example

In this example, we have a fabric named MSD. In this MSD fabric, all fabrics use the same anycast gateway mac: DE:AD:BE:EF:FE:ED. The loopback ID used for the Anycast IP configured on each Border Gateway is: 100 and the subnet used to allocate an IP is: 10.10.0.0/24.

This configuration uses external ISN devices as the Route Server. In this configuration there are four (4) IP addresses:

• 100.64.0.1
• 100.64.0.2
• 100.64.0.3
• 100.64.0.4

Each IP will be associated to an Autonomous System (ASN), which can be different for each Route Server. In this example, all Route Servers will use the ASN 65000.100. When underlay_autoconfig is true, you need to provide a subnet for the DCI point-to-point (p2p) connection(s). Here we used the subnet 10.10.1.0/24. Each p2p will used CIDR /30.

warning

The value in bgp_asn should between double quotes. Example:

global:
    bgp_asn: "6512"

global:
  bgp_asn: "65000.1"

global.nac.yaml

---
vxlan:
  fabric:
    name: MSD
    type: MSD
  multisite:
    child_fabrics:
      - name: nac-ndfc1
      - name: nac-ndfc2
      - name: nac-ndfc3
      - name: nac-isn
    anycast_gateway_mac: de:ad:be:ef:fe:ed
    bgw_ip_tag: "54321"  # Use quotes
    vtep_loopback_id: 100
    ipv4_vtep_loopback_range: 10.10.0.0/24
    overlay_dci:
      underlay_autoconfig: true
      deployment_method: 'Centralized_To_Route_Server'
      ipv4_dci_subnet_range: 10.10.1.0/24
      ipv4_dci_subnet_mask: 30
      route_server:
        peers:
          - ip_address: 100.64.0.1
            bgp_asn: "65000.100"  # Use quotes
          - ip_address: 100.64.0.2
            bgp_asn: "65000.100"  # Use quotes
          - ip_address: 100.64.0.3
            bgp_asn: "65000.100"  # Use quotes
          - ip_address: 100.64.0.4
            bgp_asn: "65000.100"  # Use quotes
        redistribute_direct: false
        ip_tag: 54321
      enable_bgp_bfd: false
      enable_bgp_log_neighbor_change: true
      enable_bgp_send_community: true
      enable_ebgp_password: false
      enable_trm: false
      delay_restore: 300

ISN Example

This fabric named nac-isn, uses a fabric type: ISN. Here, we must configure an ASN. In addition we can add a range of vlan for sub-interface.

global.nac.yaml

---
vxlan:
  fabric:
    name: nac-isn
    type: ISN
  multisite:
   isn:
    bgp_asn: "65000.100"    # Use quotes
    sub_int_range: 2-511

note

With MSD, VRFs and Networks should be configured under the multisite section of the data model. In MSD, we have a new key: multisite for the overlay.

Examples:

vrfs.nac.yaml

vxlan:
  multisite:
    overlay:
      vrfs:
        # simple example
        - vrf_name: NetAsCodeVrf1
          vrf_id: 150001
          vlan_id: 2001
          attach_group: NetAsCodeVrf1_AttachGroup
      vrf_attach_groups:
        - name: NetAsCodeVrf1_AttachGroup
          switches:
            - hostname: dc1-leaf1
            - hostname: dc1-leaf2

networks.nac.yaml

vxlan:
  multisite:
    overlay:
      networks:
          # simple example
        - name: NetworkDMZ
          is_l2_only: true
          net_id: 33100
          vlan_id: 3100
          attach_group: dmz
      network_attach_groups:
        - name: dmz
          switches:
            - hostname: dc1-leaf1
              ports:
                - e1/23
                - e1/24
            - hostname: dc1-leaf2
              ports:
                - e1/23
                - e1/24