Merge pull request #486 from nautobot/pvillar-aci-ssot-WIP-mzb

Fixing issues in ACI integration

changes/491.changed

@@ -0,0 +1 @@
+Fixed tenant names and introduced tag for multisite.

nautobot_ssot/integrations/aci/diffsync/adapters/aci.py

@@ -83,11 +83,16 @@ def load_tenants(self):
         for _tenant in tenant_list:
             if not _tenant["name"] in PLUGIN_CFG.get("ignore_tenants"):
                 tenant_name = f"{self.tenant_prefix}:{_tenant['name']}"
+                if ":mso" in _tenant.get("annotation").lower():  # pylint: disable=simplifiable-if-statement
+                    _msite_tag = True
+                else:
+                    _msite_tag = False
                 new_tenant = self.tenant(
                     name=tenant_name,
                     description=_tenant["description"],
                     comments=PLUGIN_CFG.get("comments", ""),
                     site_tag=self.site,
+                    msite_tag=_msite_tag,
                 )
                 self.add(new_tenant)
 
@@ -210,15 +215,19 @@ def load_ipaddresses(self):
                     vrf_tenant = f"{self.tenant_prefix}:{bd_value['vrf_tenant']}"
                 else:
                     vrf_tenant = None
+                if bd_value.get("tenant") == "mgmt":
+                    _namespace = "Global"
+                else:
+                    _namespace = vrf_tenant or tenant_name
                 for subnet in bd_value["subnets"]:
                     prefix = ip_network(subnet[0], strict=False).with_prefixlen
                     self.load_subnet_as_prefix(
                         prefix=prefix,
-                        namespace=tenant_name,
+                        namespace=_namespace,
                         site=self.site,
                         vrf=bd_value["vrf"],
                         vrf_tenant=vrf_tenant,
-                        tenant=tenant_name,
+                        tenant=vrf_tenant or tenant_name,
                     )
                     new_ipaddress = self.ip_address(
                         address=subnet[0],
@@ -227,8 +236,8 @@ def load_ipaddresses(self):
                         description=f"ACI Bridge Domain: {bd_key}",
                         device=None,
                         interface=None,
-                        tenant=tenant_name,
-                        namespace=tenant_name,
+                        tenant=vrf_tenant or tenant_name,
+                        namespace=_namespace,
                         site=self.site,
                         site_tag=self.site,
                     )
@@ -241,7 +250,7 @@ def load_ipaddresses(self):
                         self.add(new_ipaddress)
                     else:
                         self.job.logger.warning(
-                            "Duplicate DiffSync IPAddress Object found and has not been loaded.",
+                            f"Duplicate DiffSync IPAddress Object found: {new_ipaddress.address} in Tenant {new_ipaddress.tenant} and has not been loaded.",
                         )
 
     def load_prefixes(self):
@@ -255,15 +264,19 @@ def load_prefixes(self):
                     vrf_tenant = f"{self.tenant_prefix}:{bd_value['vrf_tenant']}"
                 else:
                     vrf_tenant = None
-                if tenant_name not in PLUGIN_CFG.get("ignore_tenants"):
+                if bd_value.get("tenant") == "mgmt":
+                    _namespace = "Global"
+                else:
+                    _namespace = vrf_tenant or tenant_name
+                if bd_value.get("tenant") not in PLUGIN_CFG.get("ignore_tenants"):
                     for subnet in bd_value["subnets"]:
                         new_prefix = self.prefix(
                             prefix=str(ip_network(subnet[0], strict=False)),
-                            namespace=tenant_name,
+                            namespace=_namespace,
                             status="Active",
                             site=self.site,
                             description=f"ACI Bridge Domain: {bd_key}",
-                            tenant=tenant_name,
+                            tenant=vrf_tenant or tenant_name,
                             vrf=bd_value["vrf"] if bd_value.get("vrf") != "" else None,
                             vrf_tenant=vrf_tenant,
                             site_tag=self.site,
@@ -282,7 +295,7 @@ def load_prefixes(self):
                                 self.add(new_prefix)
                             else:
                                 self.job.logger.warning(
-                                    "Duplicate DiffSync Prefix Object found and has not been loaded.",
+                                    f"Duplicate DiffSync Prefix Object found {new_prefix.prefix} in Namespace {new_prefix.namespace} and has not been loaded.",
                                 )
 
     def load_devicetypes(self):

nautobot_ssot/integrations/aci/diffsync/adapters/nautobot.py

@@ -99,7 +99,11 @@ def load_tenants(self):
         """Method to load Tenants from Nautobot."""
         for nbtenant in Tenant.objects.filter(tags=self.site_tag):
             _tenant = self.tenant(
-                name=nbtenant.name, description=nbtenant.description, comments=nbtenant.comments, site_tag=self.site
+                name=nbtenant.name,
+                description=nbtenant.description,
+                comments=nbtenant.comments,
+                site_tag=self.site,
+                msite_tag=nbtenant.tags.filter(name="ACI_MULTISITE").exists(),
             )
             self.add(_tenant)
 

nautobot_ssot/integrations/aci/diffsync/client.py

@@ -1,18 +1,25 @@
 """All interactions with ACI."""  # pylint: disable=too-many-lines, too-many-instance-attributes, too-many-arguments
 
 # pylint: disable=invalid-name
-
-import sys
 import logging
-from datetime import datetime
-from datetime import timedelta
 import re
+import sys
+from copy import deepcopy
+from datetime import datetime, timedelta
 from ipaddress import ip_network
+
 import requests
 import urllib3
 
-from .utils import tenant_from_dn, ap_from_dn, node_from_dn, pod_from_dn, fex_id_from_dn, interface_from_dn
-
+from .utils import (
+    ap_from_dn,
+    bd_from_dn,
+    fex_id_from_dn,
+    interface_from_dn,
+    node_from_dn,
+    pod_from_dn,
+    tenant_from_dn,
+)
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -132,6 +139,7 @@ def get_tenants(self) -> list:
             {
                 "name": data["fvTenant"]["attributes"]["name"],
                 "description": data["fvTenant"]["attributes"]["descr"],
+                "annotation": data["fvTenant"]["attributes"].get("annotation", ""),
             }
             for data in resp.json()["imdata"]
         ]
@@ -327,45 +335,67 @@ def get_vrfs(self, tenant: str) -> list:
         ]
         return vrf_list
 
-    def get_bds(self, tenant: str) -> dict:
+    def get_bds(self, tenant: str = "all") -> dict:
         """Return Bridge Domains and Subnets from the Cisco APIC."""
+        # TODO: rewrite using one API call -> https://<ip>/api/node/class/fvBD.json?query-target=subtree&target-subtree-class=fvBD,fvRsCtx,fvSubnet
         if tenant == "all":
-            resp = self._get("/api/node/class/fvBD.json")
+            resp = self._get(
+                "/api/node/class/fvBD.json?query-target=subtree&target-subtree-class=fvBD,fvRsCtx,fvSubnet"
+            )
         else:
-            resp = self._get(f"/api/node/mo/uni/tn-{tenant}.json?query-target=children&target-subtree-class=fvBD")
-
+            resp = self._get(
+                f"/api/node/mo/uni/tn-{tenant}.json?query-target=children&target-subtree-class=fvBD"
+            )  # test this
         bd_dict = {}
+        bd_dict_schema = {
+            "name": "",
+            "tenant": "",
+            "description": "",
+            "vrf": None,
+            "vrf_tenant": None,
+            "subnets": [],
+        }
         for data in resp.json()["imdata"]:
-            bd_dict.setdefault(data["fvBD"]["attributes"]["name"], {})
-            bd_dict[data["fvBD"]["attributes"]["name"]]["tenant"] = tenant_from_dn(data["fvBD"]["attributes"]["dn"])
-            bd_dict[data["fvBD"]["attributes"]["name"]]["description"] = data["fvBD"]["attributes"]["descr"]
-            bd_dict[data["fvBD"]["attributes"]["name"]]["unicast_routing"] = data["fvBD"]["attributes"]["unicastRoute"]
-            bd_dict[data["fvBD"]["attributes"]["name"]]["mac"] = data["fvBD"]["attributes"]["mac"]
-            bd_dict[data["fvBD"]["attributes"]["name"]]["l2unicast"] = data["fvBD"]["attributes"]["unkMacUcastAct"]
-
-        for key, value in bd_dict.items():
-            # get the containing VRF
-            resp = self._get(
-                f"/api/node/mo/uni/tn-{value['tenant']}/BD-{key}.json?query-target=children&target-subtree-class=fvRsCtx"
-            )
-            for data in resp.json()["imdata"]:
-                value["vrf"] = data["fvRsCtx"]["attributes"].get("tnFvCtxName", "default")
-                vrf_tenant = data["fvRsCtx"]["attributes"].get("tDn", None)
+            if "fvBD" in data.keys():
+                bd_tenant = tenant_from_dn(data["fvBD"]["attributes"]["dn"])
+                bd_name = data["fvBD"]["attributes"]["name"]
+                unique_name = f"{bd_name}:{bd_tenant}"
+                try:
+                    bd_dict[unique_name]
+                except KeyError:
+                    bd_dict.setdefault(unique_name, deepcopy(bd_dict_schema))
+                bd_dict[unique_name]["tenant"] = tenant_from_dn(data["fvBD"]["attributes"]["dn"])
+                bd_dict[unique_name]["name"] = data["fvBD"]["attributes"]["name"]
+                bd_dict[unique_name]["description"] = data["fvBD"]["attributes"]["descr"]
+
+            elif "fvRsCtx" in data.keys():
+                bd_tenant = tenant_from_dn(data["fvRsCtx"]["attributes"]["dn"])
+                bd_name = bd_from_dn(data["fvRsCtx"]["attributes"]["dn"])
+                unique_name = f"{bd_name}:{bd_tenant}"
+                try:
+                    bd_dict[unique_name]
+                except KeyError:
+                    bd_dict.setdefault(unique_name, deepcopy(bd_dict_schema))
+                bd_dict[unique_name]["vrf"] = data["fvRsCtx"]["attributes"].get("tnFvCtxName") or "default"
+                vrf_tenant = data["fvRsCtx"]["attributes"].get("tDn")
                 if vrf_tenant:
-                    value["vrf_tenant"] = tenant_from_dn(vrf_tenant)
-                else:
-                    value["vrf_tenant"] = None
-            # get subnets
-            resp = self._get(
-                f"/api/node/mo/uni/tn-{value['tenant']}/BD-{key}.json?query-target=children&target-subtree-class=fvSubnet"
-            )
-            subnet_list = [
-                (data["fvSubnet"]["attributes"]["ip"], data["fvSubnet"]["attributes"]["scope"])
-                for data in resp.json()["imdata"]
-            ]
-            for subnet in subnet_list:
-                value.setdefault("subnets", [])
-                value["subnets"].append(subnet)
+                    bd_dict[unique_name]["vrf_tenant"] = tenant_from_dn(vrf_tenant)
+
+            elif "fvSubnet" in data.keys():
+                bd_tenant = tenant_from_dn(data["fvSubnet"]["attributes"]["dn"])
+                bd_name = bd_from_dn(data["fvSubnet"]["attributes"]["dn"])
+                unique_name = f"{bd_name}:{bd_tenant}"
+                try:
+                    bd_dict[unique_name]
+                except KeyError:
+                    bd_dict.setdefault(unique_name, deepcopy(bd_dict_schema))
+                subnet = (data["fvSubnet"]["attributes"]["ip"], data["fvSubnet"]["attributes"]["scope"])
+                (bd_dict[unique_name]["subnets"]).append(subnet)
+            else:
+                logger.error(
+                    msg=f"Failed to load Bridge Domains data, unexpected response in {data}. Skipping Record..."
+                )
+                continue
         return bd_dict
 
     def get_nodes(self) -> dict:
@@ -394,10 +424,10 @@ def get_nodes(self) -> dict:
                 mgmt_addr = f"{node['topSystem']['attributes']['address']}/{ip_network(node['topSystem']['attributes']['tepPool'], strict=False).prefixlen}"
             else:
                 mgmt_addr = ""
-            if node["topSystem"]["attributes"]["tepPool"] != "0.0.0.0":  # nosec: B104
-                subnet = node["topSystem"]["attributes"]["tepPool"]
-            elif mgmt_addr:
+            if mgmt_addr:
                 subnet = ip_network(mgmt_addr, strict=False).with_prefixlen
+            elif node["topSystem"]["attributes"]["tepPool"] != "0.0.0.0":  # nosec: B104
+                subnet = node["topSystem"]["attributes"]["tepPool"]
             else:
                 subnet = ""
             node_id = node["topSystem"]["attributes"]["id"]
@@ -424,7 +454,7 @@ def get_nodes(self) -> dict:
         return node_dict
 
     def get_controllers(self) -> dict:
-        """Return list of Leaf/Spine nodes in the ACI fabric."""
+        """Return list of Controller nodes in the ACI fabric."""
         resp = self._get('/api/class/fabricNode.json?query-target-filter=eq(fabricNode.role,"controller")')
         node_dict = {}
         for node in resp.json()["imdata"]:
@@ -447,10 +477,10 @@ def get_controllers(self) -> dict:
                 mgmt_addr = f"{node['topSystem']['attributes']['address']}/{ip_network(node['topSystem']['attributes']['tepPool'], strict=False).prefixlen}"
             else:
                 mgmt_addr = ""
-            if node["topSystem"]["attributes"]["tepPool"] != "0.0.0.0":  # nosec: B104
-                subnet = node["topSystem"]["attributes"]["tepPool"]
-            elif mgmt_addr:
+            if mgmt_addr:
                 subnet = ip_network(mgmt_addr, strict=False).with_prefixlen
+            elif node["topSystem"]["attributes"]["tepPool"] != "0.0.0.0":  # nosec: B104
+                subnet = node["topSystem"]["attributes"]["tepPool"]
             else:
                 subnet = ""
             node_id = node["topSystem"]["attributes"]["id"]

nautobot_ssot/integrations/aci/diffsync/device-types/APIC-SERVER-L3.yaml

@@ -35,6 +35,9 @@ interfaces:
   - name: ILO
     type: 1000base-t
     mgmt_only: true
+  - name: mgmt0
+    type: 10gbase-t
+    mgmt_only: true
   - name: LAN-1
     type: 10gbase-t
     mgmt_only: true

nautobot_ssot/integrations/aci/diffsync/device-types/APIC-SERVER-M3.yaml

@@ -35,6 +35,9 @@ interfaces:
   - name: ILO
     type: 1000base-t
     mgmt_only: true
+  - name: mgmt0
+    type: 10gbase-t
+    mgmt_only: true
   - name: LAN-1
     type: 10gbase-t
     mgmt_only: true