Merge pull request #242 from nautobot/patch-fix_infoblox_subnet_filter

Fix Infoblox import_subnets setting

nautobot_ssot/integrations/infoblox/diffsync/adapters/infoblox.py

@@ -25,13 +25,13 @@ class InfobloxAdapter(DiffSync):
 
     top_level = ["vlangroup", "vlan", "prefix", "ipaddress"]
 
-    def __init__(self, *args, job=None, sync=None, conn=None, **kwargs):
+    def __init__(self, *args, job=None, sync=None, conn, **kwargs):
         """Initialize Infoblox.
 
         Args:
             job (object, optional): Infoblox job. Defaults to None.
             sync (object, optional): Infoblox DiffSync. Defaults to None.
-            conn (object, optional): InfobloxAPI connection. Defaults to None.
+            conn (object): InfobloxAPI connection.
         """
         super().__init__(*args, **kwargs)
         self.job = job
@@ -47,11 +47,25 @@ def __init__(self, *args, job=None, sync=None, conn=None, **kwargs):
 
     def load_prefixes(self):
         """Load InfobloxNetwork DiffSync model."""
-        if PLUGIN_CFG.get("import_subnets"):
+        if PLUGIN_CFG.get("infoblox_import_subnets"):
             subnets = []
-            for prefix in PLUGIN_CFG["import_subnets"]:
-                subnets.extend(self.conn.get_all_subnets(prefix=prefix))
-            all_networks = subnets
+            containers = []
+            for prefix in PLUGIN_CFG["infoblox_import_subnets"]:
+                # Get all child containers and subnets
+                tree = self.conn.get_tree_from_container(prefix)
+                containers.extend(tree)
+
+                # Need to check if the container has children. If it does, we need to get all subnets from the children
+                # If it doesn't, we can just get all subnets from the container
+                if tree:
+                    for subnet in tree:
+                        subnets.extend(self.conn.get_child_subnets_from_container(prefix=subnet["network"]))
+                else:
+                    subnets.extend(self.conn.get_all_subnets(prefix=prefix))
+
+            # Remove duplicates if a child subnet is included infoblox_import_subnets config
+            subnets = self.conn.remove_duplicates(subnets)
+            all_networks = self.conn.remove_duplicates(containers) + subnets
         else:
             # Need to load containers here to prevent duplicates when syncing back to Infoblox
             containers = self.conn.get_network_containers()

nautobot_ssot/integrations/infoblox/utils/client.py

@@ -1224,7 +1224,34 @@ def update_ipaddress(self, ip_address, **data):  # pylint: disable=inconsistent-
         logger.info("Infoblox IP Address updated: %s", response.json())
         return response.json()
 
-    def get_network_containers(self):
+    def get_tree_from_container(self, root_container: str) -> list:
+        """Returns the list of all child containers from a given root container."""
+        flattened_tree = []
+        stack = []
+        root_containers = self.get_network_containers(prefix=root_container)
+        if root_containers:
+            stack = [root_containers[0]]
+
+        while stack:
+            current_node = stack.pop()
+            flattened_tree.append(current_node)
+            children = self.get_child_network_containers(prefix=current_node["network"])
+            stack.extend(children)
+
+        return flattened_tree
+
+    def remove_duplicates(self, network_list: list) -> list:
+        """Removes duplicate networks from a list of networks."""
+        seen_networks = set()
+        new_list = []
+        for network in network_list:
+            if network["network"] not in seen_networks:
+                new_list.append(network)
+                seen_networks.add(network["network"])
+
+        return new_list
+
+    def get_network_containers(self, prefix: str = ""):
         """Get all Network Containers.
 
         Returns:
@@ -1248,10 +1275,102 @@ def get_network_containers(self):
             "_return_fields": "network,comment,network_view,extattrs,rir_organization,rir",
             "_max_results": 100000,
         }
+        if prefix:
+            params.update({"network": prefix})
         response = self._request("GET", url_path, params=params)
         response = response.json()
         logger.info(response)
         results = response.get("result", [])
         for res in results:
-            res.update({"status": "Container"})
+            res.update({"status": "container"})
         return results
+
+    def get_child_network_containers(self, prefix: str):
+        """Get all Child Network Containers for Container.
+
+        Returns:
+            (list) of record dicts
+
+        Return Response:
+        [
+            {
+                "_ref": "networkcontainer/ZG5zLm5ldHdvcmtfY29udGFpbmVyJDE5Mi4xNjguMi4wLzI0LzA:192.168.2.0/23/default",
+                "comment": "Campus LAN",
+                "extattrs": {},
+                "network": "192.168.2.0/24",
+                "network_view": "default",
+                "rir": "NONE",
+            },
+            {
+                "_ref": "networkcontainer/ZG5zLm5ldHdvcmtfY29udGFpbmVyJDE5Mi4xNjguMi4wLzI0LzA:192.168.2.0/23/default",
+                "comment": "Campus LAN 2",
+                "extattrs": {},
+                "network": "192.168.3.0/24",
+                "network_view": "default",
+                "rir": "NONE",
+            }
+        ]
+        """
+        url_path = "networkcontainer"
+        params = {
+            "_return_as_object": 1,
+            "_return_fields": "network,comment,network_view,extattrs,rir_organization,rir",
+            "_max_results": 100000,
+        }
+        params.update({"network_container": prefix})
+        response = self._request("GET", url_path, params=params)
+        response = response.json()
+        logger.info(response)
+        results = response.get("result", [])
+        for res in results:
+            res.update({"status": "container"})
+        return results
+
+    def get_child_subnets_from_container(self, prefix: str):
+        """Get child subnets from container.
+
+        Args:
+            prefix (str): Network prefix - '10.220.0.0/22'
+
+        Returns:
+            (list) of record dicts
+
+        Return Response:
+        [
+            {
+                "_ref": "network/ZG5zLm5ldHdvcmskMTAuMjIzLjAuMC8yMS8w:10.220.0.0/24/default",
+                "comment": "Campus 1",
+                "extattrs": {},
+                "network": "10.220.0.0/24",
+                "network_view": "default",
+                "rir": "NONE",
+                "vlans": [],
+            },
+            {
+                "_ref": "network/ZG5zLm5ldHdvcmskMTAuMjIzLjAuMC8yMS8w:10.220.1.0/24/default",
+                "comment": "Campus 2",
+                "extattrs": {},
+                "network": "10.220.1.0/24",
+                "network_view": "default",
+                "rir": "NONE",
+                "vlans": [],
+            },
+        ]
+        """
+        url_path = "network"
+        params = {
+            "_return_as_object": 1,
+            "_return_fields": "network,network_view,comment,extattrs,rir_organization,rir,vlans",
+            "_max_results": 10000,
+        }
+
+        params.update({"network_container": prefix})
+
+        try:
+            response = self._request("GET", url_path, params=params)
+        except HTTPError as err:
+            logger.info(err.response.text)
+            return []
+        response = response.json()
+        logger.info(response)
+        return response.get("result")

nautobot_ssot/tests/device42/unit/test_device42_adapter.py

@@ -88,8 +88,12 @@ def setUp(self):
         self.master_dev.os_version = ""
 
     @patch(
-        "nautobot_ssot.integrations.device42.diffsync.adapters.device42.PLUGIN_CFG",
-        {"device42_customer_is_facility": True, "device42_hostname_mapping": [{"AUS": "Austin"}]},
+        "nautobot_ssot.integrations.device42.utils.device42.PLUGIN_CFG",
+        {
+            "device42_customer_is_facility": True,
+            "device42_facility_prepend": "sitecode-",
+            "device42_hostname_mapping": [{"AUS": "Austin"}],
+        },
     )
     def test_data_loading(self):
         """Test the load() function."""

nautobot_ssot/tests/device42/unit/test_utils_device42.py

@@ -174,7 +174,7 @@ def test_get_netmiko_platform(self, name, sent, received):  # pylint: disable=un
         self.assertEqual(device42.get_netmiko_platform(sent), received)
 
     @patch(
-        "nautobot_ssot.integrations.device42.diffsync.adapters.device42.PLUGIN_CFG",
+        "nautobot_ssot.integrations.device42.utils.device42.PLUGIN_CFG",
         {"device42_role_prepend": "nautobot-"},
     )
     def test_find_device_role_from_tags(self):
@@ -189,7 +189,7 @@ def test_find_device_role_from_tags(self):
         self.assertEqual(device42.find_device_role_from_tags(tag_list=tags_missing_role), "Unknown")
 
     @patch(
-        "nautobot_ssot.integrations.device42.diffsync.adapters.device42.PLUGIN_CFG",
+        "nautobot_ssot.integrations.device42.utils.device42.PLUGIN_CFG",
         {"device42_facility_prepend": "sitecode-"},
     )
     def test_get_facility(self):