Merge pull request #112 from nautobot/develop

fix: Add InfobloxApi Connection (#104) 
build: ⬆️ Update dependencies to latest for project (#107) 
docs: 📝 Fix README to include SSoT framework plugin (#109) 
fix: Fix Extensibility Attributes Churn & Fix Prefix VLAN Updates (#111) 
build: ➕ Add python-semantic-release for automated CHANGELOG and versioning (#113)

CHANGELOG.md

@@ -0,0 +1,8 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+<!--next-version-placeholder-->

README.md

@@ -22,7 +22,7 @@ Once installed, the plugin needs to be enabled in your `nautobot_config.py`
 
 ```python
 # In your nautobot_config.py
-PLUGINS = ["nautobot_ssot_infoblox"]
+PLUGINS = ["nautobot_ssot", "nautobot_ssot_infoblox"]
 ```
 
 See the section below for configuration settings.
@@ -43,6 +43,9 @@ See the section below for configuration settings.
 
 ```python
 PLUGINS_CONFIG = {
+    "nautobot_ssot": {
+        "hide_example_jobs": True,  # defaults to False if unspecified
+    }
     "nautobot_ssot_infoblox": {
         "NAUTOBOT_INFOBLOX_URL": os.getenv("NAUTOBOT_INFOBLOX_URL", ""),
         "NAUTOBOT_INFOBLOX_USERNAME": os.getenv("NAUTOBOT_INFOBLOX_USERNAME", ""),

nautobot_ssot_infoblox/diffsync/adapters/infoblox.py

@@ -5,6 +5,7 @@
 from diffsync import DiffSync
 from diffsync.enum import DiffSyncFlags
 from nautobot.extras.plugins.exceptions import PluginImproperlyConfigured
+from nautobot_ssot_infoblox.utils.client import get_default_ext_attrs
 from nautobot_ssot_infoblox.utils.diffsync import get_ext_attr_dict, build_vlan_map
 from nautobot_ssot_infoblox.diffsync.models.infoblox import (
     InfobloxAggregate,
@@ -52,53 +53,64 @@ def load_prefixes(self):
         subnets = self.conn.get_all_subnets()
         self.subnets = [(x["network"], x["network_view"]) for x in subnets]
         all_networks = containers + subnets
+        default_ext_attrs = get_default_ext_attrs(review_list=all_networks)
         for _pf in all_networks:
+            pf_ext_attrs = get_ext_attr_dict(extattrs=_pf.get("extattrs", {}))
             new_pf = self.prefix(
                 network=_pf["network"],
                 description=_pf.get("comment", ""),
                 status=_pf.get("status", "active"),
-                ext_attrs=get_ext_attr_dict(extattrs=_pf.get("extattrs", {})),
+                ext_attrs={**default_ext_attrs, **pf_ext_attrs},
                 vlans=build_vlan_map(vlans=_pf["vlans"]) if _pf.get("vlans") else {},
             )
             self.add(new_pf)
 
     def load_ipaddresses(self):
         """Load InfobloxIPAddress DiffSync model."""
-        for _ip in self.conn.get_all_ipv4address_networks(prefixes=self.subnets):
+        ipaddrs = self.conn.get_all_ipv4address_networks(prefixes=self.subnets)
+        default_ext_attrs = get_default_ext_attrs(review_list=ipaddrs)
+        for _ip in ipaddrs:
             _, prefix_length = _ip["network"].split("/")
             if _ip["names"]:
+                ip_ext_attrs = get_ext_attr_dict(extattrs=_ip.get("extattrs", {}))
                 new_ip = self.ipaddress(
                     address=_ip["ip_address"],
                     prefix=_ip["network"],
                     prefix_length=prefix_length,
                     dns_name=_ip["names"][0],
                     status=self.conn.get_ipaddr_status(_ip),
                     description=_ip["comment"],
-                    ext_attrs=get_ext_attr_dict(extattrs=_ip.get("extattrs", {})),
+                    ext_attrs={**default_ext_attrs, **ip_ext_attrs},
                 )
                 self.add(new_ip)
 
     def load_vlanviews(self):
         """Load InfobloxVLANView DiffSync model."""
-        for _vv in self.conn.get_vlanviews():
+        vlanviews = self.conn.get_vlanviews()
+        default_ext_attrs = get_default_ext_attrs(review_list=vlanviews)
+        for _vv in vlanviews:
+            vv_ext_attrs = get_ext_attr_dict(extattrs=_vv.get("extattrs", {}))
             new_vv = self.vlangroup(
                 name=_vv["name"],
                 description=_vv["comment"] if _vv.get("comment") else "",
-                ext_attrs=get_ext_attr_dict(extattrs=_vv.get("extattrs", {})),
+                ext_attrs={**default_ext_attrs, **vv_ext_attrs},
             )
             self.add(new_vv)
 
     def load_vlans(self):
         """Load InfobloxVlan DiffSync model."""
-        for _vlan in self.conn.get_vlans():
+        vlans = self.conn.get_vlans()
+        default_ext_attrs = get_default_ext_attrs(review_list=vlans)
+        for _vlan in vlans:
+            vlan_ext_attrs = get_ext_attr_dict(extattrs=_vlan.get("extattrs", {}))
             vlan_group = re.search(r"(?:.+\:)(\S+)(?:\/\S+\/.+)", _vlan["_ref"])
             new_vlan = self.vlan(
                 name=_vlan["name"],
                 vid=_vlan["id"],
                 status=_vlan["status"],
                 vlangroup=vlan_group.group(1) if vlan_group else "",
                 description=_vlan["comment"] if _vlan.get("comment") else "",
-                ext_attrs=get_ext_attr_dict(extattrs=_vlan.get("extattrs", {})),
+                ext_attrs={**default_ext_attrs, **vlan_ext_attrs},
             )
             self.add(new_vlan)
 
@@ -127,6 +139,7 @@ def __init__(self, *args, job=None, sync=None, conn=None, **kwargs):
         Args:
             job (object, optional): Infoblox job. Defaults to None.
             sync (object, optional): Infoblox DiffSync. Defaults to None.
+            conn (object): InfobloxAPI connection.
         """
         super().__init__(*args, **kwargs)
         self.job = job
@@ -141,12 +154,15 @@ def __init__(self, *args, job=None, sync=None, conn=None, **kwargs):
 
     def load(self):
         """Load aggregate models."""
-        for container in self.conn.get_network_containers():
+        containers = self.conn.get_network_containers()
+        default_ext_attrs = get_default_ext_attrs(review_list=containers)
+        for container in containers:
             network = ipaddress.ip_network(container["network"])
+            container_ext_attrs = get_ext_attr_dict(extattrs=container.get("extattrs", {}))
             if network.is_private and container["network"] in ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]:
                 new_aggregate = self.aggregate(
                     network=container["network"],
                     description=container["comment"] if container.get("comment") else "",
-                    ext_attrs=container.get("extattrs", {}),
+                    ext_attrs={**default_ext_attrs, **container_ext_attrs},
                 )
                 self.add(new_aggregate)

nautobot_ssot_infoblox/diffsync/adapters/nautobot.py

@@ -15,7 +15,8 @@
     NautobotVlan,
 )
 from nautobot_ssot_infoblox.constant import TAG_COLOR
-from nautobot_ssot_infoblox.utils.diffsync import nautobot_vlan_status
+from nautobot_ssot_infoblox.utils.diffsync import nautobot_vlan_status, get_default_custom_fields
+from nautobot_ssot_infoblox.utils.nautobot import build_vlan_map_from_relations, get_prefix_vlans
 
 
 class NautobotMixin:
@@ -100,22 +101,17 @@ def __init__(self, *args, job=None, sync=None, **kwargs):
     def load_prefixes(self):
         """Load Prefixes from Nautobot."""
         all_prefixes = list(chain(Prefix.objects.all(), Aggregate.objects.all()))
+        default_cfs = get_default_custom_fields(cf_contenttype=ContentType.objects.get_for_model(Prefix))
         for prefix in all_prefixes:
-            # Reset CustomFields for Nautobot objects to blank if they failed to get linked originally.
-            if prefix.site is None:
-                prefix.custom_field_data["site"] = ""
-            if prefix.vrf is None:
-                prefix.custom_field_data["vrf"] = ""
-            if prefix.role is None:
-                prefix.custom_field_data["role"] = ""
-            if prefix.tenant is None:
-                prefix.custom_field_data["tenant"] = ""
+            if "ssot-synced-to-infoblox" in prefix.custom_field_data:
+                prefix.custom_field_data.pop("ssot-synced-to-infoblox")
+            current_vlans = get_prefix_vlans(prefix=prefix)
             _prefix = self.prefix(
                 network=str(prefix.prefix),
                 description=prefix.description,
                 status=prefix.status.slug if hasattr(prefix, "status") else "container",
-                ext_attrs=prefix.custom_field_data,
-                vlans={prefix.vlan.vid: prefix.vlan.name} if prefix.vlan is not None else {},
+                ext_attrs={**default_cfs, **prefix.custom_field_data},
+                vlans=build_vlan_map_from_relations(vlans=current_vlans),
                 pk=prefix.id,
             )
             try:
@@ -125,15 +121,8 @@ def load_prefixes(self):
 
     def load_ipaddresses(self):
         """Load IP Addresses from Nautobot."""
+        default_cfs = get_default_custom_fields(cf_contenttype=ContentType.objects.get_for_model(IPAddress))
         for ipaddr in IPAddress.objects.all():
-            # Reset CustomFields for Nautobot objects to blank if they failed to get linked originally.
-            if ipaddr.vrf is None:
-                ipaddr.custom_field_data["vrf"] = ""
-            if ipaddr.role is None:
-                ipaddr.custom_field_data["role"] = ""
-            if ipaddr.tenant is None:
-                ipaddr.custom_field_data["tenant"] = ""
-
             addr = ipaddr.host
             # the last Prefix is the most specific and is assumed the one the IP address resides in
             prefix = Prefix.objects.net_contains(addr).last()
@@ -154,14 +143,16 @@ def load_ipaddresses(self):
                 continue
 
             if ipaddr.dns_name:
+                if "ssot-synced-to-infoblox" in ipaddr.custom_field_data:
+                    ipaddr.custom_field_data.pop("ssot-synced-to-infoblox")
                 _ip = self.ipaddress(
                     address=addr,
                     prefix=str(prefix),
                     status=ipaddr.status.name if ipaddr.status else None,
                     prefix_length=prefix.prefix_length if prefix else ipaddr.prefix_length,
                     dns_name=ipaddr.dns_name,
                     description=ipaddr.description,
-                    ext_attrs=ipaddr.custom_field_data,
+                    ext_attrs={**default_cfs, **ipaddr.custom_field_data},
                     pk=ipaddr.id,
                 )
                 try:
@@ -171,31 +162,31 @@ def load_ipaddresses(self):
 
     def load_vlangroups(self):
         """Load VLAN Groups from Nautobot."""
+        default_cfs = get_default_custom_fields(cf_contenttype=ContentType.objects.get_for_model(VLANGroup))
         for grp in VLANGroup.objects.all():
-            # Reset CustomFields for Nautobot objects to blank if they failed to get linked originally.
-            if grp.site is None:
-                grp.custom_field_data["site"] = ""
-            _vg = self.vlangroup(name=grp.name, description=grp.description, ext_attrs=grp.custom_field_data, pk=grp.id)
+            if "ssot-synced-to-infoblox" in grp.custom_field_data:
+                grp.custom_field_data.pop("ssot-synced-to-infoblox")
+            _vg = self.vlangroup(
+                name=grp.name,
+                description=grp.description,
+                ext_attrs={**default_cfs, **grp.custom_field_data},
+                pk=grp.id,
+            )
             self.add(_vg)
 
     def load_vlans(self):
         """Load VLANs from Nautobot."""
+        default_cfs = get_default_custom_fields(cf_contenttype=ContentType.objects.get_for_model(VLAN))
         for vlan in VLAN.objects.all():
-            # Reset CustomFields for Nautobot objects to blank if they failed to get linked originally.
-            if vlan.site is None:
-                vlan.custom_field_data["site"] = ""
-            if vlan.role is None:
-                vlan.custom_field_data["role"] = ""
-            if vlan.tenant is None:
-                vlan.custom_field_data["tenant"] = ""
-
+            if "ssot-synced-to-infoblox" in vlan.custom_field_data:
+                vlan.custom_field_data.pop("ssot-synced-to-infoblox")
             _vlan = self.vlan(
                 vid=vlan.vid,
                 name=vlan.name,
                 description=vlan.description,
                 vlangroup=vlan.group.name if vlan.group else "",
                 status=nautobot_vlan_status(vlan.status.name),
-                ext_attrs=vlan.custom_field_data,
+                ext_attrs={**default_cfs, **vlan.custom_field_data},
                 pk=vlan.id,
             )
             self.add(_vlan)

nautobot_ssot_infoblox/diffsync/models/nautobot.py

@@ -21,6 +21,7 @@
 from nautobot.tenancy.models import Tenant as OrmTenant
 from nautobot_ssot_infoblox.diffsync.models.base import Aggregate, Network, IPAddress, Vlan, VlanView
 from nautobot_ssot_infoblox.utils.diffsync import create_tag_sync_from_infoblox
+from nautobot_ssot_infoblox.utils.nautobot import get_prefix_vlans
 
 
 def process_ext_attrs(diffsync, obj: object, extattrs: dict):
@@ -129,15 +130,45 @@ def create(cls, diffsync, ids, attrs):
         _prefix.validated_save()
         return super().create(ids=ids, diffsync=diffsync, attrs=attrs)
 
-    def update(self, attrs):
+    def update(self, attrs):  # pylint: disable=too-many-branches
         """Update Prefix object in Nautobot."""
         _pf = OrmPrefix.objects.get(id=self.pk)
+        if self.diffsync.job.kwargs.get("debug"):
+            self.diffsync.job.log_debug(message=f"Attempting to update Prefix {_pf.prefix} with {attrs}.")
         if "description" in attrs:
             _pf.description = attrs["description"]
         if "status" in attrs:
             _pf.status = OrmStatus.objects.get(slug=attrs["status"])
         if "ext_attrs" in attrs:
             process_ext_attrs(diffsync=self.diffsync, obj=_pf, extattrs=attrs["ext_attrs"])
+        if "vlans" in attrs:
+            current_vlans = get_prefix_vlans(prefix=_pf)
+            if len(current_vlans) < len(attrs["vlans"]):
+                for _, item in attrs["vlans"].items():
+                    vlan = OrmVlan.objects.get(vid=item["vid"], name=item["name"], group__name=item["group"])
+                    if vlan not in current_vlans:
+                        if self.diffsync.job.kwargs.get("debug"):
+                            self.diffsync.job.log_debug(message=f"Adding VLAN {vlan.vid} to {_pf.prefix}.")
+                        OrmRelationshipAssociation.objects.get_or_create(
+                            relationship_id=OrmRelationship.objects.get(name="Prefix -> VLAN").id,
+                            source_type=ContentType.objects.get_for_model(OrmPrefix),
+                            source_id=_pf.id,
+                            destination_type=ContentType.objects.get_for_model(OrmVlan),
+                            destination_id=vlan.id,
+                        )
+            else:
+                for vlan in current_vlans:
+                    if vlan.vid not in attrs["vlans"]:
+                        del_vlan = OrmRelationshipAssociation.objects.get(
+                            relationship_id=OrmRelationship.objects.get(name="Prefix -> VLAN").id,
+                            source_type=ContentType.objects.get_for_model(OrmPrefix),
+                            source_id=_pf.id,
+                            destination_type=ContentType.objects.get_for_model(OrmVlan),
+                            destination_id=vlan.id,
+                        )
+                        if self.diffsync.job.kwargs.get("debug"):
+                            self.diffsync.job.log_debug(message=f"Removing VLAN {vlan.vid} from {_pf.prefix}.")
+                        del_vlan.delete()
         _pf.validated_save()
         return super().update(attrs)
 

nautobot_ssot_infoblox/utils/client.py

@@ -11,6 +11,7 @@
 from dns import reversename
 from nautobot.core.settings_funcs import is_truthy
 from nautobot_ssot_infoblox.constant import PLUGIN_CFG
+from nautobot_ssot_infoblox.utils.diffsync import get_ext_attr_dict
 
 logger = logging.getLogger(__name__)
 
@@ -29,6 +30,24 @@ def parse_url(address):
     return urllib.parse.urlparse(address)
 
 
+def get_default_ext_attrs(review_list: list) -> dict:
+    """Determine the default Extensibility Attributes for an object being processed.
+
+    Args:
+        review_list (list): The list of objects that need to be reviewed to gather default Extensibility Attributes.
+
+    Returns:
+        dict: Dictionary of default Extensibility Attributes for a VLAN View, VLANs, Prefixes, or IP Addresses.
+    """
+    default_ext_attrs = {}
+    for item in review_list:
+        pf_ext_attrs = get_ext_attr_dict(extattrs=item.get("extattrs", {}))
+        for attr in pf_ext_attrs:
+            if attr not in default_ext_attrs:
+                default_ext_attrs[attr] = None
+    return default_ext_attrs
+
+
 class InvalidUrlScheme(Exception):
     """Exception raised for wrong scheme being passed for URL.
 

nautobot_ssot_infoblox/utils/diffsync.py

@@ -1,7 +1,7 @@
 """Utilities for DiffSync related stuff."""
-
+from django.contrib.contenttypes.models import ContentType
 from django.utils.text import slugify
-from nautobot.extras.models import Tag
+from nautobot.extras.models import CustomField, Tag
 from nautobot_ssot_infoblox.constant import TAG_COLOR
 
 
@@ -75,3 +75,21 @@ def build_vlan_map(vlans: list):
     for vlan in vlans:
         vlan_map[vlan["id"]] = {"vid": vlan["id"], "name": vlan["name"], "group": get_vlan_view_name(vlan["vlan"])}
     return vlan_map
+
+
+def get_default_custom_fields(cf_contenttype: ContentType) -> dict:
+    """Get default Custom Fields for specific ContentType.
+
+    Args:
+        cf_contenttype (ContentType): Specific ContentType to get all Custom Fields for.
+
+    Returns:
+        dict: Dictionary of all Custom Fields for a specific object type.
+    """
+    customfields = CustomField.objects.filter(content_types=cf_contenttype)
+    default_cfs = {}
+    for customfield in customfields:
+        if customfield.name != "ssot-synced-to-infoblox":
+            if customfield.name not in default_cfs:
+                default_cfs[customfield.name] = None
+    return default_cfs