Update the ExampleDataSource job to improve memory utilization (#526)

changes/526.changed

@@ -0,0 +1,2 @@
+Updated the ExampleDataSource job to improve memory utilization with large data sets.
+Changed memory profiling logging output to format bytes into KiB/MiB.

nautobot_ssot/jobs/base.py

@@ -112,13 +112,29 @@ def sync_data(self, memory_profiling):
         - self.job_result (as per Job API)
         """
 
+        def format_size(size):  # pylint: disable=inconsistent-return-statements
+            """Format a size in bytes to a human-readable string. Borrowed from stdlib tracemalloc."""
+            for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+                if abs(size) < 100 and unit != "B":
+                    # 3 digits (xx.x UNIT)
+                    return "%.1f %s" % (size, unit)  # pylint: disable=consider-using-f-string
+                if abs(size) < 10 * 1024 or unit == "TiB":
+                    # 4 or 5 digits (xxxx UNIT)
+                    return "%.0f %s" % (size, unit)  # pylint: disable=consider-using-f-string
+                size /= 1024
+
         def record_memory_trace(step: str):
             """Helper function to record memory usage and reset tracemalloc stats."""
             memory_final, memory_peak = tracemalloc.get_traced_memory()
             setattr(self.sync, f"{step}_memory_final", memory_final)
             setattr(self.sync, f"{step}_memory_peak", memory_peak)
             self.sync.save()
-            self.logger.info("Traced memory for %s (Final, Peak): %s bytes, %s bytes", step, memory_final, memory_peak)
+            self.logger.info(
+                "Traced memory for %s (Final, Peak): %s, %s",
+                step,
+                format_size(memory_final),
+                format_size(memory_peak),
+            )
             tracemalloc.clear_traces()
 
         if not self.sync:

nautobot_ssot/jobs/examples.py

@@ -8,7 +8,7 @@
 except ImportError:
     from typing import TypedDict  # Python>=3.9
 
-from typing import List, Mapping, Optional
+from typing import Generator, List, Optional
 
 import requests
 from diffsync import Adapter
@@ -477,14 +477,13 @@ def __init__(self, *args, url=None, token=None, job=None, **kwargs):
             "Authorization": f"Token {self.token}",
         }
 
-    def _get_api_data(self, url_path: str) -> Mapping:
+    def _get_api_data(self, url_path: str) -> Generator:
         """Returns data from a url_path using pagination."""
         data = requests.get(f"{self.url}/{url_path}", headers=self.headers, params={"limit": 200}, timeout=60).json()
-        result_data = data["results"]
+        yield from data["results"]
         while data["next"]:
             data = requests.get(data["next"], headers=self.headers, params={"limit": 200}, timeout=60).json()
-            result_data.extend(data["results"])
-        return result_data
+            yield from data["results"]
 
     def load(self):
         """Load data from the remote Nautobot instance."""