LocalRPC: add commands to help debug memory leaks

electrumx/server/session.py

@@ -26,7 +26,7 @@
 from aiorpcx import (Event, JSONRPCAutoDetect, JSONRPCConnection,
                      ReplyAndDisconnect, Request, RPCError, RPCSession,
                      handler_invocation, serve_rs, serve_ws, sleep,
-                     NewlineFramer, TaskTimeout, timeout_after)
+                     NewlineFramer, TaskTimeout, timeout_after, run_in_thread)
 
 import electrumx
 import electrumx.lib.util as util
@@ -167,7 +167,8 @@ def __init__(
 
         # Set up the RPC request handlers
         cmds = ('add_peer daemon_url disconnect getinfo groups log peers '
-                'query reorg sessions stop'.split())
+                'query reorg sessions stop debug_memusage_list_all_objects '
+                'debug_memusage_get_random_backref_chain'.split())
         LocalRPC.request_handlers = {cmd: getattr(self, 'rpc_' + cmd)
                                      for cmd in cmds}
 
@@ -593,6 +594,38 @@ async def rpc_reorg(self, count):
             raise RPCError(BAD_REQUEST, 'still catching up with daemon')
         return f'scheduled a reorg of {count:,d} blocks'
 
+    async def rpc_debug_memusage_list_all_objects(self, limit: int) -> str:
+        """Return a string listing the most common types in memory."""
+        import objgraph  # optional dependency
+        import io
+        with io.StringIO() as fd:
+            objgraph.show_most_common_types(
+                limit=limit,
+                shortnames=False,
+                file=fd)
+            return fd.getvalue()
+
+    async def rpc_debug_memusage_get_random_backref_chain(self, objtype: str) -> str:
+        """Return a dotfile as text containing the backref chain
+        for a randomly selected object of type objtype.
+
+        Warning: very slow! and it blocks the server.
+
+        To convert to image:
+        $ dot -Tps filename.dot -o outfile.ps
+        """
+        import objgraph  # optional dependency
+        import random
+        import io
+        with io.StringIO() as fd:
+            await run_in_thread(lambda:
+                objgraph.show_chain(
+                    objgraph.find_backref_chain(
+                        random.choice(objgraph.by_type(objtype)),
+                        objgraph.is_proper_module),
+                    output=fd))
+            return fd.getvalue()
+
     # --- External Interface
 
     async def serve(self, notifications, event):

electrumx_rpc

@@ -74,6 +74,22 @@ other_commands = {
             'help': 'number of blocks to back up'
         },
     ),
+    'debug_memusage_list_all_objects': (
+        'Print a table of types of most common types in memory',
+        ['--limit'], {
+            'type': int,
+            'default': 50,
+            'help': 'max number of types to return',
+        },
+    ),
+    'debug_memusage_get_random_backref_chain': (
+        'Return a dotfile as text containing the backref chain for a randomly selected object of type objtype',
+        [], {
+            'type': str,
+            'dest': 'objtype',
+            'help': 'e.g. "_asyncio.Task"',
+        },
+    ),
 }
 
 
@@ -125,6 +141,12 @@ def main():
                     if method in ('query', ):
                         for line in result:
                             print(line)
+                    elif method in (
+                            'debug_memusage_list_all_objects',
+                            'debug_memusage_get_random_backref_chain',
+                    ):
+                        for line in result.split('\n'):
+                            print(line)
                     elif method in ('groups', 'peers', 'sessions'):
                         lines_func = getattr(text, f'{method}_lines')
                         for line in lines_func(result):

setup.py

@@ -9,6 +9,7 @@
     install_requires=['aiorpcX[ws]>=0.22.0,<0.23', 'attrs',
                       'plyvel', 'pylru', 'aiohttp>=3.3,<4'],
     extras_require={
+        'dev': ['objgraph'],
         'rapidjson': ['python-rapidjson>=0.4.1,<2.0'],
         'rocksdb': ['python-rocksdb>=0.6.9'],
         'ujson': ['ujson>=2.0.0,<4.0.0'],