This is a thread dump from gdb py-bt . And python version is python-2.7.5-92.
_active_limbo_lock deadlock? lile threading.enumerate(): reentrant call during a GC collection hangs · Issue #88588 · python/cpython · GitHub
Thread 2 (Thread 0x7f4ec8f9b700 (LWP 11714)):
#4 Waiting for a lock (e.g. GIL)
#5 Waiting for a lock (e.g. GIL)
#8 Frame 0x7f4ee0137c70, for file /usr/lib64/python2.7/threading.py, line 744, in start (self=<RadosThread(_Thread__ident=None, target=<_FuncPtr(__name__='rados_aio_release') at remote 0x7f4f91688530>, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9b6a85d0>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9b6a85d0>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7f4f9b6a85d0>) at remote 0x7f4f903cb4d0>, _Thread__name='Thread-3937766', _Thread__daemonic=True, args=(<c_void_p at remote 0x7f4f902b75f0>,), _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=False, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9b6a8eb0>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9b6a8eb0>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7f4f9b6a8eb0>) at remote 0x7f4f903cb2d0>) at remot...(truncated)
with _active_limbo_lock:
#11 Frame 0x7f4ee00dd890, for file /var/lib/zstack/virtualenv/kvm/lib/python2.7/site-packages/rados.py, line 173, in run_in_thread (target=<_FuncPtr(__name__='rados_aio_release') at remote 0x7f4f91688530>, args=(<c_void_p at remote 0x7f4f902b75f0>,), timeout=0, interrupt=False, countdown=0, t=<RadosThread(_Thread__ident=None, target=<...>, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9b6a85d0>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9b6a85d0>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7f4f9b6a85d0>) at remote 0x7f4f903cb4d0>, _Thread__name='Thread-3937766', _Thread__daemonic=True, args=(...), _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=False, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9b6a8eb0>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9b6a8eb0>, _Condition__waiters=[], release=<bu...(truncated)
t.start()
#15 Frame 0x7f4ee00afeb0, for file /var/lib/zstack/virtualenv/kvm/lib/python2.7/site-packages/rados.py, line 1019, in __del__ (self=<Completion(onsafe=None, rados_comp=<c_void_p at remote 0x7f4f902b75f0>, ioctx=<Ioctx(locator_key='', name='pool-a6126efe2e44417aaf8e3135bd32906c', safe_cbs={}, librados=<CDLL(_FuncPtr=<_ctypes.PyCFuncPtrType at remote 0x7f4ee00030a0>, rados_aio_write_full=<_FuncPtr(__name__='rados_aio_write_full') at remote 0x7f4f91688c80>, _handle=139976742313344, _name='librados.so.2', rados_aio_release=<_FuncPtr(__name__='rados_aio_release') at remote 0x7f4f91688530>, rados_ioctx_create=<_FuncPtr(__name__='rados_ioctx_create') at remote 0x7f4f916881f0>, rados_create2=<_FuncPtr(__name__='rados_create2') at remote 0x7f4f90487460>, rados_conf_read_file=<_FuncPtr(__name__='rados_conf_read_file') at remote 0x7f4f904876d0>, rados_connect=<_FuncPtr(__name__='rados_connect') at remote 0x7f4f90487390>, rados_aio_create_completion=<_FuncPtr(__name__='rados_aio_create_completion') at remote 0x7f4f91688d50>,...(truncated)
(self.rados_comp,))
#26 Garbage-collecting
#32 Frame 0x7f4ee00c3bc0, for file /usr/lib64/python2.7/threading.py, line 396, in notify (self=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9030f190>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9030f190>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7f4f9030f190>) at remote 0x7f4f92798f50>, n=1, _Condition__waiters=[...], waiters=[<thread.lock at remote 0x7f4f9030fe50>])
for waiter in waiters:
#36 Frame 0x7f4f99821790, for file /usr/lib64/python2.7/threading.py, line 410, in notifyAll (self=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9030f190>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9030f190>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7f4f9030f190>) at remote 0x7f4f92798f50>)
self.notify(len(self.__waiters))
#39 Frame 0x7f4f9934ec90, for file /usr/lib64/python2.7/threading.py, line 875, in __stop (self=<RadosThread(_Thread__ident=139976355985152, target=<_FuncPtr(__name__='rados_aio_create_completion') at remote 0x7f4f91688d50>, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9030f190>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9030f190>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7f4f9030f190>) at remote 0x7f4f92798f50>, _Thread__name='Thread-3937765', _Thread__daemonic=True, retval=0, args=(<c_void_p at remote 0x7f4f901ebc20>, None, None, <CArgObject at remote 0x7f4f903cbf70>), _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9030f3d0>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9030f3d0>, _Condition__waiters=[], release=<built-in method release of ...(truncated)
self.__block.notify_all()
#42 Frame 0x7f4ee011c200, for file /usr/lib64/python2.7/threading.py, line 860, in __bootstrap_inner (self=<RadosThread(_Thread__ident=139976355985152, target=<_FuncPtr(__name__='rados_aio_create_completion') at remote 0x7f4f91688d50>, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9030f190>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9030f190>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7f4f9030f190>) at remote 0x7f4f92798f50>, _Thread__name='Thread-3937765', _Thread__daemonic=True, retval=0, args=(<c_void_p at remote 0x7f4f901ebc20>, None, None, <CArgObject at remote 0x7f4f903cbf70>), _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9030f3d0>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9030f3d0>, _Condition__waiters=[], release=<built-in method ...(truncated)
self.__stop()
#45 Frame 0x7f4ee0145d40, for file /usr/lib64/python2.7/threading.py, line 785, in __bootstrap (self=<RadosThread(_Thread__ident=139976355985152, target=<_FuncPtr(__name__='rados_aio_create_completion') at remote 0x7f4f91688d50>, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9030f190>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9030f190>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7f4f9030f190>) at remote 0x7f4f92798f50>, _Thread__name='Thread-3937765', _Thread__daemonic=True, retval=0, args=(<c_void_p at remote 0x7f4f901ebc20>, None, None, <CArgObject at remote 0x7f4f903cbf70>), _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7f4f9030f3d0>, acquire=<built-in method acquire of thread.lock object at remote 0x7f4f9030f3d0>, _Condition__waiters=[], release=<built-in method releas...(truncated)
self.__bootstrap_inner()