I ran into an issue that I have traced back to _multiprocessing.SemLock and I wanted to see if anybody has ran into this or knows what’s going on.
Here is my code that doesn’t work on my system
# taken from /usr/local/lib/python3.9/multiprocessing/synchronize.py(57)__init__() and simplified down.
import _multiprocessing
kind = 1
value = 1
maxvalue = 1
name = 'blah'
unlink_now = False
_multiprocessing.SemLock(kind, value, maxvalue, name, unlink_now)
Here is the stack trace I get from that:
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
Cell In[6], line 8
6 name = 'blah'
7 unlink_now = False
----> 8 _multiprocessing.SemLock(kind, value, maxvalue, name, unlink_now)
FileNotFoundError: [Errno 2] No such file or directory
Does anybody know what is going on here? Here is a minimal example of what I was originally trying:
from fastcore.parallel import parallel
parallel(lambda:"",[])
and that stack trace (in case the real problem is further upstream):
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
Cell In[10], line 2
1 from fastcore.parallel import parallel
----> 2 parallel(lambda:"",[])
File /usr/local/lib/python3.9/site-packages/fastcore/parallel.py:112, in parallel(f, items, n_workers, total, progress, pause, method, threadpool, timeout, chunksize, *args, **kwargs)
110 if method: kwpool['mp_context'] = get_context(method)
111 pool = ProcessPoolExecutor
--> 112 with pool(n_workers, pause=pause, **kwpool) as ex:
113 r = ex.map(f,items, *args, timeout=timeout, chunksize=chunksize, **kwargs)
114 if progress and progress_bar:
File /usr/local/lib/python3.9/site-packages/fastcore/parallel.py:84, in ProcessPoolExecutor.__init__(self, max_workers, on_exc, pause, **kwargs)
82 self.not_parallel = max_workers==0
83 if self.not_parallel: max_workers=1
---> 84 super().__init__(max_workers, **kwargs)
File /usr/local/lib/python3.9/concurrent/futures/process.py:649, in ProcessPoolExecutor.__init__(self, max_workers, mp_context, initializer, initargs)
644 # Create communication channels for the executor
645 # Make the call queue slightly larger than the number of processes to
646 # prevent the worker processes from idling. But don't make it too big
647 # because futures in the call queue cannot be cancelled.
648 queue_size = self._max_workers + EXTRA_QUEUED_CALLS
--> 649 self._call_queue = _SafeQueue(
650 max_size=queue_size, ctx=self._mp_context,
651 pending_work_items=self._pending_work_items,
652 shutdown_lock=self._shutdown_lock,
653 thread_wakeup=self._executor_manager_thread_wakeup)
654 # Killed worker processes can produce spurious "broken pipe"
655 # tracebacks in the queue's own worker thread. But we detect killed
656 # processes anyway, so silence the tracebacks.
657 self._call_queue._ignore_epipe = True
File /usr/local/lib/python3.9/concurrent/futures/process.py:168, in _SafeQueue.__init__(self, max_size, ctx, pending_work_items, shutdown_lock, thread_wakeup)
166 self.shutdown_lock = shutdown_lock
167 self.thread_wakeup = thread_wakeup
--> 168 super().__init__(max_size, ctx=ctx)
File /usr/local/lib/python3.9/multiprocessing/queues.py:43, in Queue.__init__(self, maxsize, ctx)
41 self._maxsize = maxsize
42 self._reader, self._writer = connection.Pipe(duplex=False)
---> 43 self._rlock = ctx.Lock()
44 self._opid = os.getpid()
45 if sys.platform == 'win32':
File /usr/local/lib/python3.9/multiprocessing/context.py:68, in BaseContext.Lock(self)
66 '''Returns a non-recursive lock object'''
67 from .synchronize import Lock
---> 68 return Lock(ctx=self.get_context())
File /usr/local/lib/python3.9/multiprocessing/synchronize.py:162, in Lock.__init__(self, ctx)
161 def __init__(self, *, ctx):
--> 162 SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx)
File /usr/local/lib/python3.9/multiprocessing/synchronize.py:57, in SemLock.__init__(self, kind, value, maxvalue, ctx)
55 for i in range(100):
56 try:
---> 57 sl = self._semlock = _multiprocessing.SemLock(
58 kind, value, maxvalue, self._make_name(),
59 unlink_now)
60 except FileExistsError:
61 pass
FileNotFoundError: [Errno 2] No such file or directory
Additional Details:
I am running in a dockerfile via devcontainer on windows11. The issue only presents itself when I have the following runarg: "--ipc=host"
The base dockerfile is python 3.9 and I am specifying the containerUser to be uid 1000 rather that running as root. I just tested and this issue does not happen if I run the container as root rather than uid 1000. Any idea what file it would be looking for?
Please let me know if any other information would be helpful and I would be more than happy to provide them!