Why not let sys.excepthook handle unhandled Exception in mutiprocessing.Process

I’m trying to collect all unhandled Exception. sys.excepthook is perfect for this except using with mutiprocessing.Process.

mutiprocessing.Process would catch the unhandled Exception in the target and print it out on stderr:

def _bootstrap(self):
    from . import util, context
    global _current_process, _process_counter, _children

    try:
        if self._start_method is not None:
            context._force_start_method(self._start_method)
        _process_counter = itertools.count(1)
        _children = set()
        util._close_stdin()
        old_process = _current_process
        _current_process = self
        try:
            util._finalizer_registry.clear()
            util._run_after_forkers()
        finally:
            # delay finalization of the old process object until after
            # _run_after_forkers() is executed
            del old_process
        util.info('child process calling self.run()')
        try:
            self.run()
            exitcode = 0
        finally:
            util._exit_function()
    except SystemExit as e:
        if not e.args:
            exitcode = 1
        elif isinstance(e.args[0], int):
            exitcode = e.args[0]
        else:
            sys.stderr.write(str(e.args[0]) + '\n')
            exitcode = 1
    except:
        exitcode = 1
        import traceback
        sys.stderr.write('Process %s:\n' % self.name)
        traceback.print_exc()
    finally:
        threading._shutdown()
        util.info('process exiting with exitcode %d' % exitcode)
        util._flush_std_streams()

    return exitcode

Why not simply delete

except:
        exitcode = 1
        import traceback
        sys.stderr.write('Process %s:\n' % self.name)
        traceback.print_exc()

and leave the unhandled Exception to sys.excepthook?

To me, the routine in mutiprocessing.Process it exactly the same in default sys.excepthook:

/*[clinic input]
sys.excepthook

    exctype:   object
    value:     object
    traceback: object
    /

Handle an exception by displaying it with a traceback on sys.stderr.
[clinic start generated code]*/

static PyObject *
sys_excepthook_impl(PyObject *module, PyObject *exctype, PyObject *value,
                    PyObject *traceback)
/*[clinic end generated code: output=18d99fdda21b6b5e input=ecf606fa826f19d9]*/
{
    PyErr_Display(exctype, value, traceback);
    Py_RETURN_NONE;
}

So, what’s the point not using sys.excepthook in mutiprocessing.Process?

Besides, threading provide threading.excepthook to modify the default behavior, is there any equivalent for mutiprocessing?

2 Likes

I find out that deleting

except:
        exitcode = 1
        import traceback
        sys.stderr.write('Process %s:\n' % self.name)
        traceback.print_exc()

cannot work with spawn.
The exception raised in _bootstrap will be ignored.

def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
    '''
    Run code specified by data received over pipe
    '''
    assert is_forking(sys.argv), "Not forking"
    if sys.platform == 'win32':
        import msvcrt
        import _winapi

        if parent_pid is not None:
            source_process = _winapi.OpenProcess(
                _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE,
                False, parent_pid)
        else:
            source_process = None
        new_handle = reduction.duplicate(pipe_handle,
                                         source_process=source_process)
        fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY)
        parent_sentinel = source_process
    else:
        from . import resource_tracker
        resource_tracker._resource_tracker._fd = tracker_fd
        fd = pipe_handle
        parent_sentinel = os.dup(pipe_handle)
    exitcode = _main(fd, parent_sentinel)
    sys.exit(exitcode)


def _main(fd, parent_sentinel):
    with os.fdopen(fd, 'rb', closefd=True) as from_parent:
        process.current_process()._inheriting = True
        try:
            preparation_data = reduction.pickle.load(from_parent)
            prepare(preparation_data)
            self = reduction.pickle.load(from_parent)
        finally:
            del process.current_process()._inheriting
    return self._bootstrap(parent_sentinel)

I cannot find where Python pass unhandled Exception to sys.excepthook, and cannot understand why Exception raised in _bootstrap got ignored.

Looking forward to any explanation :upside_down_face: