Calling PyRun_SimpleString("import numpy as np") in an thread is blocking

Python Help
1

Hi,
I’m definitely missing something and couldn’t find a complete example
execution of this code crashes when I import numpy but not for time
The problem only occurs in one thread (detached), the code called in the main thread works as expected

class PythonRunner
{
public:
    //=============================================================================
    void
    runPythonCode(PyObject* globalDict, const wstringVector& commands)
    {

        std::string str = wstring_to_utf8(commands[0]) + "\n";
        PyRun_SimpleString(str.c_str());

        outputMessage += getPythonStandardOutput();

        resultReady = true;
        resultCondition.notify_one();
    }
    //=============================================================================
    bool
    isResultReady()
    {
        std::lock_guard<std::mutex> lock(resultMutex);
        return resultReady;
    }
    //=============================================================================
    std::wstring
    getErrorMessage()
    {
        std::unique_lock<std::mutex> lock(resultMutex);
        resultCondition.wait(lock, [this]() { return resultReady; });
        return errorMessage;
    }
    //=============================================================================
    std::wstring
    getStandardOutput()
    {
        std::unique_lock<std::mutex> lock(resultMutex);
        resultCondition.wait(lock, [this]() { return resultReady; });
        return outputMessage;
    }
    //=============================================================================
private:
    std::mutex resultMutex;
    std::condition_variable resultCondition;
    std::wstring errorMessage;
    std::wstring outputMessage;
    bool resultReady = false;
    //=============================================================================
};

    PythonRunner runner;
    std::thread thread(
        [&runner, &globalDict, &commands]() { runner.runPythonCode(globalDict, commands); });
    thread.detach();
    while (!runner.isResultReady()) {
        // do others things
    }

commands[0] = “import time; time.sleep(10)” OK it works
but commands[0] = “import numpy as np” not working in a detached thread but ok if called directly without thread

Help will be really appreciated :slight_smile:

2

You should be holding the GIL while calling the Python API because Python isn’t threadsafe internally. The only safe exceptions are the calls to claim and release the GIL itself.

3

I already tried this but it does not work

   void
    runPythonCode(PyObject* globalDict, const wstringVector& commands)
    {

        std::string str = wstring_to_utf8(commands[0]) + "\n";
    
        PyGILState_STATE gstate = PyGILState_Ensure();

        PyRun_SimpleString(str.c_str());
       
        PyGILState_Release(gstate);

        resultReady = true;
        resultCondition.notify_one();
    }
4

Fixed :slight_smile: Thanks

Here full code

    void
    runPythonCode(PyObject* globalDict, const wstringVector& commands)
    {

        std::string str = wstring_to_utf8(commands[0]) + "\n";
    
        PyGILState_STATE gstate = PyGILState_Ensure();

        PyRun_SimpleString(str.c_str());
       
        PyGILState_Release(gstate);
        //outputMessage += getPythonStandardOutput();

        resultReady = true;
        resultCondition.notify_one();
    }

Important part:

    PythonRunner runner;

    Py_BEGIN_ALLOW_THREADS
    std::thread thread(
           [&runner, &globalDict, &commands]() { runner.runPythonCode(globalDict, commands); });
      thread.detach();

      while (!runner.isResultReady())
    {
        // do others things
    }
      Py_END_ALLOW_THREADS