Enable/disable the GIL at runtime

The GIL is controlled by the environment variable PYTHONGIL or the flag
"-X nogil". The GIL is enabled by default. The interpreter will
currently crash when running multi-threaded programs without the GIL.

Author: colesbury

Include/cpython/initconfig.h

@@ -119,6 +119,8 @@ typedef struct PyPreConfig {
        environment variable. */
     int dev_mode;
 
+    int disable_gil;        /* Disable the global interpreter lock */
+
     /* Memory allocator: PYTHONMALLOC env var.
        See PyMemAllocatorName for valid values. */
     int allocator;
@@ -137,6 +139,7 @@ typedef struct PyConfig {
     int isolated;
     int use_environment;
     int dev_mode;
+    int disable_gil;
     int install_signal_handlers;
     int use_hash_seed;
     unsigned long hash_seed;

Include/internal/pycore_gil.h

@@ -21,6 +21,8 @@ extern "C" {
 #define FORCE_SWITCHING
 
 struct _gil_runtime_state {
+    /* Should we use the GIL? */
+    int enabled;
     /* microseconds (the Python API uses seconds, though) */
     unsigned long interval;
     /* Last PyThreadState holding / having held the GIL. This helps us

Include/internal/pycore_initconfig.h

@@ -103,13 +103,15 @@ typedef struct {
     int use_environment;      /* -E option */
     int dev_mode;             /* -X dev and PYTHONDEVMODE */
     int warn_default_encoding;     /* -X warn_default_encoding and PYTHONWARNDEFAULTENCODING */
+    int disable_gil;          /* -X nogil and PYTHONGIL=0 */
 } _PyPreCmdline;
 
 #define _PyPreCmdline_INIT \
     (_PyPreCmdline){ \
         .use_environment = -1, \
         .isolated = -1, \
-        .dev_mode = -1}
+        .dev_mode = -1, \
+        .disable_gil = -1}
 /* Note: _PyPreCmdline_INIT sets other fields to 0/NULL */
 
 extern void _PyPreCmdline_Clear(_PyPreCmdline *cmdline);

Lib/subprocess.py

@@ -345,6 +345,8 @@ def _args_from_interpreter_flags():
     # -X options
     if dev_mode:
         args.extend(('-X', 'dev'))
+    if sys.flags.nogil:
+        args.extend(('-X', 'nogil'))
     for opt in ('faulthandler', 'tracemalloc', 'importtime',
                 'showrefcount', 'utf8'):
         if opt in xoptions:

Lib/test/test_embed.py

@@ -426,11 +426,13 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         use_environment=0,
         utf8_mode=0,
         dev_mode=0,
+        disable_gil=0,
         coerce_c_locale=0,
     )
 
     COPY_PRE_CONFIG = [
         'dev_mode',
+        'disable_gil',
         'isolated',
         'use_environment',
     ]
@@ -440,6 +442,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'isolated': 0,
         'use_environment': 1,
         'dev_mode': 0,
+        'disable_gil': 0,
 
         'install_signal_handlers': 1,
         'use_hash_seed': 0,

Python/ceval_gil.c

@@ -235,14 +235,16 @@ static int gil_created(struct _gil_runtime_state *gil)
 static void create_gil(struct _gil_runtime_state *gil)
 {
     MUTEX_INIT(gil->mutex);
+    if (gil->enabled) {
 #ifdef FORCE_SWITCHING
-    MUTEX_INIT(gil->switch_mutex);
+        MUTEX_INIT(gil->switch_mutex);
 #endif
-    COND_INIT(gil->cond);
+        COND_INIT(gil->cond);
 #ifdef FORCE_SWITCHING
-    COND_INIT(gil->switch_cond);
+        COND_INIT(gil->switch_cond);
 #endif
-    _Py_atomic_store_relaxed(&gil->last_holder, 0);
+        _Py_atomic_store_relaxed(&gil->last_holder, 0);
+    }
     _Py_ANNOTATE_RWLOCK_CREATE(&gil->locked);
     _Py_atomic_store_explicit(&gil->locked, 0, _Py_memory_order_release);
 }
@@ -252,12 +254,14 @@ static void destroy_gil(struct _gil_runtime_state *gil)
     /* some pthread-like implementations tie the mutex to the cond
      * and must have the cond destroyed first.
      */
-    COND_FINI(gil->cond);
-    MUTEX_FINI(gil->mutex);
+    if (gil->enabled) {
+        COND_FINI(gil->cond);
+        MUTEX_FINI(gil->mutex);
 #ifdef FORCE_SWITCHING
-    COND_FINI(gil->switch_cond);
-    MUTEX_FINI(gil->switch_mutex);
+        COND_FINI(gil->switch_cond);
+        MUTEX_FINI(gil->switch_mutex);
 #endif
+    }
     _Py_atomic_store_explicit(&gil->locked, -1,
                               _Py_memory_order_release);
     _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);
@@ -277,6 +281,9 @@ drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2,
          PyThreadState *tstate)
 {
     struct _gil_runtime_state *gil = &ceval->gil;
+    if (!gil->enabled) {
+        return;
+    }
     if (!_Py_atomic_load_relaxed(&gil->locked)) {
         Py_FatalError("drop_gil: GIL is not locked");
     }
@@ -362,6 +369,10 @@ take_gil(PyThreadState *tstate)
     struct _ceval_state *ceval2 = &interp->ceval;
     struct _gil_runtime_state *gil = &ceval->gil;
 
+    if (!gil->enabled) {
+        return;
+    }
+
     /* Check that _PyEval_InitThreads() was called to create the lock */
     assert(gil_created(gil));
 
@@ -498,6 +509,7 @@ _PyEval_InitGIL(PyThreadState *tstate)
     }
 
     struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil;
+    gil->enabled = !_PyRuntime.preconfig.disable_gil;
     assert(!gil_created(gil));
 
     PyThread_init_thread();
@@ -569,6 +581,9 @@ PyEval_ReleaseLock(void)
 void
 _PyEval_ReleaseLock(PyThreadState *tstate)
 {
+    if (!_PyRuntime.ceval.gil.enabled) {
+        return;
+    }
     struct _ceval_runtime_state *ceval = &tstate->interp->runtime->ceval;
     struct _ceval_state *ceval2 = &tstate->interp->ceval;
     drop_gil(ceval, ceval2, tstate);

Python/initconfig.c

@@ -656,6 +656,7 @@ config_check_consistency(const PyConfig *config)
     assert(config->isolated >= 0);
     assert(config->use_environment >= 0);
     assert(config->dev_mode >= 0);
+    assert(config->disable_gil >= 0);
     assert(config->install_signal_handlers >= 0);
     assert(config->use_hash_seed >= 0);
     assert(config->hash_seed <= MAX_HASH_SEED);
@@ -763,6 +764,7 @@ _PyConfig_InitCompatConfig(PyConfig *config)
     config->isolated = -1;
     config->use_environment = -1;
     config->dev_mode = -1;
+    config->disable_gil = -1;
     config->install_signal_handlers = 1;
     config->use_hash_seed = -1;
     config->faulthandler = -1;
@@ -848,6 +850,7 @@ PyConfig_InitIsolatedConfig(PyConfig *config)
     config->use_environment = 0;
     config->user_site_directory = 0;
     config->dev_mode = 0;
+    config->disable_gil = 0;
     config->install_signal_handlers = 0;
     config->use_hash_seed = 0;
     config->faulthandler = 0;
@@ -959,6 +962,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_ATTR(isolated);
     COPY_ATTR(use_environment);
     COPY_ATTR(dev_mode);
+    COPY_ATTR(disable_gil);
     COPY_ATTR(install_signal_handlers);
     COPY_ATTR(use_hash_seed);
     COPY_ATTR(hash_seed);
@@ -1071,6 +1075,7 @@ _PyConfig_AsDict(const PyConfig *config)
     SET_ITEM_INT(isolated);
     SET_ITEM_INT(use_environment);
     SET_ITEM_INT(dev_mode);
+    SET_ITEM_INT(disable_gil);
     SET_ITEM_INT(install_signal_handlers);
     SET_ITEM_INT(use_hash_seed);
     SET_ITEM_UINT(hash_seed);
@@ -2329,6 +2334,7 @@ _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime)
     preconfig->isolated = config->isolated;
     preconfig->use_environment = config->use_environment;
     preconfig->dev_mode = config->dev_mode;
+    preconfig->disable_gil = config->disable_gil;
 
     if (_Py_SetArgcArgv(config->orig_argv.length,
                         config->orig_argv.items) < 0)

Python/preconfig.c

@@ -138,6 +138,7 @@ precmdline_get_preconfig(_PyPreCmdline *cmdline, const PyPreConfig *config)
     COPY_ATTR(isolated);
     COPY_ATTR(use_environment);
     COPY_ATTR(dev_mode);
+    COPY_ATTR(disable_gil);
 
 #undef COPY_ATTR
 }
@@ -152,6 +153,7 @@ precmdline_set_preconfig(const _PyPreCmdline *cmdline, PyPreConfig *config)
     COPY_ATTR(isolated);
     COPY_ATTR(use_environment);
     COPY_ATTR(dev_mode);
+    COPY_ATTR(disable_gil);
 
 #undef COPY_ATTR
 }
@@ -171,6 +173,7 @@ _PyPreCmdline_SetConfig(const _PyPreCmdline *cmdline, PyConfig *config)
     COPY_ATTR(isolated);
     COPY_ATTR(use_environment);
     COPY_ATTR(dev_mode);
+    COPY_ATTR(disable_gil);
     COPY_ATTR(warn_default_encoding);
     return _PyStatus_OK();
 
@@ -267,9 +270,24 @@ _PyPreCmdline_Read(_PyPreCmdline *cmdline, const PyPreConfig *preconfig)
         cmdline->warn_default_encoding = 1;
     }
 
+    /* GIL */
+    if (cmdline->disable_gil < 0) {
+        const char* env = NULL;
+        if (_Py_get_xoption(&cmdline->xoptions, L"nogil")) {
+            cmdline->disable_gil = 1;
+        }
+        else if ((env = _Py_GetEnv(cmdline->use_environment, "PYTHONGIL"))) {
+            cmdline->disable_gil = (strcmp(env, "0") == 0);
+        }
+        else {
+            cmdline->disable_gil = 0;
+        }
+    }
+
     assert(cmdline->use_environment >= 0);
     assert(cmdline->isolated >= 0);
     assert(cmdline->dev_mode >= 0);
+    assert(cmdline->disable_gil >= 0);
     assert(cmdline->warn_default_encoding >= 0);
 
     return _PyStatus_OK();
@@ -300,6 +318,7 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config)
     config->coerce_c_locale_warn = 0;
 
     config->dev_mode = -1;
+    config->disable_gil = -1;
     config->allocator = PYMEM_ALLOCATOR_NOT_SET;
 #ifdef MS_WINDOWS
     config->legacy_windows_fs_encoding = -1;
@@ -386,6 +405,7 @@ preconfig_copy(PyPreConfig *config, const PyPreConfig *config2)
     COPY_ATTR(use_environment);
     COPY_ATTR(configure_locale);
     COPY_ATTR(dev_mode);
+    COPY_ATTR(disable_gil);
     COPY_ATTR(coerce_c_locale);
     COPY_ATTR(coerce_c_locale_warn);
     COPY_ATTR(utf8_mode);
@@ -433,6 +453,7 @@ _PyPreConfig_AsDict(const PyPreConfig *config)
     SET_ITEM_INT(legacy_windows_fs_encoding);
 #endif
     SET_ITEM_INT(dev_mode);
+    SET_ITEM_INT(disable_gil);
     SET_ITEM_INT(allocator);
     return dict;
 
@@ -456,6 +477,7 @@ _PyPreConfig_GetConfig(PyPreConfig *preconfig, const PyConfig *config)
     COPY_ATTR(isolated);
     COPY_ATTR(use_environment);
     COPY_ATTR(dev_mode);
+    COPY_ATTR(disable_gil);
 
 #undef COPY_ATTR
 }

Python/sysmodule.c

@@ -2726,6 +2726,7 @@ static PyStructSequence_Field flags_fields[] = {
     {"hash_randomization",      "-R"},
     {"isolated",                "-I"},
     {"dev_mode",                "-X dev"},
+    {"nogil",                   "-X nogil"},
     {"utf8_mode",               "-X utf8"},
     {"warn_default_encoding",   "-X warn_default_encoding"},
     {"safe_path", "-P"},
@@ -2775,6 +2776,7 @@ set_flags_from_config(PyInterpreterState *interp, PyObject *flags)
     SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0);
     SetFlag(config->isolated);
     SetFlagObj(PyBool_FromLong(config->dev_mode));
+    SetFlagObj(PyBool_FromLong(config->disable_gil));
     SetFlag(preconfig->utf8_mode);
     SetFlag(config->warn_default_encoding);
     SetFlagObj(PyBool_FromLong(config->safe_path));