Branch data Line data Source code
1 : : /*
2 : : * Implementation of the Global Interpreter Lock (GIL).
3 : : */
4 : :
5 : : #include <stdlib.h>
6 : : #include <errno.h>
7 : :
8 : : #include "pycore_atomic.h"
9 : :
10 : :
11 : : /*
12 : : Notes about the implementation:
13 : :
14 : : - The GIL is just a boolean variable (locked) whose access is protected
15 : : by a mutex (gil_mutex), and whose changes are signalled by a condition
16 : : variable (gil_cond). gil_mutex is taken for short periods of time,
17 : : and therefore mostly uncontended.
18 : :
19 : : - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be
20 : : able to release the GIL on demand by another thread. A volatile boolean
21 : : variable (gil_drop_request) is used for that purpose, which is checked
22 : : at every turn of the eval loop. That variable is set after a wait of
23 : : `interval` microseconds on `gil_cond` has timed out.
24 : :
25 : : [Actually, another volatile boolean variable (eval_breaker) is used
26 : : which ORs several conditions into one. Volatile booleans are
27 : : sufficient as inter-thread signalling means since Python is run
28 : : on cache-coherent architectures only.]
29 : :
30 : : - A thread wanting to take the GIL will first let pass a given amount of
31 : : time (`interval` microseconds) before setting gil_drop_request. This
32 : : encourages a defined switching period, but doesn't enforce it since
33 : : opcodes can take an arbitrary time to execute.
34 : :
35 : : The `interval` value is available for the user to read and modify
36 : : using the Python API `sys.{get,set}switchinterval()`.
37 : :
38 : : - When a thread releases the GIL and gil_drop_request is set, that thread
39 : : ensures that another GIL-awaiting thread gets scheduled.
40 : : It does so by waiting on a condition variable (switch_cond) until
41 : : the value of last_holder is changed to something else than its
42 : : own thread state pointer, indicating that another thread was able to
43 : : take the GIL.
44 : :
45 : : This is meant to prohibit the latency-adverse behaviour on multi-core
46 : : machines where one thread would speculatively release the GIL, but still
47 : : run and end up being the first to re-acquire it, making the "timeslices"
48 : : much longer than expected.
49 : : (Note: this mechanism is enabled with FORCE_SWITCHING above)
50 : : */
51 : :
52 : : #include "condvar.h"
53 : :
54 : : #define MUTEX_INIT(mut) \
55 : : if (PyMUTEX_INIT(&(mut))) { \
56 : : Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); };
57 : : #define MUTEX_FINI(mut) \
58 : : if (PyMUTEX_FINI(&(mut))) { \
59 : : Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); };
60 : : #define MUTEX_LOCK(mut) \
61 : : if (PyMUTEX_LOCK(&(mut))) { \
62 : : Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); };
63 : : #define MUTEX_UNLOCK(mut) \
64 : : if (PyMUTEX_UNLOCK(&(mut))) { \
65 : : Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); };
66 : :
67 : : #define COND_INIT(cond) \
68 : : if (PyCOND_INIT(&(cond))) { \
69 : : Py_FatalError("PyCOND_INIT(" #cond ") failed"); };
70 : : #define COND_FINI(cond) \
71 : : if (PyCOND_FINI(&(cond))) { \
72 : : Py_FatalError("PyCOND_FINI(" #cond ") failed"); };
73 : : #define COND_SIGNAL(cond) \
74 : : if (PyCOND_SIGNAL(&(cond))) { \
75 : : Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); };
76 : : #define COND_WAIT(cond, mut) \
77 : : if (PyCOND_WAIT(&(cond), &(mut))) { \
78 : : Py_FatalError("PyCOND_WAIT(" #cond ") failed"); };
79 : : #define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \
80 : : { \
81 : : int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \
82 : : if (r < 0) \
83 : : Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \
84 : : if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \
85 : : timeout_result = 1; \
86 : : else \
87 : : timeout_result = 0; \
88 : : } \
89 : :
90 : :
91 : : #define DEFAULT_INTERVAL 5000
92 : :
93 : 2988 : static void _gil_initialize(struct _gil_runtime_state *gil)
94 : : {
95 : 2988 : _Py_atomic_int uninitialized = {-1};
96 : 2988 : gil->locked = uninitialized;
97 : 2988 : gil->interval = DEFAULT_INTERVAL;
98 : 2988 : }
99 : :
100 : 2975 : static int gil_created(struct _gil_runtime_state *gil)
101 : : {
102 : 2975 : return (_Py_atomic_load_explicit(&gil->locked, _Py_memory_order_acquire) >= 0);
103 : : }
104 : :
105 : 2975 : static void create_gil(struct _gil_runtime_state *gil)
106 : : {
107 [ - + ]: 2975 : MUTEX_INIT(gil->mutex);
108 : : #ifdef FORCE_SWITCHING
109 [ - + ]: 2975 : MUTEX_INIT(gil->switch_mutex);
110 : : #endif
111 [ - + ]: 2975 : COND_INIT(gil->cond);
112 : : #ifdef FORCE_SWITCHING
113 [ - + ]: 2975 : COND_INIT(gil->switch_cond);
114 : : #endif
115 : 2975 : _Py_atomic_store_relaxed(&gil->last_holder, 0);
116 : : _Py_ANNOTATE_RWLOCK_CREATE(&gil->locked);
117 : 2975 : _Py_atomic_store_explicit(&gil->locked, 0, _Py_memory_order_release);
118 : 2975 : }
119 : :
120 : 0 : static void destroy_gil(struct _gil_runtime_state *gil)
121 : : {
122 : : /* some pthread-like implementations tie the mutex to the cond
123 : : * and must have the cond destroyed first.
124 : : */
125 [ # # ]: 0 : COND_FINI(gil->cond);
126 [ # # ]: 0 : MUTEX_FINI(gil->mutex);
127 : : #ifdef FORCE_SWITCHING
128 [ # # ]: 0 : COND_FINI(gil->switch_cond);
129 [ # # ]: 0 : MUTEX_FINI(gil->switch_mutex);
130 : : #endif
131 : 0 : _Py_atomic_store_explicit(&gil->locked, -1,
132 : : _Py_memory_order_release);
133 : : _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);
134 : 0 : }
135 : :
136 : 8 : static void recreate_gil(struct _gil_runtime_state *gil)
137 : : {
138 : : _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);
139 : : /* XXX should we destroy the old OS resources here? */
140 : 8 : create_gil(gil);
141 : 8 : }
142 : :
143 : : static void
144 : 9305820 : drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2,
145 : : PyThreadState *tstate)
146 : : {
147 : 9305820 : struct _gil_runtime_state *gil = &ceval->gil;
148 [ - + ]: 9305820 : if (!_Py_atomic_load_relaxed(&gil->locked)) {
149 : : Py_FatalError("drop_gil: GIL is not locked");
150 : : }
151 : :
152 : : /* tstate is allowed to be NULL (early interpreter init) */
153 [ + - ]: 9305820 : if (tstate != NULL) {
154 : : /* Sub-interpreter support: threads might have been switched
155 : : under our feet using PyThreadState_Swap(). Fix the GIL last
156 : : holder variable so that our heuristics work. */
157 : 9305820 : _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate);
158 : : }
159 : :
160 [ - + ]: 9305820 : MUTEX_LOCK(gil->mutex);
161 : : _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1);
162 : 9305820 : _Py_atomic_store_relaxed(&gil->locked, 0);
163 [ - + ]: 9305820 : COND_SIGNAL(gil->cond);
164 [ - + ]: 9305820 : MUTEX_UNLOCK(gil->mutex);
165 : :
166 : : #ifdef FORCE_SWITCHING
167 [ + + + - ]: 9305815 : if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request) && tstate != NULL) {
168 [ - + ]: 30825 : MUTEX_LOCK(gil->switch_mutex);
169 : : /* Not switched yet => wait */
170 [ + + ]: 30825 : if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate)
171 : : {
172 : : assert(is_tstate_valid(tstate));
173 : 27349 : RESET_GIL_DROP_REQUEST(tstate->interp);
174 : : /* NOTE: if COND_WAIT does not atomically start waiting when
175 : : releasing the mutex, another thread can run through, take
176 : : the GIL and drop it again, and reset the condition
177 : : before we even had a chance to wait for it. */
178 [ - + ]: 27349 : COND_WAIT(gil->switch_cond, gil->switch_mutex);
179 : : }
180 [ - + ]: 30825 : MUTEX_UNLOCK(gil->switch_mutex);
181 : : }
182 : : #endif
183 : 9305815 : }
184 : :
185 : :
186 : : /* Check if a Python thread must exit immediately, rather than taking the GIL
187 : : if Py_Finalize() has been called.
188 : :
189 : : When this function is called by a daemon thread after Py_Finalize() has been
190 : : called, the GIL does no longer exist.
191 : :
192 : : tstate must be non-NULL. */
193 : : static inline int
194 : 18708178 : tstate_must_exit(PyThreadState *tstate)
195 : : {
196 : : /* bpo-39877: Access _PyRuntime directly rather than using
197 : : tstate->interp->runtime to support calls from Python daemon threads.
198 : : After Py_Finalize() has been called, tstate can be a dangling pointer:
199 : : point to PyThreadState freed memory. */
200 : 18708178 : PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(&_PyRuntime);
201 [ + + + + ]: 18708122 : return (finalizing != NULL && finalizing != tstate);
202 : : }
203 : :
204 : :
205 : : /* Take the GIL.
206 : :
207 : : The function saves errno at entry and restores its value at exit.
208 : :
209 : : tstate must be non-NULL. */
210 : : static void
211 : 9308857 : take_gil(PyThreadState *tstate)
212 : : {
213 : 9308857 : int err = errno;
214 : :
215 : : assert(tstate != NULL);
216 : :
217 [ + + ]: 9308857 : if (tstate_must_exit(tstate)) {
218 : : /* bpo-39877: If Py_Finalize() has been called and tstate is not the
219 : : thread which called Py_Finalize(), exit immediately the thread.
220 : :
221 : : This code path can be reached by a daemon thread after Py_Finalize()
222 : : completes. In this case, tstate is a dangling pointer: points to
223 : : PyThreadState freed memory. */
224 : 69 : PyThread_exit_thread();
225 : : }
226 : :
227 : : assert(is_tstate_valid(tstate));
228 : 9308794 : PyInterpreterState *interp = tstate->interp;
229 : 9308794 : struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
230 : 9308794 : struct _ceval_state *ceval2 = &interp->ceval;
231 : 9308794 : struct _gil_runtime_state *gil = &ceval->gil;
232 : :
233 : : /* Check that _PyEval_InitThreads() was called to create the lock */
234 : : assert(gil_created(gil));
235 : :
236 [ - + ]: 9308794 : MUTEX_LOCK(gil->mutex);
237 : :
238 [ + + ]: 9308806 : if (!_Py_atomic_load_relaxed(&gil->locked)) {
239 : 9217510 : goto _ready;
240 : : }
241 : :
242 [ + + ]: 380992 : while (_Py_atomic_load_relaxed(&gil->locked)) {
243 : 198413 : unsigned long saved_switchnum = gil->switch_number;
244 : :
245 [ + - ]: 198413 : unsigned long interval = (gil->interval >= 1 ? gil->interval : 1);
246 : 198413 : int timed_out = 0;
247 [ - + + + ]: 198413 : COND_TIMED_WAIT(gil->cond, gil->mutex, interval, timed_out);
248 : :
249 : : /* If we timed out and no switch occurred in the meantime, it is time
250 : : to ask the GIL-holding thread to drop it. */
251 [ + + ]: 198413 : if (timed_out &&
252 [ + + ]: 92726 : _Py_atomic_load_relaxed(&gil->locked) &&
253 [ + + ]: 92255 : gil->switch_number == saved_switchnum)
254 : : {
255 [ + + ]: 90531 : if (tstate_must_exit(tstate)) {
256 [ - + ]: 13 : MUTEX_UNLOCK(gil->mutex);
257 : 13 : PyThread_exit_thread();
258 : : }
259 : : assert(is_tstate_valid(tstate));
260 : :
261 : 90518 : SET_GIL_DROP_REQUEST(interp);
262 : : }
263 : : }
264 : :
265 : 91283 : _ready:
266 : : #ifdef FORCE_SWITCHING
267 : : /* This mutex must be taken before modifying gil->last_holder:
268 : : see drop_gil(). */
269 [ - + ]: 9308793 : MUTEX_LOCK(gil->switch_mutex);
270 : : #endif
271 : : /* We now hold the GIL */
272 : 9308793 : _Py_atomic_store_relaxed(&gil->locked, 1);
273 : : _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil->locked, /*is_write=*/1);
274 : :
275 [ + + ]: 9308793 : if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) {
276 : 170847 : _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate);
277 : 170847 : ++gil->switch_number;
278 : : }
279 : :
280 : : #ifdef FORCE_SWITCHING
281 [ - + ]: 9308793 : COND_SIGNAL(gil->switch_cond);
282 [ - + ]: 9308793 : MUTEX_UNLOCK(gil->switch_mutex);
283 : : #endif
284 : :
285 [ - + ]: 9308793 : if (tstate_must_exit(tstate)) {
286 : : /* bpo-36475: If Py_Finalize() has been called and tstate is not
287 : : the thread which called Py_Finalize(), exit immediately the
288 : : thread.
289 : :
290 : : This code path can be reached by a daemon thread which was waiting
291 : : in take_gil() while the main thread called
292 : : wait_for_thread_shutdown() from Py_Finalize(). */
293 [ # # ]: 0 : MUTEX_UNLOCK(gil->mutex);
294 : 0 : drop_gil(ceval, ceval2, tstate);
295 : 0 : PyThread_exit_thread();
296 : : }
297 : : assert(is_tstate_valid(tstate));
298 : :
299 [ + + ]: 9308793 : if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request)) {
300 : 8081 : RESET_GIL_DROP_REQUEST(interp);
301 : : }
302 : : else {
303 : : /* bpo-40010: eval_breaker should be recomputed to be set to 1 if there
304 : : is a pending signal: signal received by another thread which cannot
305 : : handle signals.
306 : :
307 : : Note: RESET_GIL_DROP_REQUEST() calls COMPUTE_EVAL_BREAKER(). */
308 : 9300712 : COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
309 : : }
310 : :
311 : : /* Don't access tstate if the thread must exit */
312 [ + + ]: 9308793 : if (tstate->async_exc != NULL) {
313 : 1 : _PyEval_SignalAsyncExc(tstate->interp);
314 : : }
315 : :
316 [ - + ]: 9308793 : MUTEX_UNLOCK(gil->mutex);
317 : :
318 : 9308793 : errno = err;
319 : 9308793 : }
320 : :
321 : 133 : void _PyEval_SetSwitchInterval(unsigned long microseconds)
322 : : {
323 : 133 : struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil;
324 : 133 : gil->interval = microseconds;
325 : 133 : }
326 : :
327 : 20 : unsigned long _PyEval_GetSwitchInterval()
328 : : {
329 : 20 : struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil;
330 : 20 : return gil->interval;
331 : : }
|
