1
2 /*
3 * Copyright (C) Igor Sysoev
4 * Copyright (C) NGINX, Inc.
5 */
6
7 #include <nxt_main.h>
8
9
10 static char *nxt_fiber_create_stack(nxt_task_t *task, nxt_fiber_t *fib);
11 static void nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent);
12 static void nxt_fiber_switch_handler(nxt_task_t *task, void *obj, void *data);
13 static void nxt_fiber_switch(nxt_task_t *task, nxt_fiber_t *fib);
14 static void nxt_fiber_timer_handler(nxt_task_t *task, void *obj, void *data);
15
16
17 #define nxt_fiber_enqueue(thr, task, fib) \
18 nxt_work_queue_add(&(thr)->engine->fast_work_queue, \
19 nxt_fiber_switch_handler, task, fib, NULL)
20
21
22 nxt_fiber_main_t *
nxt_fiber_main_create(nxt_event_engine_t * engine)23 nxt_fiber_main_create(nxt_event_engine_t *engine)
24 {
25 nxt_fiber_main_t *fm;
26
27 fm = nxt_zalloc(sizeof(nxt_fiber_main_t));
28 if (nxt_slow_path(fm == NULL)) {
29 return NULL;
30 }
31
32 fm->engine = engine;
33 fm->stack_size = 512 * 1024 - nxt_pagesize;
34 fm->idle = NULL;
35
36 return fm;
37 }
38
39
40 nxt_int_t
nxt_fiber_create(nxt_fiber_start_t start,void * data,size_t stack)41 nxt_fiber_create(nxt_fiber_start_t start, void *data, size_t stack)
42 {
43 int ret;
44 jmp_buf parent;
45 nxt_fid_t fid;
46 nxt_fiber_t *fib;
47 nxt_thread_t *thr;
48 nxt_fiber_main_t *fm;
49
50 thr = nxt_thread();
51 fm = thr->engine->fibers;
52
53 fid = ++fm->fid;
54
55 if (fid == 0) {
56 fid = ++fm->fid;
57 }
58
59 fib = fm->idle;
60
61 if (fib != NULL) {
62 fm->idle = fib->next;
63 fib->fid = fid;
64 fib->start = start;
65 fib->data = data;
66 fib->main = fm;
67
68 fib->task.thread = thr;
69 fib->task.log = thr->log;
70 fib->task.ident = nxt_task_next_ident();
71
72 nxt_debug(&fib->task, "fiber create cached: %PF", fib->fid);
73
74 nxt_fiber_enqueue(thr, &fm->engine->task, fib);
75
76 return NXT_OK;
77 }
78
79 nxt_log_debug(thr->log, "fiber create");
80
81 fib = nxt_malloc(sizeof(nxt_fiber_t));
82 if (nxt_slow_path(fib == NULL)) {
83 return NXT_ERROR;
84 }
85
86 fib->fid = fid;
87 fib->start = start;
88 fib->data = data;
89 fib->stack_size = fm->stack_size;
90 fib->main = fm;
91
92 fib->task.thread = thr;
93 fib->task.log = thr->log;
94 fib->task.ident = nxt_task_next_ident();
95
96 fib->stack = nxt_fiber_create_stack(&fib->task, fib);
97
98 if (nxt_fast_path(fib->stack != NULL)) {
99
100 if (_setjmp(parent) != 0) {
101 nxt_log_debug(thr->log, "fiber create: %PF", fib->fid);
102 return NXT_OK;
103 }
104
105 nxt_fiber_switch_stack(fib, &parent);
106 /* It does not return if the switch was successful. */
107 }
108
109 ret = munmap(fib->stack - nxt_pagesize, fib->stack_size + nxt_pagesize);
110
111 if (nxt_slow_path(ret != 0)) {
112 nxt_log_alert(thr->log, "munmap() failed %E", nxt_errno);
113 }
114
115 nxt_free(fib);
116
117 return NXT_ERROR;
118 }
119
120
121 #if (NXT_LINUX)
122
123 static char *
nxt_fiber_create_stack(nxt_task_t * task,nxt_fiber_t * fib)124 nxt_fiber_create_stack(nxt_task_t *task, nxt_fiber_t *fib)
125 {
126 char *s;
127 size_t size;
128
129 size = fib->stack_size + nxt_pagesize;
130
131 s = mmap(NULL, size, PROT_READ | PROT_WRITE,
132 MAP_PRIVATE | MAP_ANON | MAP_GROWSDOWN, -1, 0);
133
134 if (nxt_slow_path(s == MAP_FAILED)) {
135 nxt_alert(task, "fiber stack "
136 "mmap(%uz, MAP_PRIVATE|MAP_ANON|MAP_GROWSDOWN) failed %E",
137 size, nxt_errno);
138
139 return NULL;
140 }
141
142 if (nxt_slow_path(mprotect(s, nxt_pagesize, PROT_NONE) != 0)) {
143 nxt_alert(task, "fiber stack mprotect(%uz, PROT_NONE) failed %E",
144 size, nxt_errno);
145
146 return NULL;
147 }
148
149 s += nxt_pagesize;
150
151 nxt_debug(task, "fiber stack mmap: %p", s);
152
153 return s;
154 }
155
156 #else /* Generic version. */
157
158 static char *
nxt_fiber_create_stack(nxt_task_t * task,nxt_fiber_t * fib)159 nxt_fiber_create_stack(nxt_task_t *task, nxt_fiber_t *fib)
160 {
161 char *s;
162 size_t size;
163
164 size = fib->stack_size + nxt_pagesize;
165
166 s = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
167
168 if (nxt_slow_path(s == MAP_FAILED)) {
169 nxt_alert(task, "fiber stack mmap(%uz, MAP_PRIVATE|MAP_ANON) failed %E",
170 size, nxt_errno);
171
172 return NULL;
173 }
174
175 if (nxt_slow_path(mprotect(s, nxt_pagesize, PROT_NONE) != 0)) {
176 nxt_alert(task, "fiber stack mprotect(%uz, PROT_NONE) failed %E",
177 size, nxt_errno);
178
179 return NULL;
180 }
181
182 s += nxt_pagesize;
183
184 nxt_debug(task, "fiber stack mmap: %p", s);
185
186 return s;
187 }
188
189 #endif
190
191
192 #if (NXT_LINUX && NXT_64BIT)
193
194 /*
195 * Linux 64-bit ucontext version. 64-bit glibc makecontext() passes
196 * pointers as signed int's. The bug has been fixed in glibc 2.8.
197 */
198
199 static void nxt_fiber_trampoline(uint32_t fh, uint32_t fl, uint32_t ph,
200 uint32_t pl);
201
202
203 static void
nxt_fiber_switch_stack(nxt_fiber_t * fib,jmp_buf * parent)204 nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent)
205 {
206 ucontext_t uc;
207
208 nxt_debug(&fib->task, "fiber switch to stack: %p", fib->stack);
209
210 if (nxt_slow_path(getcontext(&uc) != 0)) {
211 nxt_alert(&fib->task, "getcontext() failed");
212 return;
213 }
214
215 uc.uc_link = NULL;
216 uc.uc_stack.ss_sp = fib->stack;
217 uc.uc_stack.ss_size = fib->stack_size;
218
219 makecontext(&uc, (void (*)(void)) nxt_fiber_trampoline, 4,
220 (uint32_t) ((uintptr_t) fib >> 32),
221 (uint32_t) ((uintptr_t) fib & 0xFFFFFFFF),
222 (uint32_t) ((uintptr_t) parent >> 32),
223 (uint32_t) ((uintptr_t) parent & 0xFFFFFFFF));
224
225 setcontext(&uc);
226
227 nxt_alert(&fib->task, "setcontext() failed");
228 }
229
230
231 static void
nxt_fiber_trampoline(uint32_t fh,uint32_t fl,uint32_t ph,uint32_t pl)232 nxt_fiber_trampoline(uint32_t fh, uint32_t fl, uint32_t ph, uint32_t pl)
233 {
234 jmp_buf *parent;
235 nxt_task_t *task;
236 nxt_fiber_t *fib;
237
238 fib = (nxt_fiber_t *) (((uintptr_t) fh << 32) + fl);
239 parent = (jmp_buf *) (((uintptr_t) ph << 32) + pl);
240
241 task = &fib->task;
242
243 if (_setjmp(fib->jmp) == 0) {
244 nxt_debug(task, "fiber return to parent stack");
245
246 nxt_fiber_enqueue(task->thread, task, fib);
247
248 _longjmp(*parent, 1);
249
250 nxt_unreachable();
251 }
252
253 nxt_debug(task, "fiber start");
254
255 fib->start(fib->data);
256
257 nxt_fiber_exit(task, &fib->main->fiber, NULL);
258
259 nxt_unreachable();
260 }
261
262 #elif (NXT_HAVE_UCONTEXT)
263
264 /* Generic ucontext version. */
265
266 static void nxt_fiber_trampoline(nxt_fiber_t *fib, jmp_buf *parent);
267
268
269 static void
nxt_fiber_switch_stack(nxt_fiber_t * fib,jmp_buf * parent)270 nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent)
271 {
272 ucontext_t uc;
273
274 nxt_debug(&fib->task, "fiber switch to stack: %p", fib->stack);
275
276 if (nxt_slow_path(getcontext(&uc) != 0)) {
277 nxt_alert(&fib->task, "getcontext() failed");
278 return;
279 }
280
281 uc.uc_link = NULL;
282 uc.uc_stack.ss_sp = fib->stack;
283 uc.uc_stack.ss_size = fib->stack_size;
284
285 makecontext(&uc, (void (*)(void)) nxt_fiber_trampoline, 2, fib, parent);
286
287 setcontext(&uc);
288
289 #if !(NXT_SOLARIS)
290 /* Solaris declares setcontext() as __NORETURN. */
291
292 nxt_alert(&fib->task, "setcontext() failed");
293 #endif
294 }
295
296
297 static void
nxt_fiber_trampoline(nxt_fiber_t * fib,jmp_buf * parent)298 nxt_fiber_trampoline(nxt_fiber_t *fib, jmp_buf *parent)
299 {
300 nxt_task_t *task;
301
302 task = &fib->task;
303
304 if (_setjmp(fib->jmp) == 0) {
305 nxt_debug(task, "fiber return to parent stack");
306
307 nxt_fiber_enqueue(task->thread, task, fib);
308
309 _longjmp(*parent, 1);
310
311 nxt_unreachable();
312 }
313
314 nxt_debug(task, "fiber start");
315
316 fib->start(fib->data);
317
318 nxt_fiber_exit(task, &fib->main->fiber, NULL);
319
320 nxt_unreachable();
321 }
322
323 #else
324
325 #error No ucontext(3) interface.
326
327 #endif
328
329
330 static void
nxt_fiber_switch_handler(nxt_task_t * task,void * obj,void * data)331 nxt_fiber_switch_handler(nxt_task_t *task, void *obj, void *data)
332 {
333 nxt_fiber_t *fib;
334
335 fib = obj;
336
337 nxt_fiber_switch(task, fib);
338 nxt_unreachable();
339 }
340
341
342 static void
nxt_fiber_switch(nxt_task_t * task,nxt_fiber_t * fib)343 nxt_fiber_switch(nxt_task_t *task, nxt_fiber_t *fib)
344 {
345 nxt_debug(task, "fiber switch: %PF", fib->fid);
346
347 task->thread->fiber = fib;
348
349 _longjmp(fib->jmp, 1);
350
351 nxt_unreachable();
352 }
353
354
355 nxt_fiber_t *
nxt_fiber_self(nxt_thread_t * thr)356 nxt_fiber_self(nxt_thread_t *thr)
357 {
358 return (nxt_fast_path(thr != NULL)) ? thr->fiber : NULL;
359 }
360
361
362 void
nxt_fiber_yield(nxt_task_t * task)363 nxt_fiber_yield(nxt_task_t *task)
364 {
365 nxt_fiber_t *fib;
366
367 fib = task->thread->fiber;
368
369 if (_setjmp(fib->jmp) == 0) {
370
371 nxt_debug(task, "fiber yield");
372
373 nxt_fiber_enqueue(task->thread, &fib->main->engine->task, fib);
374
375 nxt_fiber_switch(task, &fib->main->fiber);
376
377 nxt_unreachable();
378 }
379
380 nxt_debug(task, "fiber yield return");
381 }
382
383
384 void
nxt_fiber_sleep(nxt_task_t * task,nxt_msec_t timeout)385 nxt_fiber_sleep(nxt_task_t *task, nxt_msec_t timeout)
386 {
387 nxt_fiber_t *fib;
388
389 fib = task->thread->fiber;
390
391 fib->timer.work_queue = &task->thread->engine->fast_work_queue;
392 fib->timer.handler = nxt_fiber_timer_handler;
393 fib->timer.log = &nxt_main_log;
394
395 task = &fib->task;
396
397 nxt_timer_add(task->thread->engine, &fib->timer, timeout);
398
399 if (_setjmp(fib->jmp) == 0) {
400
401 nxt_debug(task, "fiber sleep: %T", timeout);
402
403 nxt_fiber_switch(task, &fib->main->fiber);
404
405 nxt_unreachable();
406 }
407
408 nxt_debug(task, "fiber sleep return");
409 }
410
411
412 static void
nxt_fiber_timer_handler(nxt_task_t * task,void * obj,void * data)413 nxt_fiber_timer_handler(nxt_task_t *task, void *obj, void *data)
414 {
415 nxt_fiber_t *fib;
416 nxt_timer_t *ev;
417
418 ev = obj;
419
420 nxt_debug(task, "fiber timer handler");
421
422 fib = nxt_timer_data(ev, nxt_fiber_t, timer);
423
424 nxt_fiber_switch(task, fib);
425
426 nxt_unreachable();
427 }
428
429
430 void
nxt_fiber_wait(nxt_task_t * task)431 nxt_fiber_wait(nxt_task_t *task)
432 {
433 nxt_fiber_t *fib;
434
435 fib = task->thread->fiber;
436
437 if (_setjmp(fib->jmp) == 0) {
438 nxt_debug(task, "fiber wait");
439
440 nxt_fiber_switch(task, &fib->main->fiber);
441
442 nxt_unreachable();
443 }
444
445 nxt_debug(task, "fiber wait return");
446 }
447
448
449 void
nxt_fiber_exit(nxt_task_t * task,nxt_fiber_t * next,void * data)450 nxt_fiber_exit(nxt_task_t *task, nxt_fiber_t *next, void *data)
451 {
452 nxt_fiber_t *fib;
453
454 fib = task->thread->fiber;
455
456 nxt_debug(task, "fiber exit");
457
458 /* TODO: limit idle fibers. */
459 fib->next = fib->main->idle;
460 fib->main->idle = fib;
461
462 nxt_fiber_switch(task, next);
463
464 nxt_unreachable();
465 }
466