Back to home page

Nginx displayed by LXR

Source navigation ]
Diff markup ]
Identifier search ]
general search ]
 
 
Version: nginx-1.13.12 ]​[ nginx-1.12.2 ]​

0001 
0002 /*
0003  * Copyright (C) Igor Sysoev
0004  * Copyright (C) Nginx, Inc.
0005  */
0006 
0007 
0008 #include <ngx_config.h>
0009 #include <ngx_core.h>
0010 #include <ngx_event.h>
0011 
0012 
0013 #if (NGX_TEST_BUILD_EPOLL)
0014 
0015 /* epoll declarations */
0016 
0017 #define EPOLLIN        0x001
0018 #define EPOLLPRI       0x002
0019 #define EPOLLOUT       0x004
0020 #define EPOLLERR       0x008
0021 #define EPOLLHUP       0x010
0022 #define EPOLLRDNORM    0x040
0023 #define EPOLLRDBAND    0x080
0024 #define EPOLLWRNORM    0x100
0025 #define EPOLLWRBAND    0x200
0026 #define EPOLLMSG       0x400
0027 
0028 #define EPOLLRDHUP     0x2000
0029 
0030 #define EPOLLEXCLUSIVE 0x10000000
0031 #define EPOLLONESHOT   0x40000000
0032 #define EPOLLET        0x80000000
0033 
0034 #define EPOLL_CTL_ADD  1
0035 #define EPOLL_CTL_DEL  2
0036 #define EPOLL_CTL_MOD  3
0037 
0038 typedef union epoll_data {
0039     void         *ptr;
0040     int           fd;
0041     uint32_t      u32;
0042     uint64_t      u64;
0043 } epoll_data_t;
0044 
0045 struct epoll_event {
0046     uint32_t      events;
0047     epoll_data_t  data;
0048 };
0049 
0050 
0051 int epoll_create(int size);
0052 
0053 int epoll_create(int size)
0054 {
0055     return -1;
0056 }
0057 
0058 
0059 int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
0060 
0061 int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event)
0062 {
0063     return -1;
0064 }
0065 
0066 
0067 int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout);
0068 
0069 int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout)
0070 {
0071     return -1;
0072 }
0073 
0074 #if (NGX_HAVE_EVENTFD)
0075 #define SYS_eventfd       323
0076 #endif
0077 
0078 #if (NGX_HAVE_FILE_AIO)
0079 
0080 #define SYS_io_setup      245
0081 #define SYS_io_destroy    246
0082 #define SYS_io_getevents  247
0083 
0084 typedef u_int  aio_context_t;
0085 
0086 struct io_event {
0087     uint64_t  data;  /* the data field from the iocb */
0088     uint64_t  obj;   /* what iocb this event came from */
0089     int64_t   res;   /* result code for this event */
0090     int64_t   res2;  /* secondary result */
0091 };
0092 
0093 
0094 #endif
0095 #endif /* NGX_TEST_BUILD_EPOLL */
0096 
0097 
0098 typedef struct {
0099     ngx_uint_t  events;
0100     ngx_uint_t  aio_requests;
0101 } ngx_epoll_conf_t;
0102 
0103 
0104 static ngx_int_t ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer);
0105 #if (NGX_HAVE_EVENTFD)
0106 static ngx_int_t ngx_epoll_notify_init(ngx_log_t *log);
0107 static void ngx_epoll_notify_handler(ngx_event_t *ev);
0108 #endif
0109 #if (NGX_HAVE_EPOLLRDHUP)
0110 static void ngx_epoll_test_rdhup(ngx_cycle_t *cycle);
0111 #endif
0112 static void ngx_epoll_done(ngx_cycle_t *cycle);
0113 static ngx_int_t ngx_epoll_add_event(ngx_event_t *ev, ngx_int_t event,
0114     ngx_uint_t flags);
0115 static ngx_int_t ngx_epoll_del_event(ngx_event_t *ev, ngx_int_t event,
0116     ngx_uint_t flags);
0117 static ngx_int_t ngx_epoll_add_connection(ngx_connection_t *c);
0118 static ngx_int_t ngx_epoll_del_connection(ngx_connection_t *c,
0119     ngx_uint_t flags);
0120 #if (NGX_HAVE_EVENTFD)
0121 static ngx_int_t ngx_epoll_notify(ngx_event_handler_pt handler);
0122 #endif
0123 static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
0124     ngx_uint_t flags);
0125 
0126 #if (NGX_HAVE_FILE_AIO)
0127 static void ngx_epoll_eventfd_handler(ngx_event_t *ev);
0128 #endif
0129 
0130 static void *ngx_epoll_create_conf(ngx_cycle_t *cycle);
0131 static char *ngx_epoll_init_conf(ngx_cycle_t *cycle, void *conf);
0132 
0133 static int                  ep = -1;
0134 static struct epoll_event  *event_list;
0135 static ngx_uint_t           nevents;
0136 
0137 #if (NGX_HAVE_EVENTFD)
0138 static int                  notify_fd = -1;
0139 static ngx_event_t          notify_event;
0140 static ngx_connection_t     notify_conn;
0141 #endif
0142 
0143 #if (NGX_HAVE_FILE_AIO)
0144 
0145 int                         ngx_eventfd = -1;
0146 aio_context_t               ngx_aio_ctx = 0;
0147 
0148 static ngx_event_t          ngx_eventfd_event;
0149 static ngx_connection_t     ngx_eventfd_conn;
0150 
0151 #endif
0152 
0153 #if (NGX_HAVE_EPOLLRDHUP)
0154 ngx_uint_t                  ngx_use_epoll_rdhup;
0155 #endif
0156 
0157 static ngx_str_t      epoll_name = ngx_string("epoll");
0158 
0159 static ngx_command_t  ngx_epoll_commands[] = {
0160 
0161     { ngx_string("epoll_events"),
0162       NGX_EVENT_CONF|NGX_CONF_TAKE1,
0163       ngx_conf_set_num_slot,
0164       0,
0165       offsetof(ngx_epoll_conf_t, events),
0166       NULL },
0167 
0168     { ngx_string("worker_aio_requests"),
0169       NGX_EVENT_CONF|NGX_CONF_TAKE1,
0170       ngx_conf_set_num_slot,
0171       0,
0172       offsetof(ngx_epoll_conf_t, aio_requests),
0173       NULL },
0174 
0175       ngx_null_command
0176 };
0177 
0178 
0179 static ngx_event_module_t  ngx_epoll_module_ctx = {
0180     &epoll_name,
0181     ngx_epoll_create_conf,               /* create configuration */
0182     ngx_epoll_init_conf,                 /* init configuration */
0183 
0184     {
0185         ngx_epoll_add_event,             /* add an event */
0186         ngx_epoll_del_event,             /* delete an event */
0187         ngx_epoll_add_event,             /* enable an event */
0188         ngx_epoll_del_event,             /* disable an event */
0189         ngx_epoll_add_connection,        /* add an connection */
0190         ngx_epoll_del_connection,        /* delete an connection */
0191 #if (NGX_HAVE_EVENTFD)
0192         ngx_epoll_notify,                /* trigger a notify */
0193 #else
0194         NULL,                            /* trigger a notify */
0195 #endif
0196         ngx_epoll_process_events,        /* process the events */
0197         ngx_epoll_init,                  /* init the events */
0198         ngx_epoll_done,                  /* done the events */
0199     }
0200 };
0201 
0202 ngx_module_t  ngx_epoll_module = {
0203     NGX_MODULE_V1,
0204     &ngx_epoll_module_ctx,               /* module context */
0205     ngx_epoll_commands,                  /* module directives */
0206     NGX_EVENT_MODULE,                    /* module type */
0207     NULL,                                /* init master */
0208     NULL,                                /* init module */
0209     NULL,                                /* init process */
0210     NULL,                                /* init thread */
0211     NULL,                                /* exit thread */
0212     NULL,                                /* exit process */
0213     NULL,                                /* exit master */
0214     NGX_MODULE_V1_PADDING
0215 };
0216 
0217 
0218 #if (NGX_HAVE_FILE_AIO)
0219 
0220 /*
0221  * We call io_setup(), io_destroy() io_submit(), and io_getevents() directly
0222  * as syscalls instead of libaio usage, because the library header file
0223  * supports eventfd() since 0.3.107 version only.
0224  */
0225 
0226 static int
0227 io_setup(u_int nr_reqs, aio_context_t *ctx)
0228 {
0229     return syscall(SYS_io_setup, nr_reqs, ctx);
0230 }
0231 
0232 
0233 static int
0234 io_destroy(aio_context_t ctx)
0235 {
0236     return syscall(SYS_io_destroy, ctx);
0237 }
0238 
0239 
0240 static int
0241 io_getevents(aio_context_t ctx, long min_nr, long nr, struct io_event *events,
0242     struct timespec *tmo)
0243 {
0244     return syscall(SYS_io_getevents, ctx, min_nr, nr, events, tmo);
0245 }
0246 
0247 
0248 static void
0249 ngx_epoll_aio_init(ngx_cycle_t *cycle, ngx_epoll_conf_t *epcf)
0250 {
0251     int                 n;
0252     struct epoll_event  ee;
0253 
0254 #if (NGX_HAVE_SYS_EVENTFD_H)
0255     ngx_eventfd = eventfd(0, 0);
0256 #else
0257     ngx_eventfd = syscall(SYS_eventfd, 0);
0258 #endif
0259 
0260     if (ngx_eventfd == -1) {
0261         ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
0262                       "eventfd() failed");
0263         ngx_file_aio = 0;
0264         return;
0265     }
0266 
0267     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
0268                    "eventfd: %d", ngx_eventfd);
0269 
0270     n = 1;
0271 
0272     if (ioctl(ngx_eventfd, FIONBIO, &n) == -1) {
0273         ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
0274                       "ioctl(eventfd, FIONBIO) failed");
0275         goto failed;
0276     }
0277 
0278     if (io_setup(epcf->aio_requests, &ngx_aio_ctx) == -1) {
0279         ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
0280                       "io_setup() failed");
0281         goto failed;
0282     }
0283 
0284     ngx_eventfd_event.data = &ngx_eventfd_conn;
0285     ngx_eventfd_event.handler = ngx_epoll_eventfd_handler;
0286     ngx_eventfd_event.log = cycle->log;
0287     ngx_eventfd_event.active = 1;
0288     ngx_eventfd_conn.fd = ngx_eventfd;
0289     ngx_eventfd_conn.read = &ngx_eventfd_event;
0290     ngx_eventfd_conn.log = cycle->log;
0291 
0292     ee.events = EPOLLIN|EPOLLET;
0293     ee.data.ptr = &ngx_eventfd_conn;
0294 
0295     if (epoll_ctl(ep, EPOLL_CTL_ADD, ngx_eventfd, &ee) != -1) {
0296         return;
0297     }
0298 
0299     ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
0300                   "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed");
0301 
0302     if (io_destroy(ngx_aio_ctx) == -1) {
0303         ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0304                       "io_destroy() failed");
0305     }
0306 
0307 failed:
0308 
0309     if (close(ngx_eventfd) == -1) {
0310         ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0311                       "eventfd close() failed");
0312     }
0313 
0314     ngx_eventfd = -1;
0315     ngx_aio_ctx = 0;
0316     ngx_file_aio = 0;
0317 }
0318 
0319 #endif
0320 
0321 
0322 static ngx_int_t
0323 ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer)
0324 {
0325     ngx_epoll_conf_t  *epcf;
0326 
0327     epcf = ngx_event_get_conf(cycle->conf_ctx, ngx_epoll_module);
0328 
0329     if (ep == -1) {
0330         ep = epoll_create(cycle->connection_n / 2);
0331 
0332         if (ep == -1) {
0333             ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
0334                           "epoll_create() failed");
0335             return NGX_ERROR;
0336         }
0337 
0338 #if (NGX_HAVE_EVENTFD)
0339         if (ngx_epoll_notify_init(cycle->log) != NGX_OK) {
0340             ngx_epoll_module_ctx.actions.notify = NULL;
0341         }
0342 #endif
0343 
0344 #if (NGX_HAVE_FILE_AIO)
0345         ngx_epoll_aio_init(cycle, epcf);
0346 #endif
0347 
0348 #if (NGX_HAVE_EPOLLRDHUP)
0349         ngx_epoll_test_rdhup(cycle);
0350 #endif
0351     }
0352 
0353     if (nevents < epcf->events) {
0354         if (event_list) {
0355             ngx_free(event_list);
0356         }
0357 
0358         event_list = ngx_alloc(sizeof(struct epoll_event) * epcf->events,
0359                                cycle->log);
0360         if (event_list == NULL) {
0361             return NGX_ERROR;
0362         }
0363     }
0364 
0365     nevents = epcf->events;
0366 
0367     ngx_io = ngx_os_io;
0368 
0369     ngx_event_actions = ngx_epoll_module_ctx.actions;
0370 
0371 #if (NGX_HAVE_CLEAR_EVENT)
0372     ngx_event_flags = NGX_USE_CLEAR_EVENT
0373 #else
0374     ngx_event_flags = NGX_USE_LEVEL_EVENT
0375 #endif
0376                       |NGX_USE_GREEDY_EVENT
0377                       |NGX_USE_EPOLL_EVENT;
0378 
0379     return NGX_OK;
0380 }
0381 
0382 
0383 #if (NGX_HAVE_EVENTFD)
0384 
0385 static ngx_int_t
0386 ngx_epoll_notify_init(ngx_log_t *log)
0387 {
0388     struct epoll_event  ee;
0389 
0390 #if (NGX_HAVE_SYS_EVENTFD_H)
0391     notify_fd = eventfd(0, 0);
0392 #else
0393     notify_fd = syscall(SYS_eventfd, 0);
0394 #endif
0395 
0396     if (notify_fd == -1) {
0397         ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "eventfd() failed");
0398         return NGX_ERROR;
0399     }
0400 
0401     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, log, 0,
0402                    "notify eventfd: %d", notify_fd);
0403 
0404     notify_event.handler = ngx_epoll_notify_handler;
0405     notify_event.log = log;
0406     notify_event.active = 1;
0407 
0408     notify_conn.fd = notify_fd;
0409     notify_conn.read = &notify_event;
0410     notify_conn.log = log;
0411 
0412     ee.events = EPOLLIN|EPOLLET;
0413     ee.data.ptr = &notify_conn;
0414 
0415     if (epoll_ctl(ep, EPOLL_CTL_ADD, notify_fd, &ee) == -1) {
0416         ngx_log_error(NGX_LOG_EMERG, log, ngx_errno,
0417                       "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed");
0418 
0419         if (close(notify_fd) == -1) {
0420             ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
0421                             "eventfd close() failed");
0422         }
0423 
0424         return NGX_ERROR;
0425     }
0426 
0427     return NGX_OK;
0428 }
0429 
0430 
0431 static void
0432 ngx_epoll_notify_handler(ngx_event_t *ev)
0433 {
0434     ssize_t               n;
0435     uint64_t              count;
0436     ngx_err_t             err;
0437     ngx_event_handler_pt  handler;
0438 
0439     if (++ev->index == NGX_MAX_UINT32_VALUE) {
0440         ev->index = 0;
0441 
0442         n = read(notify_fd, &count, sizeof(uint64_t));
0443 
0444         err = ngx_errno;
0445 
0446         ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
0447                        "read() eventfd %d: %z count:%uL", notify_fd, n, count);
0448 
0449         if ((size_t) n != sizeof(uint64_t)) {
0450             ngx_log_error(NGX_LOG_ALERT, ev->log, err,
0451                           "read() eventfd %d failed", notify_fd);
0452         }
0453     }
0454 
0455     handler = ev->data;
0456     handler(ev);
0457 }
0458 
0459 #endif
0460 
0461 
0462 #if (NGX_HAVE_EPOLLRDHUP)
0463 
0464 static void
0465 ngx_epoll_test_rdhup(ngx_cycle_t *cycle)
0466 {
0467     int                 s[2], events;
0468     struct epoll_event  ee;
0469 
0470     if (socketpair(AF_UNIX, SOCK_STREAM, 0, s) == -1) {
0471         ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0472                       "socketpair() failed");
0473         return;
0474     }
0475 
0476     ee.events = EPOLLET|EPOLLIN|EPOLLRDHUP;
0477 
0478     if (epoll_ctl(ep, EPOLL_CTL_ADD, s[0], &ee) == -1) {
0479         ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0480                       "epoll_ctl() failed");
0481         goto failed;
0482     }
0483 
0484     if (close(s[1]) == -1) {
0485         ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0486                       "close() failed");
0487         s[1] = -1;
0488         goto failed;
0489     }
0490 
0491     s[1] = -1;
0492 
0493     events = epoll_wait(ep, &ee, 1, 5000);
0494 
0495     if (events == -1) {
0496         ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0497                       "epoll_wait() failed");
0498         goto failed;
0499     }
0500 
0501     if (events) {
0502         ngx_use_epoll_rdhup = ee.events & EPOLLRDHUP;
0503 
0504     } else {
0505         ngx_log_error(NGX_LOG_ALERT, cycle->log, NGX_ETIMEDOUT,
0506                       "epoll_wait() timed out");
0507     }
0508 
0509     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
0510                    "testing the EPOLLRDHUP flag: %s",
0511                    ngx_use_epoll_rdhup ? "success" : "fail");
0512 
0513 failed:
0514 
0515     if (s[1] != -1 && close(s[1]) == -1) {
0516         ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0517                       "close() failed");
0518     }
0519 
0520     if (close(s[0]) == -1) {
0521         ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0522                       "close() failed");
0523     }
0524 }
0525 
0526 #endif
0527 
0528 
0529 static void
0530 ngx_epoll_done(ngx_cycle_t *cycle)
0531 {
0532     if (close(ep) == -1) {
0533         ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0534                       "epoll close() failed");
0535     }
0536 
0537     ep = -1;
0538 
0539 #if (NGX_HAVE_EVENTFD)
0540 
0541     if (close(notify_fd) == -1) {
0542         ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0543                       "eventfd close() failed");
0544     }
0545 
0546     notify_fd = -1;
0547 
0548 #endif
0549 
0550 #if (NGX_HAVE_FILE_AIO)
0551 
0552     if (ngx_eventfd != -1) {
0553 
0554         if (io_destroy(ngx_aio_ctx) == -1) {
0555             ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0556                           "io_destroy() failed");
0557         }
0558 
0559         if (close(ngx_eventfd) == -1) {
0560             ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
0561                           "eventfd close() failed");
0562         }
0563 
0564         ngx_eventfd = -1;
0565     }
0566 
0567     ngx_aio_ctx = 0;
0568 
0569 #endif
0570 
0571     ngx_free(event_list);
0572 
0573     event_list = NULL;
0574     nevents = 0;
0575 }
0576 
0577 
0578 static ngx_int_t
0579 ngx_epoll_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
0580 {
0581     int                  op;
0582     uint32_t             events, prev;
0583     ngx_event_t         *e;
0584     ngx_connection_t    *c;
0585     struct epoll_event   ee;
0586 
0587     c = ev->data;
0588 
0589     events = (uint32_t) event;
0590 
0591     if (event == NGX_READ_EVENT) {
0592         e = c->write;
0593         prev = EPOLLOUT;
0594 #if (NGX_READ_EVENT != EPOLLIN|EPOLLRDHUP)
0595         events = EPOLLIN|EPOLLRDHUP;
0596 #endif
0597 
0598     } else {
0599         e = c->read;
0600         prev = EPOLLIN|EPOLLRDHUP;
0601 #if (NGX_WRITE_EVENT != EPOLLOUT)
0602         events = EPOLLOUT;
0603 #endif
0604     }
0605 
0606     if (e->active) {
0607         op = EPOLL_CTL_MOD;
0608         events |= prev;
0609 
0610     } else {
0611         op = EPOLL_CTL_ADD;
0612     }
0613 
0614 #if (NGX_HAVE_EPOLLEXCLUSIVE && NGX_HAVE_EPOLLRDHUP)
0615     if (flags & NGX_EXCLUSIVE_EVENT) {
0616         events &= ~EPOLLRDHUP;
0617     }
0618 #endif
0619 
0620     ee.events = events | (uint32_t) flags;
0621     ee.data.ptr = (void *) ((uintptr_t) c | ev->instance);
0622 
0623     ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
0624                    "epoll add event: fd:%d op:%d ev:%08XD",
0625                    c->fd, op, ee.events);
0626 
0627     if (epoll_ctl(ep, op, c->fd, &ee) == -1) {
0628         ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
0629                       "epoll_ctl(%d, %d) failed", op, c->fd);
0630         return NGX_ERROR;
0631     }
0632 
0633     ev->active = 1;
0634 #if 0
0635     ev->oneshot = (flags & NGX_ONESHOT_EVENT) ? 1 : 0;
0636 #endif
0637 
0638     return NGX_OK;
0639 }
0640 
0641 
0642 static ngx_int_t
0643 ngx_epoll_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
0644 {
0645     int                  op;
0646     uint32_t             prev;
0647     ngx_event_t         *e;
0648     ngx_connection_t    *c;
0649     struct epoll_event   ee;
0650 
0651     /*
0652      * when the file descriptor is closed, the epoll automatically deletes
0653      * it from its queue, so we do not need to delete explicitly the event
0654      * before the closing the file descriptor
0655      */
0656 
0657     if (flags & NGX_CLOSE_EVENT) {
0658         ev->active = 0;
0659         return NGX_OK;
0660     }
0661 
0662     c = ev->data;
0663 
0664     if (event == NGX_READ_EVENT) {
0665         e = c->write;
0666         prev = EPOLLOUT;
0667 
0668     } else {
0669         e = c->read;
0670         prev = EPOLLIN|EPOLLRDHUP;
0671     }
0672 
0673     if (e->active) {
0674         op = EPOLL_CTL_MOD;
0675         ee.events = prev | (uint32_t) flags;
0676         ee.data.ptr = (void *) ((uintptr_t) c | ev->instance);
0677 
0678     } else {
0679         op = EPOLL_CTL_DEL;
0680         ee.events = 0;
0681         ee.data.ptr = NULL;
0682     }
0683 
0684     ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
0685                    "epoll del event: fd:%d op:%d ev:%08XD",
0686                    c->fd, op, ee.events);
0687 
0688     if (epoll_ctl(ep, op, c->fd, &ee) == -1) {
0689         ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
0690                       "epoll_ctl(%d, %d) failed", op, c->fd);
0691         return NGX_ERROR;
0692     }
0693 
0694     ev->active = 0;
0695 
0696     return NGX_OK;
0697 }
0698 
0699 
0700 static ngx_int_t
0701 ngx_epoll_add_connection(ngx_connection_t *c)
0702 {
0703     struct epoll_event  ee;
0704 
0705     ee.events = EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP;
0706     ee.data.ptr = (void *) ((uintptr_t) c | c->read->instance);
0707 
0708     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
0709                    "epoll add connection: fd:%d ev:%08XD", c->fd, ee.events);
0710 
0711     if (epoll_ctl(ep, EPOLL_CTL_ADD, c->fd, &ee) == -1) {
0712         ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
0713                       "epoll_ctl(EPOLL_CTL_ADD, %d) failed", c->fd);
0714         return NGX_ERROR;
0715     }
0716 
0717     c->read->active = 1;
0718     c->write->active = 1;
0719 
0720     return NGX_OK;
0721 }
0722 
0723 
0724 static ngx_int_t
0725 ngx_epoll_del_connection(ngx_connection_t *c, ngx_uint_t flags)
0726 {
0727     int                 op;
0728     struct epoll_event  ee;
0729 
0730     /*
0731      * when the file descriptor is closed the epoll automatically deletes
0732      * it from its queue so we do not need to delete explicitly the event
0733      * before the closing the file descriptor
0734      */
0735 
0736     if (flags & NGX_CLOSE_EVENT) {
0737         c->read->active = 0;
0738         c->write->active = 0;
0739         return NGX_OK;
0740     }
0741 
0742     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
0743                    "epoll del connection: fd:%d", c->fd);
0744 
0745     op = EPOLL_CTL_DEL;
0746     ee.events = 0;
0747     ee.data.ptr = NULL;
0748 
0749     if (epoll_ctl(ep, op, c->fd, &ee) == -1) {
0750         ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
0751                       "epoll_ctl(%d, %d) failed", op, c->fd);
0752         return NGX_ERROR;
0753     }
0754 
0755     c->read->active = 0;
0756     c->write->active = 0;
0757 
0758     return NGX_OK;
0759 }
0760 
0761 
0762 #if (NGX_HAVE_EVENTFD)
0763 
0764 static ngx_int_t
0765 ngx_epoll_notify(ngx_event_handler_pt handler)
0766 {
0767     static uint64_t inc = 1;
0768 
0769     notify_event.data = handler;
0770 
0771     if ((size_t) write(notify_fd, &inc, sizeof(uint64_t)) != sizeof(uint64_t)) {
0772         ngx_log_error(NGX_LOG_ALERT, notify_event.log, ngx_errno,
0773                       "write() to eventfd %d failed", notify_fd);
0774         return NGX_ERROR;
0775     }
0776 
0777     return NGX_OK;
0778 }
0779 
0780 #endif
0781 
0782 
0783 static ngx_int_t
0784 ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
0785 {
0786     int                events;
0787     uint32_t           revents;
0788     ngx_int_t          instance, i;
0789     ngx_uint_t         level;
0790     ngx_err_t          err;
0791     ngx_event_t       *rev, *wev;
0792     ngx_queue_t       *queue;
0793     ngx_connection_t  *c;
0794 
0795     /* NGX_TIMER_INFINITE == INFTIM */
0796 
0797     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
0798                    "epoll timer: %M", timer);
0799 
0800     events = epoll_wait(ep, event_list, (int) nevents, timer);
0801 
0802     err = (events == -1) ? ngx_errno : 0;
0803 
0804     if (flags & NGX_UPDATE_TIME || ngx_event_timer_alarm) {
0805         ngx_time_update();
0806     }
0807 
0808     if (err) {
0809         if (err == NGX_EINTR) {
0810 
0811             if (ngx_event_timer_alarm) {
0812                 ngx_event_timer_alarm = 0;
0813                 return NGX_OK;
0814             }
0815 
0816             level = NGX_LOG_INFO;
0817 
0818         } else {
0819             level = NGX_LOG_ALERT;
0820         }
0821 
0822         ngx_log_error(level, cycle->log, err, "epoll_wait() failed");
0823         return NGX_ERROR;
0824     }
0825 
0826     if (events == 0) {
0827         if (timer != NGX_TIMER_INFINITE) {
0828             return NGX_OK;
0829         }
0830 
0831         ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
0832                       "epoll_wait() returned no events without timeout");
0833         return NGX_ERROR;
0834     }
0835 
0836     for (i = 0; i < events; i++) {
0837         c = event_list[i].data.ptr;
0838 
0839         instance = (uintptr_t) c & 1;
0840         c = (ngx_connection_t *) ((uintptr_t) c & (uintptr_t) ~1);
0841 
0842         rev = c->read;
0843 
0844         if (c->fd == -1 || rev->instance != instance) {
0845 
0846             /*
0847              * the stale event from a file descriptor
0848              * that was just closed in this iteration
0849              */
0850 
0851             ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
0852                            "epoll: stale event %p", c);
0853             continue;
0854         }
0855 
0856         revents = event_list[i].events;
0857 
0858         ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
0859                        "epoll: fd:%d ev:%04XD d:%p",
0860                        c->fd, revents, event_list[i].data.ptr);
0861 
0862         if (revents & (EPOLLERR|EPOLLHUP)) {
0863             ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
0864                            "epoll_wait() error on fd:%d ev:%04XD",
0865                            c->fd, revents);
0866 
0867             /*
0868              * if the error events were returned, add EPOLLIN and EPOLLOUT
0869              * to handle the events at least in one active handler
0870              */
0871 
0872             revents |= EPOLLIN|EPOLLOUT;
0873         }
0874 
0875 #if 0
0876         if (revents & ~(EPOLLIN|EPOLLOUT|EPOLLERR|EPOLLHUP)) {
0877             ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
0878                           "strange epoll_wait() events fd:%d ev:%04XD",
0879                           c->fd, revents);
0880         }
0881 #endif
0882 
0883         if ((revents & EPOLLIN) && rev->active) {
0884 
0885 #if (NGX_HAVE_EPOLLRDHUP)
0886             if (revents & EPOLLRDHUP) {
0887                 rev->pending_eof = 1;
0888             }
0889 
0890             rev->available = 1;
0891 #endif
0892 
0893             rev->ready = 1;
0894 
0895             if (flags & NGX_POST_EVENTS) {
0896                 queue = rev->accept ? &ngx_posted_accept_events
0897                                     : &ngx_posted_events;
0898 
0899                 ngx_post_event(rev, queue);
0900 
0901             } else {
0902                 rev->handler(rev);
0903             }
0904         }
0905 
0906         wev = c->write;
0907 
0908         if ((revents & EPOLLOUT) && wev->active) {
0909 
0910             if (c->fd == -1 || wev->instance != instance) {
0911 
0912                 /*
0913                  * the stale event from a file descriptor
0914                  * that was just closed in this iteration
0915                  */
0916 
0917                 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
0918                                "epoll: stale event %p", c);
0919                 continue;
0920             }
0921 
0922             wev->ready = 1;
0923 #if (NGX_THREADS)
0924             wev->complete = 1;
0925 #endif
0926 
0927             if (flags & NGX_POST_EVENTS) {
0928                 ngx_post_event(wev, &ngx_posted_events);
0929 
0930             } else {
0931                 wev->handler(wev);
0932             }
0933         }
0934     }
0935 
0936     return NGX_OK;
0937 }
0938 
0939 
0940 #if (NGX_HAVE_FILE_AIO)
0941 
0942 static void
0943 ngx_epoll_eventfd_handler(ngx_event_t *ev)
0944 {
0945     int               n, events;
0946     long              i;
0947     uint64_t          ready;
0948     ngx_err_t         err;
0949     ngx_event_t      *e;
0950     ngx_event_aio_t  *aio;
0951     struct io_event   event[64];
0952     struct timespec   ts;
0953 
0954     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd handler");
0955 
0956     n = read(ngx_eventfd, &ready, 8);
0957 
0958     err = ngx_errno;
0959 
0960     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd: %d", n);
0961 
0962     if (n != 8) {
0963         if (n == -1) {
0964             if (err == NGX_EAGAIN) {
0965                 return;
0966             }
0967 
0968             ngx_log_error(NGX_LOG_ALERT, ev->log, err, "read(eventfd) failed");
0969             return;
0970         }
0971 
0972         ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
0973                       "read(eventfd) returned only %d bytes", n);
0974         return;
0975     }
0976 
0977     ts.tv_sec = 0;
0978     ts.tv_nsec = 0;
0979 
0980     while (ready) {
0981 
0982         events = io_getevents(ngx_aio_ctx, 1, 64, event, &ts);
0983 
0984         ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0,
0985                        "io_getevents: %d", events);
0986 
0987         if (events > 0) {
0988             ready -= events;
0989 
0990             for (i = 0; i < events; i++) {
0991 
0992                 ngx_log_debug4(NGX_LOG_DEBUG_EVENT, ev->log, 0,
0993                                "io_event: %XL %XL %L %L",
0994                                 event[i].data, event[i].obj,
0995                                 event[i].res, event[i].res2);
0996 
0997                 e = (ngx_event_t *) (uintptr_t) event[i].data;
0998 
0999                 e->complete = 1;
1000                 e->active = 0;
1001                 e->ready = 1;
1002 
1003                 aio = e->data;
1004                 aio->res = event[i].res;
1005 
1006                 ngx_post_event(e, &ngx_posted_events);
1007             }
1008 
1009             continue;
1010         }
1011 
1012         if (events == 0) {
1013             return;
1014         }
1015 
1016         /* events == -1 */
1017         ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
1018                       "io_getevents() failed");
1019         return;
1020     }
1021 }
1022 
1023 #endif
1024 
1025 
1026 static void *
1027 ngx_epoll_create_conf(ngx_cycle_t *cycle)
1028 {
1029     ngx_epoll_conf_t  *epcf;
1030 
1031     epcf = ngx_palloc(cycle->pool, sizeof(ngx_epoll_conf_t));
1032     if (epcf == NULL) {
1033         return NULL;
1034     }
1035 
1036     epcf->events = NGX_CONF_UNSET;
1037     epcf->aio_requests = NGX_CONF_UNSET;
1038 
1039     return epcf;
1040 }
1041 
1042 
1043 static char *
1044 ngx_epoll_init_conf(ngx_cycle_t *cycle, void *conf)
1045 {
1046     ngx_epoll_conf_t *epcf = conf;
1047 
1048     ngx_conf_init_uint_value(epcf->events, 512);
1049     ngx_conf_init_uint_value(epcf->aio_requests, 32);
1050 
1051     return NGX_CONF_OK;
1052 }