120Sigor@sysoev.ru 220Sigor@sysoev.ru /* 320Sigor@sysoev.ru * Copyright (C) Igor Sysoev 420Sigor@sysoev.ru * Copyright (C) Valentin V. Bartenev 520Sigor@sysoev.ru * Copyright (C) NGINX, Inc. 620Sigor@sysoev.ru */ 720Sigor@sysoev.ru 853Sigor@sysoev.ru #include <nxt_router.h> 9115Sigor@sysoev.ru #include <nxt_conf.h> 10774Svbart@nginx.com #if (NXT_TLS) 11774Svbart@nginx.com #include <nxt_cert.h> 12774Svbart@nginx.com #endif 13431Sigor@sysoev.ru #include <nxt_http.h> 14743Smax.romanov@nginx.com #include <nxt_port_memory_int.h> 15743Smax.romanov@nginx.com #include <nxt_unit_request.h> 16743Smax.romanov@nginx.com #include <nxt_unit_response.h> 171131Smax.romanov@nginx.com #include <nxt_router_request.h> 1820Sigor@sysoev.ru 19115Sigor@sysoev.ru typedef struct { 20318Smax.romanov@nginx.com nxt_str_t type; 21507Smax.romanov@nginx.com uint32_t processes; 22507Smax.romanov@nginx.com uint32_t max_processes; 23507Smax.romanov@nginx.com uint32_t spare_processes; 24318Smax.romanov@nginx.com nxt_msec_t timeout; 25427Smax.romanov@nginx.com nxt_msec_t res_timeout; 26507Smax.romanov@nginx.com nxt_msec_t idle_timeout; 27318Smax.romanov@nginx.com uint32_t requests; 28318Smax.romanov@nginx.com nxt_conf_value_t *limits_value; 29507Smax.romanov@nginx.com nxt_conf_value_t *processes_value; 301473Svbart@nginx.com nxt_conf_value_t *targets_value; 31133Sigor@sysoev.ru } nxt_router_app_conf_t; 32133Sigor@sysoev.ru 33133Sigor@sysoev.ru 34133Sigor@sysoev.ru typedef struct { 35964Sigor@sysoev.ru nxt_str_t pass; 36964Sigor@sysoev.ru nxt_str_t application; 37115Sigor@sysoev.ru } nxt_router_listener_conf_t; 38115Sigor@sysoev.ru 39115Sigor@sysoev.ru 40774Svbart@nginx.com #if (NXT_TLS) 41774Svbart@nginx.com 42774Svbart@nginx.com typedef struct { 43774Svbart@nginx.com nxt_str_t name; 44774Svbart@nginx.com nxt_socket_conf_t *conf; 45774Svbart@nginx.com 46774Svbart@nginx.com nxt_queue_link_t link; /* for nxt_socket_conf_t.tls */ 47774Svbart@nginx.com } nxt_router_tlssock_t; 48774Svbart@nginx.com 49774Svbart@nginx.com #endif 50774Svbart@nginx.com 51774Svbart@nginx.com 52198Sigor@sysoev.ru typedef struct { 53198Sigor@sysoev.ru nxt_socket_conf_t *socket_conf; 54198Sigor@sysoev.ru nxt_router_temp_conf_t *temp_conf; 55198Sigor@sysoev.ru } nxt_socket_rpc_t; 56198Sigor@sysoev.ru 57198Sigor@sysoev.ru 58507Smax.romanov@nginx.com typedef struct { 59507Smax.romanov@nginx.com nxt_app_t *app; 60507Smax.romanov@nginx.com nxt_router_temp_conf_t *temp_conf; 61507Smax.romanov@nginx.com } nxt_app_rpc_t; 62507Smax.romanov@nginx.com 63507Smax.romanov@nginx.com 641488St.nateldemoura@f5.com static nxt_int_t nxt_router_prefork(nxt_task_t *task, nxt_process_t *process, 651488St.nateldemoura@f5.com nxt_mp_t *mp); 661488St.nateldemoura@f5.com static nxt_int_t nxt_router_start(nxt_task_t *task, nxt_process_data_t *data); 67662Smax.romanov@nginx.com static void nxt_router_greet_controller(nxt_task_t *task, 68662Smax.romanov@nginx.com nxt_port_t *controller_port); 69662Smax.romanov@nginx.com 70507Smax.romanov@nginx.com static nxt_int_t nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app); 71425Smax.romanov@nginx.com 72*1552Smax.romanov@nginx.com static void nxt_router_new_port_handler(nxt_task_t *task, 73*1552Smax.romanov@nginx.com nxt_port_recv_msg_t *msg); 74*1552Smax.romanov@nginx.com static void nxt_router_conf_data_handler(nxt_task_t *task, 75*1552Smax.romanov@nginx.com nxt_port_recv_msg_t *msg); 76*1552Smax.romanov@nginx.com static void nxt_router_remove_pid_handler(nxt_task_t *task, 77*1552Smax.romanov@nginx.com nxt_port_recv_msg_t *msg); 78*1552Smax.romanov@nginx.com static void nxt_router_access_log_reopen_handler(nxt_task_t *task, 79*1552Smax.romanov@nginx.com nxt_port_recv_msg_t *msg); 80*1552Smax.romanov@nginx.com 81139Sigor@sysoev.ru static nxt_router_temp_conf_t *nxt_router_temp_conf(nxt_task_t *task); 82198Sigor@sysoev.ru static void nxt_router_conf_apply(nxt_task_t *task, void *obj, void *data); 83198Sigor@sysoev.ru static void nxt_router_conf_ready(nxt_task_t *task, 84139Sigor@sysoev.ru nxt_router_temp_conf_t *tmcf); 85139Sigor@sysoev.ru static void nxt_router_conf_error(nxt_task_t *task, 86139Sigor@sysoev.ru nxt_router_temp_conf_t *tmcf); 87139Sigor@sysoev.ru static void nxt_router_conf_send(nxt_task_t *task, 88193Smax.romanov@nginx.com nxt_router_temp_conf_t *tmcf, nxt_port_msg_type_t type); 8953Sigor@sysoev.ru 90115Sigor@sysoev.ru static nxt_int_t nxt_router_conf_create(nxt_task_t *task, 91115Sigor@sysoev.ru nxt_router_temp_conf_t *tmcf, u_char *start, u_char *end); 921183Svbart@nginx.com static nxt_int_t nxt_router_conf_process_static(nxt_task_t *task, 931183Svbart@nginx.com nxt_router_conf_t *rtcf, nxt_conf_value_t *conf); 94133Sigor@sysoev.ru static nxt_app_t *nxt_router_app_find(nxt_queue_t *queue, nxt_str_t *name); 95198Sigor@sysoev.ru static void nxt_router_listen_socket_rpc_create(nxt_task_t *task, 96198Sigor@sysoev.ru nxt_router_temp_conf_t *tmcf, nxt_socket_conf_t *skcf); 97198Sigor@sysoev.ru static void nxt_router_listen_socket_ready(nxt_task_t *task, 98198Sigor@sysoev.ru nxt_port_recv_msg_t *msg, void *data); 99198Sigor@sysoev.ru static void nxt_router_listen_socket_error(nxt_task_t *task, 100198Sigor@sysoev.ru nxt_port_recv_msg_t *msg, void *data); 101774Svbart@nginx.com #if (NXT_TLS) 102774Svbart@nginx.com static void nxt_router_tls_rpc_create(nxt_task_t *task, 103774Svbart@nginx.com nxt_router_temp_conf_t *tmcf, nxt_router_tlssock_t *tls); 104774Svbart@nginx.com static void nxt_router_tls_rpc_handler(nxt_task_t *task, 105774Svbart@nginx.com nxt_port_recv_msg_t *msg, void *data); 106774Svbart@nginx.com #endif 107507Smax.romanov@nginx.com static void nxt_router_app_rpc_create(nxt_task_t *task, 108507Smax.romanov@nginx.com nxt_router_temp_conf_t *tmcf, nxt_app_t *app); 109507Smax.romanov@nginx.com static void nxt_router_app_prefork_ready(nxt_task_t *task, 110507Smax.romanov@nginx.com nxt_port_recv_msg_t *msg, void *data); 111507Smax.romanov@nginx.com static void nxt_router_app_prefork_error(nxt_task_t *task, 112507Smax.romanov@nginx.com nxt_port_recv_msg_t *msg, void *data); 113359Sigor@sysoev.ru static nxt_socket_conf_t *nxt_router_socket_conf(nxt_task_t *task, 114359Sigor@sysoev.ru nxt_router_temp_conf_t *tmcf, nxt_str_t *name); 115359Sigor@sysoev.ru static nxt_int_t nxt_router_listen_socket_find(nxt_router_temp_conf_t *tmcf, 116359Sigor@sysoev.ru nxt_socket_conf_t *nskcf, nxt_sockaddr_t *sa); 11753Sigor@sysoev.ru 11853Sigor@sysoev.ru static nxt_int_t nxt_router_engines_create(nxt_task_t *task, 11953Sigor@sysoev.ru nxt_router_t *router, nxt_router_temp_conf_t *tmcf, 12053Sigor@sysoev.ru const nxt_event_interface_t *interface); 121115Sigor@sysoev.ru static nxt_int_t nxt_router_engine_conf_create(nxt_router_temp_conf_t *tmcf, 122115Sigor@sysoev.ru nxt_router_engine_conf_t *recf); 123115Sigor@sysoev.ru static nxt_int_t nxt_router_engine_conf_update(nxt_router_temp_conf_t *tmcf, 124115Sigor@sysoev.ru nxt_router_engine_conf_t *recf); 125115Sigor@sysoev.ru static nxt_int_t nxt_router_engine_conf_delete(nxt_router_temp_conf_t *tmcf, 126115Sigor@sysoev.ru nxt_router_engine_conf_t *recf); 127154Sigor@sysoev.ru static nxt_int_t nxt_router_engine_joints_create(nxt_router_temp_conf_t *tmcf, 128154Sigor@sysoev.ru nxt_router_engine_conf_t *recf, nxt_queue_t *sockets, 129154Sigor@sysoev.ru nxt_work_handler_t handler); 130313Sigor@sysoev.ru static nxt_int_t nxt_router_engine_quit(nxt_router_temp_conf_t *tmcf, 131313Sigor@sysoev.ru nxt_router_engine_conf_t *recf); 132139Sigor@sysoev.ru static nxt_int_t nxt_router_engine_joints_delete(nxt_router_temp_conf_t *tmcf, 133139Sigor@sysoev.ru nxt_router_engine_conf_t *recf, nxt_queue_t *sockets); 13453Sigor@sysoev.ru 13553Sigor@sysoev.ru static nxt_int_t nxt_router_threads_create(nxt_task_t *task, nxt_runtime_t *rt, 13653Sigor@sysoev.ru nxt_router_temp_conf_t *tmcf); 13753Sigor@sysoev.ru static nxt_int_t nxt_router_thread_create(nxt_task_t *task, nxt_runtime_t *rt, 13853Sigor@sysoev.ru nxt_event_engine_t *engine); 139343Smax.romanov@nginx.com static void nxt_router_apps_sort(nxt_task_t *task, nxt_router_t *router, 140133Sigor@sysoev.ru nxt_router_temp_conf_t *tmcf); 14153Sigor@sysoev.ru 142315Sigor@sysoev.ru static void nxt_router_engines_post(nxt_router_t *router, 143315Sigor@sysoev.ru nxt_router_temp_conf_t *tmcf); 144315Sigor@sysoev.ru static void nxt_router_engine_post(nxt_event_engine_t *engine, 145315Sigor@sysoev.ru nxt_work_t *jobs); 14653Sigor@sysoev.ru 14753Sigor@sysoev.ru static void nxt_router_thread_start(void *data); 1481545Smax.romanov@nginx.com static void nxt_router_rt_add_port(nxt_task_t *task, void *obj, 1491545Smax.romanov@nginx.com void *data); 15053Sigor@sysoev.ru static void nxt_router_listen_socket_create(nxt_task_t *task, void *obj, 15153Sigor@sysoev.ru void *data); 15253Sigor@sysoev.ru static void nxt_router_listen_socket_update(nxt_task_t *task, void *obj, 15353Sigor@sysoev.ru void *data); 15453Sigor@sysoev.ru static void nxt_router_listen_socket_delete(nxt_task_t *task, void *obj, 15553Sigor@sysoev.ru void *data); 156313Sigor@sysoev.ru static void nxt_router_worker_thread_quit(nxt_task_t *task, void *obj, 157313Sigor@sysoev.ru void *data); 15853Sigor@sysoev.ru static void nxt_router_listen_socket_close(nxt_task_t *task, void *obj, 15953Sigor@sysoev.ru void *data); 16053Sigor@sysoev.ru static void nxt_router_thread_exit_handler(nxt_task_t *task, void *obj, 16153Sigor@sysoev.ru void *data); 1621547Smax.romanov@nginx.com static void nxt_router_req_headers_ack_handler(nxt_task_t *task, 1631547Smax.romanov@nginx.com nxt_port_recv_msg_t *msg, nxt_request_rpc_data_t *req_rpc_data); 164359Sigor@sysoev.ru static void nxt_router_listen_socket_release(nxt_task_t *task, 165359Sigor@sysoev.ru nxt_socket_conf_t *skcf); 16653Sigor@sysoev.ru 167630Svbart@nginx.com static void nxt_router_access_log_writer(nxt_task_t *task, 168630Svbart@nginx.com nxt_http_request_t *r, nxt_router_access_log_t *access_log); 169630Svbart@nginx.com static u_char *nxt_router_access_log_date(u_char *buf, nxt_realtime_t *now, 170630Svbart@nginx.com struct tm *tm, size_t size, const char *format); 171630Svbart@nginx.com static void nxt_router_access_log_open(nxt_task_t *task, 172630Svbart@nginx.com nxt_router_temp_conf_t *tmcf); 173630Svbart@nginx.com static void nxt_router_access_log_ready(nxt_task_t *task, 174630Svbart@nginx.com nxt_port_recv_msg_t *msg, void *data); 175630Svbart@nginx.com static void nxt_router_access_log_error(nxt_task_t *task, 176630Svbart@nginx.com nxt_port_recv_msg_t *msg, void *data); 177630Svbart@nginx.com static void nxt_router_access_log_release(nxt_task_t *task, 178630Svbart@nginx.com nxt_thread_spinlock_t *lock, nxt_router_access_log_t *access_log); 179651Svbart@nginx.com static void nxt_router_access_log_reopen_completion(nxt_task_t *task, void *obj, 180651Svbart@nginx.com void *data); 181631Svbart@nginx.com static void nxt_router_access_log_reopen_ready(nxt_task_t *task, 182631Svbart@nginx.com nxt_port_recv_msg_t *msg, void *data); 183631Svbart@nginx.com static void nxt_router_access_log_reopen_error(nxt_task_t *task, 184631Svbart@nginx.com nxt_port_recv_msg_t *msg, void *data); 185630Svbart@nginx.com 186343Smax.romanov@nginx.com static void nxt_router_app_port_ready(nxt_task_t *task, 187343Smax.romanov@nginx.com nxt_port_recv_msg_t *msg, void *data); 1881547Smax.romanov@nginx.com static nxt_int_t nxt_router_app_shared_port_send(nxt_task_t *task, 1891547Smax.romanov@nginx.com nxt_port_t *app_port); 190343Smax.romanov@nginx.com static void nxt_router_app_port_error(nxt_task_t *task, 191343Smax.romanov@nginx.com nxt_port_recv_msg_t *msg, void *data); 192343Smax.romanov@nginx.com 193753Smax.romanov@nginx.com static void nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app); 1941123Smax.romanov@nginx.com 195343Smax.romanov@nginx.com static void nxt_router_app_port_release(nxt_task_t *task, nxt_port_t *port, 1961123Smax.romanov@nginx.com nxt_apr_action_t action); 1971547Smax.romanov@nginx.com static void nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, 1981547Smax.romanov@nginx.com nxt_request_rpc_data_t *req_rpc_data); 1991547Smax.romanov@nginx.com static void nxt_router_http_request_done(nxt_task_t *task, void *obj, 2001547Smax.romanov@nginx.com void *data); 201141Smax.romanov@nginx.com 202425Smax.romanov@nginx.com static void nxt_router_app_prepare_request(nxt_task_t *task, 2031547Smax.romanov@nginx.com nxt_request_rpc_data_t *req_rpc_data); 2041007Salexander.borisov@nginx.com static nxt_buf_t *nxt_router_prepare_msg(nxt_task_t *task, 2051547Smax.romanov@nginx.com nxt_http_request_t *r, nxt_app_t *app, const nxt_str_t *prefix); 206510Salexander.borisov@nginx.com 207318Smax.romanov@nginx.com static void nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data); 208507Smax.romanov@nginx.com static void nxt_router_adjust_idle_timer(nxt_task_t *task, void *obj, 209507Smax.romanov@nginx.com void *data); 210507Smax.romanov@nginx.com static void nxt_router_app_idle_timeout(nxt_task_t *task, void *obj, 211507Smax.romanov@nginx.com void *data); 212753Smax.romanov@nginx.com static void nxt_router_app_joint_release_handler(nxt_task_t *task, void *obj, 213507Smax.romanov@nginx.com void *data); 214753Smax.romanov@nginx.com static void nxt_router_free_app(nxt_task_t *task, void *obj, void *data); 215431Sigor@sysoev.ru 216431Sigor@sysoev.ru static const nxt_http_request_state_t nxt_http_request_send_state; 217431Sigor@sysoev.ru static void nxt_http_request_send_body(nxt_task_t *task, void *obj, void *data); 218141Smax.romanov@nginx.com 219753Smax.romanov@nginx.com static void nxt_router_app_joint_use(nxt_task_t *task, 220753Smax.romanov@nginx.com nxt_app_joint_t *app_joint, int i); 221753Smax.romanov@nginx.com 2221547Smax.romanov@nginx.com static void nxt_router_http_request_release_post(nxt_task_t *task, 2231007Salexander.borisov@nginx.com nxt_http_request_t *r); 2241007Salexander.borisov@nginx.com static void nxt_router_http_request_release(nxt_task_t *task, void *obj, 2251007Salexander.borisov@nginx.com void *data); 2261321Smax.romanov@nginx.com static void nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); 2271545Smax.romanov@nginx.com static void nxt_router_get_port_handler(nxt_task_t *task, 2281545Smax.romanov@nginx.com nxt_port_recv_msg_t *msg); 2291546Smax.romanov@nginx.com static void nxt_router_get_mmap_handler(nxt_task_t *task, 2301546Smax.romanov@nginx.com nxt_port_recv_msg_t *msg); 2311007Salexander.borisov@nginx.com 2321149Smax.romanov@nginx.com extern const nxt_http_request_state_t nxt_http_websocket; 2331131Smax.romanov@nginx.com 234119Smax.romanov@nginx.com static nxt_router_t *nxt_router; 23520Sigor@sysoev.ru 236743Smax.romanov@nginx.com static const nxt_str_t http_prefix = nxt_string("HTTP_"); 237743Smax.romanov@nginx.com static const nxt_str_t empty_prefix = nxt_string(""); 238743Smax.romanov@nginx.com 239743Smax.romanov@nginx.com static const nxt_str_t *nxt_app_msg_prefix[] = { 240804Svbart@nginx.com &empty_prefix, 241743Smax.romanov@nginx.com &http_prefix, 242743Smax.romanov@nginx.com &http_prefix, 243743Smax.romanov@nginx.com &http_prefix, 244743Smax.romanov@nginx.com &http_prefix, 245977Smax.romanov@gmail.com &empty_prefix, 246216Sigor@sysoev.ru }; 247216Sigor@sysoev.ru 248216Sigor@sysoev.ru 2491488St.nateldemoura@f5.com static const nxt_port_handlers_t nxt_router_process_port_handlers = { 2501488St.nateldemoura@f5.com .quit = nxt_signal_quit_handler, 251662Smax.romanov@nginx.com .new_port = nxt_router_new_port_handler, 2521545Smax.romanov@nginx.com .get_port = nxt_router_get_port_handler, 253662Smax.romanov@nginx.com .change_file = nxt_port_change_log_file_handler, 254662Smax.romanov@nginx.com .mmap = nxt_port_mmap_handler, 2551546Smax.romanov@nginx.com .get_mmap = nxt_router_get_mmap_handler, 256662Smax.romanov@nginx.com .data = nxt_router_conf_data_handler, 257662Smax.romanov@nginx.com .remove_pid = nxt_router_remove_pid_handler, 258662Smax.romanov@nginx.com .access_log = nxt_router_access_log_reopen_handler, 259662Smax.romanov@nginx.com .rpc_ready = nxt_port_rpc_handler, 260662Smax.romanov@nginx.com .rpc_error = nxt_port_rpc_handler, 2611321Smax.romanov@nginx.com .oosm = nxt_router_oosm_handler, 262662Smax.romanov@nginx.com }; 263662Smax.romanov@nginx.com 264662Smax.romanov@nginx.com 2651488St.nateldemoura@f5.com const nxt_process_init_t nxt_router_process = { 2661488St.nateldemoura@f5.com .name = "router", 2671488St.nateldemoura@f5.com .type = NXT_PROCESS_ROUTER, 2681488St.nateldemoura@f5.com .prefork = nxt_router_prefork, 2691488St.nateldemoura@f5.com .restart = 1, 2701488St.nateldemoura@f5.com .setup = nxt_process_core_setup, 2711488St.nateldemoura@f5.com .start = nxt_router_start, 2721488St.nateldemoura@f5.com .port_handlers = &nxt_router_process_port_handlers, 2731488St.nateldemoura@f5.com .signals = nxt_process_signals, 2741488St.nateldemoura@f5.com }; 2751488St.nateldemoura@f5.com 2761488St.nateldemoura@f5.com 2771509Sigor@sysoev.ru /* Queues of nxt_socket_conf_t */ 2781509Sigor@sysoev.ru nxt_queue_t creating_sockets; 2791509Sigor@sysoev.ru nxt_queue_t pending_sockets; 2801509Sigor@sysoev.ru nxt_queue_t updating_sockets; 2811509Sigor@sysoev.ru nxt_queue_t keeping_sockets; 2821509Sigor@sysoev.ru nxt_queue_t deleting_sockets; 2831509Sigor@sysoev.ru 2841509Sigor@sysoev.ru 2851488St.nateldemoura@f5.com static nxt_int_t 2861488St.nateldemoura@f5.com nxt_router_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp) 2871488St.nateldemoura@f5.com { 2881488St.nateldemoura@f5.com nxt_runtime_stop_app_processes(task, task->thread->runtime); 2891488St.nateldemoura@f5.com 2901488St.nateldemoura@f5.com return NXT_OK; 2911488St.nateldemoura@f5.com } 2921488St.nateldemoura@f5.com 2931488St.nateldemoura@f5.com 2941488St.nateldemoura@f5.com static nxt_int_t 2951488St.nateldemoura@f5.com nxt_router_start(nxt_task_t *task, nxt_process_data_t *data) 29620Sigor@sysoev.ru { 297141Smax.romanov@nginx.com nxt_int_t ret; 298662Smax.romanov@nginx.com nxt_port_t *controller_port; 299141Smax.romanov@nginx.com nxt_router_t *router; 300141Smax.romanov@nginx.com nxt_runtime_t *rt; 301141Smax.romanov@nginx.com 302141Smax.romanov@nginx.com rt = task->thread->runtime; 30353Sigor@sysoev.ru 3041488St.nateldemoura@f5.com nxt_log(task, NXT_LOG_INFO, "router started"); 3051488St.nateldemoura@f5.com 306771Sigor@sysoev.ru #if (NXT_TLS) 307771Sigor@sysoev.ru rt->tls = nxt_service_get(rt->services, "SSL/TLS", "OpenSSL"); 308771Sigor@sysoev.ru if (nxt_slow_path(rt->tls == NULL)) { 309771Sigor@sysoev.ru return NXT_ERROR; 310771Sigor@sysoev.ru } 311771Sigor@sysoev.ru 312771Sigor@sysoev.ru ret = rt->tls->library_init(task); 313771Sigor@sysoev.ru if (nxt_slow_path(ret != NXT_OK)) { 314771Sigor@sysoev.ru return ret; 315771Sigor@sysoev.ru } 316771Sigor@sysoev.ru #endif 317771Sigor@sysoev.ru 3181459Smax.romanov@nginx.com ret = nxt_http_init(task); 31988Smax.romanov@nginx.com if (nxt_slow_path(ret != NXT_OK)) { 32088Smax.romanov@nginx.com return ret; 32188Smax.romanov@nginx.com } 32288Smax.romanov@nginx.com 32353Sigor@sysoev.ru router = nxt_zalloc(sizeof(nxt_router_t)); 32453Sigor@sysoev.ru if (nxt_slow_path(router == NULL)) { 32553Sigor@sysoev.ru return NXT_ERROR; 32653Sigor@sysoev.ru } 32753Sigor@sysoev.ru 32853Sigor@sysoev.ru nxt_queue_init(&router->engines); 32953Sigor@sysoev.ru nxt_queue_init(&router->sockets); 330133Sigor@sysoev.ru nxt_queue_init(&router->apps); 33153Sigor@sysoev.ru 332119Smax.romanov@nginx.com nxt_router = router; 333119Smax.romanov@nginx.com 334662Smax.romanov@nginx.com controller_port = rt->port_by_type[NXT_PROCESS_CONTROLLER]; 335662Smax.romanov@nginx.com if (controller_port != NULL) { 336662Smax.romanov@nginx.com nxt_router_greet_controller(task, controller_port); 337662Smax.romanov@nginx.com } 338662Smax.romanov@nginx.com 339115Sigor@sysoev.ru return NXT_OK; 340115Sigor@sysoev.ru } 341115Sigor@sysoev.ru 342115Sigor@sysoev.ru 343343Smax.romanov@nginx.com static void 344662Smax.romanov@nginx.com nxt_router_greet_controller(nxt_task_t *task, nxt_port_t *controller_port) 345662Smax.romanov@nginx.com { 346662Smax.romanov@nginx.com nxt_port_socket_write(task, controller_port, NXT_PORT_MSG_PROCESS_READY, 347662Smax.romanov@nginx.com -1, 0, 0, NULL); 348662Smax.romanov@nginx.com } 349662Smax.romanov@nginx.com 350662Smax.romanov@nginx.com 351662Smax.romanov@nginx.com static void 352507Smax.romanov@nginx.com nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, 353507Smax.romanov@nginx.com void *data) 354167Smax.romanov@nginx.com { 355343Smax.romanov@nginx.com size_t size; 356343Smax.romanov@nginx.com uint32_t stream; 357430Sigor@sysoev.ru nxt_mp_t *mp; 358648Svbart@nginx.com nxt_int_t ret; 359343Smax.romanov@nginx.com nxt_app_t *app; 360343Smax.romanov@nginx.com nxt_buf_t *b; 361343Smax.romanov@nginx.com nxt_port_t *main_port; 362343Smax.romanov@nginx.com nxt_runtime_t *rt; 363343Smax.romanov@nginx.com 364343Smax.romanov@nginx.com app = data; 365167Smax.romanov@nginx.com 366167Smax.romanov@nginx.com rt = task->thread->runtime; 367240Sigor@sysoev.ru main_port = rt->port_by_type[NXT_PROCESS_MAIN]; 368167Smax.romanov@nginx.com 369507Smax.romanov@nginx.com nxt_debug(task, "app '%V' %p start process", &app->name, app); 370343Smax.romanov@nginx.com 371343Smax.romanov@nginx.com size = app->name.length + 1 + app->conf.length; 372343Smax.romanov@nginx.com 373343Smax.romanov@nginx.com b = nxt_buf_mem_ts_alloc(task, task->thread->engine->mem_pool, size); 374343Smax.romanov@nginx.com 375343Smax.romanov@nginx.com if (nxt_slow_path(b == NULL)) { 376343Smax.romanov@nginx.com goto failed; 377167Smax.romanov@nginx.com } 378167Smax.romanov@nginx.com 379343Smax.romanov@nginx.com nxt_buf_cpystr(b, &app->name); 380343Smax.romanov@nginx.com *b->mem.free++ = '\0'; 381343Smax.romanov@nginx.com nxt_buf_cpystr(b, &app->conf); 382343Smax.romanov@nginx.com 383753Smax.romanov@nginx.com nxt_router_app_joint_use(task, app->joint, 1); 384753Smax.romanov@nginx.com 385343Smax.romanov@nginx.com stream = nxt_port_rpc_register_handler(task, port, 386343Smax.romanov@nginx.com nxt_router_app_port_ready, 387343Smax.romanov@nginx.com nxt_router_app_port_error, 388753Smax.romanov@nginx.com -1, app->joint); 389343Smax.romanov@nginx.com 390343Smax.romanov@nginx.com if (nxt_slow_path(stream == 0)) { 391753Smax.romanov@nginx.com nxt_router_app_joint_use(task, app->joint, -1); 392753Smax.romanov@nginx.com 393343Smax.romanov@nginx.com goto failed; 394343Smax.romanov@nginx.com } 395343Smax.romanov@nginx.com 3961488St.nateldemoura@f5.com ret = nxt_port_socket_write(task, main_port, NXT_PORT_MSG_START_PROCESS, 3971488St.nateldemoura@f5.com -1, stream, port->id, b); 398648Svbart@nginx.com 399648Svbart@nginx.com if (nxt_slow_path(ret != NXT_OK)) { 400648Svbart@nginx.com nxt_port_rpc_cancel(task, port, stream); 401753Smax.romanov@nginx.com 402753Smax.romanov@nginx.com nxt_router_app_joint_use(task, app->joint, -1); 403753Smax.romanov@nginx.com 404648Svbart@nginx.com goto failed; 405648Svbart@nginx.com } 406343Smax.romanov@nginx.com 407753Smax.romanov@nginx.com nxt_router_app_use(task, app, -1); 408753Smax.romanov@nginx.com 409343Smax.romanov@nginx.com return; 410343Smax.romanov@nginx.com 411343Smax.romanov@nginx.com failed: 412343Smax.romanov@nginx.com 413648Svbart@nginx.com if (b != NULL) { 414648Svbart@nginx.com mp = b->data; 415648Svbart@nginx.com nxt_mp_free(mp, b); 416648Svbart@nginx.com nxt_mp_release(mp); 417648Svbart@nginx.com } 418648Svbart@nginx.com 419343Smax.romanov@nginx.com nxt_thread_mutex_lock(&app->mutex); 420343Smax.romanov@nginx.com 421507Smax.romanov@nginx.com app->pending_processes--; 422343Smax.romanov@nginx.com 423343Smax.romanov@nginx.com nxt_thread_mutex_unlock(&app->mutex); 424343Smax.romanov@nginx.com 425343Smax.romanov@nginx.com nxt_router_app_use(task, app, -1); 426167Smax.romanov@nginx.com } 427167Smax.romanov@nginx.com 428167Smax.romanov@nginx.com 429753Smax.romanov@nginx.com static void 430753Smax.romanov@nginx.com nxt_router_app_joint_use(nxt_task_t *task, nxt_app_joint_t *app_joint, int i) 431753Smax.romanov@nginx.com { 432753Smax.romanov@nginx.com app_joint->use_count += i; 433753Smax.romanov@nginx.com 434753Smax.romanov@nginx.com if (app_joint->use_count == 0) { 435753Smax.romanov@nginx.com nxt_assert(app_joint->app == NULL); 436753Smax.romanov@nginx.com 437753Smax.romanov@nginx.com nxt_free(app_joint); 438753Smax.romanov@nginx.com } 439753Smax.romanov@nginx.com } 440753Smax.romanov@nginx.com 441753Smax.romanov@nginx.com 442343Smax.romanov@nginx.com static nxt_int_t 443507Smax.romanov@nginx.com nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app) 444141Smax.romanov@nginx.com { 445343Smax.romanov@nginx.com nxt_int_t res; 446343Smax.romanov@nginx.com nxt_port_t *router_port; 447343Smax.romanov@nginx.com nxt_runtime_t *rt; 448343Smax.romanov@nginx.com 4491549Smax.romanov@nginx.com nxt_debug(task, "app '%V' start process", &app->name); 4501549Smax.romanov@nginx.com 451343Smax.romanov@nginx.com rt = task->thread->runtime; 452343Smax.romanov@nginx.com router_port = rt->port_by_type[NXT_PROCESS_ROUTER]; 453343Smax.romanov@nginx.com 454343Smax.romanov@nginx.com nxt_router_app_use(task, app, 1); 455343Smax.romanov@nginx.com 456507Smax.romanov@nginx.com res = nxt_port_post(task, router_port, nxt_router_start_app_process_handler, 457343Smax.romanov@nginx.com app); 458343Smax.romanov@nginx.com 459343Smax.romanov@nginx.com if (res == NXT_OK) { 460343Smax.romanov@nginx.com return res; 461318Smax.romanov@nginx.com } 462318Smax.romanov@nginx.com 463343Smax.romanov@nginx.com nxt_thread_mutex_lock(&app->mutex); 464343Smax.romanov@nginx.com 465507Smax.romanov@nginx.com app->pending_processes--; 466343Smax.romanov@nginx.com 467343Smax.romanov@nginx.com nxt_thread_mutex_unlock(&app->mutex); 468343Smax.romanov@nginx.com 469343Smax.romanov@nginx.com nxt_router_app_use(task, app, -1); 470343Smax.romanov@nginx.com 471343Smax.romanov@nginx.com return NXT_ERROR; 472318Smax.romanov@nginx.com } 473318Smax.romanov@nginx.com 474318Smax.romanov@nginx.com 475423Smax.romanov@nginx.com nxt_inline nxt_bool_t 476423Smax.romanov@nginx.com nxt_router_msg_cancel(nxt_task_t *task, nxt_msg_info_t *msg_info, 477423Smax.romanov@nginx.com uint32_t stream) 478423Smax.romanov@nginx.com { 479423Smax.romanov@nginx.com nxt_buf_t *b, *next; 480423Smax.romanov@nginx.com nxt_bool_t cancelled; 481423Smax.romanov@nginx.com 482423Smax.romanov@nginx.com if (msg_info->buf == NULL) { 483423Smax.romanov@nginx.com return 0; 484423Smax.romanov@nginx.com } 485423Smax.romanov@nginx.com 486423Smax.romanov@nginx.com cancelled = nxt_port_mmap_tracking_cancel(task, &msg_info->tracking, 487423Smax.romanov@nginx.com stream); 488423Smax.romanov@nginx.com 489423Smax.romanov@nginx.com if (cancelled) { 490423Smax.romanov@nginx.com nxt_debug(task, "stream #%uD: cancelled by router", stream); 491423Smax.romanov@nginx.com } 492423Smax.romanov@nginx.com 493423Smax.romanov@nginx.com for (b = msg_info->buf; b != NULL; b = next) { 494423Smax.romanov@nginx.com next = b->next; 4951269Sigor@sysoev.ru b->next = NULL; 496423Smax.romanov@nginx.com 497423Smax.romanov@nginx.com b->completion_handler = msg_info->completion_handler; 498423Smax.romanov@nginx.com 499423Smax.romanov@nginx.com if (b->is_port_mmap_sent) { 500423Smax.romanov@nginx.com b->is_port_mmap_sent = cancelled == 0; 501423Smax.romanov@nginx.com b->completion_handler(task, b, b->parent); 502423Smax.romanov@nginx.com } 503423Smax.romanov@nginx.com } 504423Smax.romanov@nginx.com 505423Smax.romanov@nginx.com msg_info->buf = NULL; 506423Smax.romanov@nginx.com 507423Smax.romanov@nginx.com return cancelled; 508423Smax.romanov@nginx.com } 509423Smax.romanov@nginx.com 510423Smax.romanov@nginx.com 511425Smax.romanov@nginx.com nxt_inline nxt_bool_t 512425Smax.romanov@nginx.com nxt_queue_chk_remove(nxt_queue_link_t *lnk) 513425Smax.romanov@nginx.com { 514425Smax.romanov@nginx.com if (lnk->next != NULL) { 515425Smax.romanov@nginx.com nxt_queue_remove(lnk); 516425Smax.romanov@nginx.com 517425Smax.romanov@nginx.com lnk->next = NULL; 518425Smax.romanov@nginx.com 519425Smax.romanov@nginx.com return 1; 520425Smax.romanov@nginx.com } 521425Smax.romanov@nginx.com 522425Smax.romanov@nginx.com return 0; 523425Smax.romanov@nginx.com } 524425Smax.romanov@nginx.com 525425Smax.romanov@nginx.com 526343Smax.romanov@nginx.com nxt_inline void 5271123Smax.romanov@nginx.com nxt_request_rpc_data_unlink(nxt_task_t *task, 5281123Smax.romanov@nginx.com nxt_request_rpc_data_t *req_rpc_data) 529343Smax.romanov@nginx.com { 5301547Smax.romanov@nginx.com nxt_http_request_t *r; 5311547Smax.romanov@nginx.com 5321547Smax.romanov@nginx.com nxt_router_msg_cancel(task, &req_rpc_data->msg_info, req_rpc_data->stream); 5331123Smax.romanov@nginx.com 5341123Smax.romanov@nginx.com if (req_rpc_data->app_port != NULL) { 5351123Smax.romanov@nginx.com nxt_router_app_port_release(task, req_rpc_data->app_port, 5361123Smax.romanov@nginx.com req_rpc_data->apr_action); 5371123Smax.romanov@nginx.com 5381123Smax.romanov@nginx.com req_rpc_data->app_port = NULL; 5391123Smax.romanov@nginx.com } 5401123Smax.romanov@nginx.com 5411123Smax.romanov@nginx.com if (req_rpc_data->app != NULL) { 5421123Smax.romanov@nginx.com nxt_router_app_use(task, req_rpc_data->app, -1); 5431123Smax.romanov@nginx.com 5441123Smax.romanov@nginx.com req_rpc_data->app = NULL; 5451123Smax.romanov@nginx.com } 5461123Smax.romanov@nginx.com 5471547Smax.romanov@nginx.com r = req_rpc_data->request; 5481547Smax.romanov@nginx.com 5491547Smax.romanov@nginx.com if (r != NULL) { 5501547Smax.romanov@nginx.com r->timer_data = NULL; 5511547Smax.romanov@nginx.com 5521547Smax.romanov@nginx.com nxt_router_http_request_release_post(task, r); 5531547Smax.romanov@nginx.com 5541547Smax.romanov@nginx.com r->req_rpc_data = NULL; 5551123Smax.romanov@nginx.com req_rpc_data->request = NULL; 556346Smax.romanov@nginx.com } 5571547Smax.romanov@nginx.com 5581547Smax.romanov@nginx.com if (req_rpc_data->msg_info.body_fd != -1) { 5591547Smax.romanov@nginx.com nxt_fd_close(req_rpc_data->msg_info.body_fd); 5601547Smax.romanov@nginx.com 5611547Smax.romanov@nginx.com req_rpc_data->msg_info.body_fd = -1; 5621547Smax.romanov@nginx.com } 5631547Smax.romanov@nginx.com 5641547Smax.romanov@nginx.com if (req_rpc_data->rpc_cancel) { 5651547Smax.romanov@nginx.com req_rpc_data->rpc_cancel = 0; 5661547Smax.romanov@nginx.com 5671547Smax.romanov@nginx.com nxt_port_rpc_cancel(task, task->thread->engine->port, 5681547Smax.romanov@nginx.com req_rpc_data->stream); 5691547Smax.romanov@nginx.com } 570343Smax.romanov@nginx.com } 571343Smax.romanov@nginx.com 572343Smax.romanov@nginx.com 573*1552Smax.romanov@nginx.com static void 574141Smax.romanov@nginx.com nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) 575141Smax.romanov@nginx.com { 5761547Smax.romanov@nginx.com nxt_app_t *app; 5771547Smax.romanov@nginx.com nxt_port_t *port, *main_app_port; 5781547Smax.romanov@nginx.com nxt_runtime_t *rt; 5791547Smax.romanov@nginx.com 580141Smax.romanov@nginx.com nxt_port_new_port_handler(task, msg); 581141Smax.romanov@nginx.com 5821547Smax.romanov@nginx.com port = msg->u.new_port; 5831547Smax.romanov@nginx.com 5841547Smax.romanov@nginx.com if (port != NULL && port->type == NXT_PROCESS_CONTROLLER) { 585662Smax.romanov@nginx.com nxt_router_greet_controller(task, msg->u.new_port); 586662Smax.romanov@nginx.com } 587662Smax.romanov@nginx.com 5881547Smax.romanov@nginx.com if (port == NULL || port->type != NXT_PROCESS_APP) { 5891547Smax.romanov@nginx.com 5901547Smax.romanov@nginx.com if (msg->port_msg.stream == 0) { 5911547Smax.romanov@nginx.com return; 5921547Smax.romanov@nginx.com } 5931547Smax.romanov@nginx.com 5941547Smax.romanov@nginx.com msg->port_msg.type = _NXT_PORT_MSG_RPC_ERROR; 5951547Smax.romanov@nginx.com } 5961547Smax.romanov@nginx.com 5971547Smax.romanov@nginx.com if (msg->port_msg.stream != 0) { 5981547Smax.romanov@nginx.com nxt_port_rpc_handler(task, msg); 599141Smax.romanov@nginx.com return; 600141Smax.romanov@nginx.com } 601141Smax.romanov@nginx.com 6021547Smax.romanov@nginx.com /* 6031547Smax.romanov@nginx.com * Port with "id == 0" is application 'main' port and it always 6041547Smax.romanov@nginx.com * should come with non-zero stream. 6051547Smax.romanov@nginx.com */ 6061547Smax.romanov@nginx.com nxt_assert(port->id != 0); 6071547Smax.romanov@nginx.com 6081547Smax.romanov@nginx.com /* Find 'main' app port and get app reference. */ 6091547Smax.romanov@nginx.com rt = task->thread->runtime; 6101547Smax.romanov@nginx.com 6111547Smax.romanov@nginx.com /* 6121547Smax.romanov@nginx.com * It is safe to access 'runtime->ports' hash because 'NEW_PORT' 6131547Smax.romanov@nginx.com * sent to main port (with id == 0) and processed in main thread. 6141547Smax.romanov@nginx.com */ 6151547Smax.romanov@nginx.com main_app_port = nxt_port_hash_find(&rt->ports, port->pid, 0); 6161547Smax.romanov@nginx.com nxt_assert(main_app_port != NULL); 6171547Smax.romanov@nginx.com 6181547Smax.romanov@nginx.com app = main_app_port->app; 6191547Smax.romanov@nginx.com nxt_assert(app != NULL); 6201547Smax.romanov@nginx.com 6211547Smax.romanov@nginx.com nxt_thread_mutex_lock(&app->mutex); 6221547Smax.romanov@nginx.com 6231547Smax.romanov@nginx.com /* TODO here should be find-and-add code because there can be 6241547Smax.romanov@nginx.com port waiters in port_hash */ 6251547Smax.romanov@nginx.com nxt_port_hash_add(&app->port_hash, port); 6261547Smax.romanov@nginx.com app->port_hash_count++; 6271547Smax.romanov@nginx.com 6281547Smax.romanov@nginx.com nxt_thread_mutex_unlock(&app->mutex); 6291547Smax.romanov@nginx.com 6301547Smax.romanov@nginx.com port->app = app; 6311547Smax.romanov@nginx.com port->main_app_port = main_app_port; 632141Smax.romanov@nginx.com } 633141Smax.romanov@nginx.com 634141Smax.romanov@nginx.com 635*1552Smax.romanov@nginx.com static void 636139Sigor@sysoev.ru nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) 637115Sigor@sysoev.ru { 6381526Smax.romanov@nginx.com void *p; 6391526Smax.romanov@nginx.com size_t size; 640198Sigor@sysoev.ru nxt_int_t ret; 641139Sigor@sysoev.ru nxt_router_temp_conf_t *tmcf; 642139Sigor@sysoev.ru 643139Sigor@sysoev.ru tmcf = nxt_router_temp_conf(task); 644139Sigor@sysoev.ru if (nxt_slow_path(tmcf == NULL)) { 645139Sigor@sysoev.ru return; 64653Sigor@sysoev.ru } 64753Sigor@sysoev.ru 6481526Smax.romanov@nginx.com if (nxt_slow_path(msg->fd == -1)) { 6491526Smax.romanov@nginx.com nxt_alert(task, "conf_data_handler: invalid file shm fd"); 6501526Smax.romanov@nginx.com return; 6511526Smax.romanov@nginx.com } 6521526Smax.romanov@nginx.com 6531526Smax.romanov@nginx.com if (nxt_buf_mem_used_size(&msg->buf->mem) != sizeof(size_t)) { 6541526Smax.romanov@nginx.com nxt_alert(task, "conf_data_handler: unexpected buffer size (%d)", 6551526Smax.romanov@nginx.com (int) nxt_buf_mem_used_size(&msg->buf->mem)); 6561526Smax.romanov@nginx.com 6571526Smax.romanov@nginx.com nxt_fd_close(msg->fd); 6581526Smax.romanov@nginx.com msg->fd = -1; 6591526Smax.romanov@nginx.com 6601526Smax.romanov@nginx.com return; 6611526Smax.romanov@nginx.com } 6621526Smax.romanov@nginx.com 6631526Smax.romanov@nginx.com nxt_memcpy(&size, msg->buf->mem.pos, sizeof(size_t)); 6641526Smax.romanov@nginx.com 6651526Smax.romanov@nginx.com p = nxt_mem_mmap(NULL, size, PROT_READ, MAP_SHARED, msg->fd, 0); 6661526Smax.romanov@nginx.com 6671526Smax.romanov@nginx.com nxt_fd_close(msg->fd); 6681526Smax.romanov@nginx.com msg->fd = -1; 6691526Smax.romanov@nginx.com 6701526Smax.romanov@nginx.com if (nxt_slow_path(p == MAP_FAILED)) { 6711526Smax.romanov@nginx.com return; 6721526Smax.romanov@nginx.com } 6731526Smax.romanov@nginx.com 6741526Smax.romanov@nginx.com nxt_debug(task, "conf_data_handler(%uz): %*s", size, size, p); 675423Smax.romanov@nginx.com 676591Sigor@sysoev.ru tmcf->router_conf->router = nxt_router; 677139Sigor@sysoev.ru tmcf->stream = msg->port_msg.stream; 678139Sigor@sysoev.ru tmcf->port = nxt_runtime_port_find(task->thread->runtime, 679198Sigor@sysoev.ru msg->port_msg.pid, 680198Sigor@sysoev.ru msg->port_msg.reply_port); 681198Sigor@sysoev.ru 682779Smax.romanov@nginx.com if (nxt_slow_path(tmcf->port == NULL)) { 683779Smax.romanov@nginx.com nxt_alert(task, "reply port not found"); 684779Smax.romanov@nginx.com 6851526Smax.romanov@nginx.com goto fail; 686779Smax.romanov@nginx.com } 687779Smax.romanov@nginx.com 688779Smax.romanov@nginx.com nxt_port_use(task, tmcf->port, 1); 689779Smax.romanov@nginx.com 6901526Smax.romanov@nginx.com ret = nxt_router_conf_create(task, tmcf, p, nxt_pointer_to(p, size)); 691198Sigor@sysoev.ru 692198Sigor@sysoev.ru if (nxt_fast_path(ret == NXT_OK)) { 693198Sigor@sysoev.ru nxt_router_conf_apply(task, tmcf, NULL); 694198Sigor@sysoev.ru 695198Sigor@sysoev.ru } else { 696198Sigor@sysoev.ru nxt_router_conf_error(task, tmcf); 697139Sigor@sysoev.ru } 6981526Smax.romanov@nginx.com 6991526Smax.romanov@nginx.com fail: 7001526Smax.romanov@nginx.com 7011526Smax.romanov@nginx.com nxt_mem_munmap(p, size); 70253Sigor@sysoev.ru } 70353Sigor@sysoev.ru 70453Sigor@sysoev.ru 705347Smax.romanov@nginx.com static void 706507Smax.romanov@nginx.com nxt_router_app_process_remove_pid(nxt_task_t *task, nxt_port_t *port, 707507Smax.romanov@nginx.com void *data) 708347Smax.romanov@nginx.com { 709347Smax.romanov@nginx.com union { 710347Smax.romanov@nginx.com nxt_pid_t removed_pid; 711347Smax.romanov@nginx.com void *data; 712347Smax.romanov@nginx.com } u; 713347Smax.romanov@nginx.com 714347Smax.romanov@nginx.com u.data = data; 715347Smax.romanov@nginx.com 716347Smax.romanov@nginx.com nxt_port_rpc_remove_peer(task, port, u.removed_pid); 717347Smax.romanov@nginx.com } 718347Smax.romanov@nginx.com 719347Smax.romanov@nginx.com 720*1552Smax.romanov@nginx.com static void 721192Smax.romanov@nginx.com nxt_router_remove_pid_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) 722192Smax.romanov@nginx.com { 723347Smax.romanov@nginx.com nxt_event_engine_t *engine; 724