xref: /unit/src/nxt_router.c (revision 1998)

/*
 * Copyright (C) Igor Sysoev
 * Copyright (C) Valentin V. Bartenev
 * Copyright (C) NGINX, Inc.
 */

#include <nxt_router.h>
#include <nxt_conf.h>
#if (NXT_TLS)
#include <nxt_cert.h>
#endif
#include <nxt_http.h>
#include <nxt_port_memory_int.h>
#include <nxt_unit_request.h>
#include <nxt_unit_response.h>
#include <nxt_router_request.h>
#include <nxt_app_queue.h>
#include <nxt_port_queue.h>

#define NXT_SHARED_PORT_ID  0xFFFFu

typedef struct {
    nxt_str_t         type;
    uint32_t          processes;
    uint32_t          max_processes;
    uint32_t          spare_processes;
    nxt_msec_t        timeout;
    nxt_msec_t        idle_timeout;
    nxt_conf_value_t  *limits_value;
    nxt_conf_value_t  *processes_value;
    nxt_conf_value_t  *targets_value;
} nxt_router_app_conf_t;


typedef struct {
    nxt_str_t         pass;
    nxt_str_t         application;
} nxt_router_listener_conf_t;


#if (NXT_TLS)

typedef struct {
    nxt_str_t               name;
    nxt_socket_conf_t       *socket_conf;
    nxt_router_temp_conf_t  *temp_conf;
    nxt_tls_init_t          *tls_init;
    nxt_bool_t              last;

    nxt_queue_link_t        link;  /* for nxt_socket_conf_t.tls */
} nxt_router_tlssock_t;

#endif


typedef struct {
    nxt_str_t               *name;
    nxt_socket_conf_t       *socket_conf;
    nxt_router_temp_conf_t  *temp_conf;
    nxt_bool_t              last;
} nxt_socket_rpc_t;


typedef struct {
    nxt_app_t               *app;
    nxt_router_temp_conf_t  *temp_conf;
    uint8_t                 proto;  /* 1 bit */
} nxt_app_rpc_t;


typedef struct {
    nxt_app_joint_t         *app_joint;
    uint32_t                generation;
    uint8_t                 proto;  /* 1 bit */
} nxt_app_joint_rpc_t;


static nxt_int_t nxt_router_prefork(nxt_task_t *task, nxt_process_t *process,
    nxt_mp_t *mp);
static nxt_int_t nxt_router_start(nxt_task_t *task, nxt_process_data_t *data);
static void nxt_router_greet_controller(nxt_task_t *task,
    nxt_port_t *controller_port);

static nxt_int_t nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app);

static void nxt_router_new_port_handler(nxt_task_t *task,
    nxt_port_recv_msg_t *msg);
static void nxt_router_conf_data_handler(nxt_task_t *task,
    nxt_port_recv_msg_t *msg);
static void nxt_router_app_restart_handler(nxt_task_t *task,
    nxt_port_recv_msg_t *msg);
static void nxt_router_remove_pid_handler(nxt_task_t *task,
    nxt_port_recv_msg_t *msg);
static void nxt_router_access_log_reopen_handler(nxt_task_t *task,
    nxt_port_recv_msg_t *msg);

static nxt_router_temp_conf_t *nxt_router_temp_conf(nxt_task_t *task);
static void nxt_router_conf_apply(nxt_task_t *task, void *obj, void *data);
static void nxt_router_conf_ready(nxt_task_t *task,
    nxt_router_temp_conf_t *tmcf);
static void nxt_router_conf_error(nxt_task_t *task,
    nxt_router_temp_conf_t *tmcf);
static void nxt_router_conf_send(nxt_task_t *task,
    nxt_router_temp_conf_t *tmcf, nxt_port_msg_type_t type);

static nxt_int_t nxt_router_conf_create(nxt_task_t *task,
    nxt_router_temp_conf_t *tmcf, u_char *start, u_char *end);
static nxt_int_t nxt_router_conf_process_static(nxt_task_t *task,
    nxt_router_conf_t *rtcf, nxt_conf_value_t *conf);
static nxt_int_t nxt_router_conf_process_client_ip(nxt_task_t *task,
    nxt_router_temp_conf_t *tmcf, nxt_socket_conf_t *skcf,
    nxt_conf_value_t *conf);

static nxt_app_t *nxt_router_app_find(nxt_queue_t *queue, nxt_str_t *name);
static nxt_int_t nxt_router_apps_hash_test(nxt_lvlhsh_query_t *lhq, void *data);
static nxt_int_t nxt_router_apps_hash_add(nxt_router_conf_t *rtcf,
    nxt_app_t *app);
static nxt_app_t *nxt_router_apps_hash_get(nxt_router_conf_t *rtcf,
    nxt_str_t *name);
static void nxt_router_apps_hash_use(nxt_task_t *task, nxt_router_conf_t *rtcf,
    int i);

static nxt_int_t nxt_router_app_queue_init(nxt_task_t *task,
    nxt_port_t *port);
static nxt_int_t nxt_router_port_queue_init(nxt_task_t *task,
    nxt_port_t *port);
static nxt_int_t nxt_router_port_queue_map(nxt_task_t *task,
    nxt_port_t *port, nxt_fd_t fd);
static void nxt_router_listen_socket_rpc_create(nxt_task_t *task,
    nxt_router_temp_conf_t *tmcf, nxt_socket_conf_t *skcf);
static void nxt_router_listen_socket_ready(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, void *data);
static void nxt_router_listen_socket_error(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, void *data);
#if (NXT_TLS)
static void nxt_router_tls_rpc_handler(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, void *data);
static nxt_int_t nxt_router_conf_tls_insert(nxt_router_temp_conf_t *tmcf,
    nxt_conf_value_t *value, nxt_socket_conf_t *skcf, nxt_tls_init_t *tls_init,
    nxt_bool_t last);
#endif
static void nxt_router_app_rpc_create(nxt_task_t *task,
    nxt_router_temp_conf_t *tmcf, nxt_app_t *app);
static void nxt_router_app_prefork_ready(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, void *data);
static void nxt_router_app_prefork_error(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, void *data);
static nxt_socket_conf_t *nxt_router_socket_conf(nxt_task_t *task,
    nxt_router_temp_conf_t *tmcf, nxt_str_t *name);
static nxt_int_t nxt_router_listen_socket_find(nxt_router_temp_conf_t *tmcf,
    nxt_socket_conf_t *nskcf, nxt_sockaddr_t *sa);

static nxt_int_t nxt_router_engines_create(nxt_task_t *task,
    nxt_router_t *router, nxt_router_temp_conf_t *tmcf,
    const nxt_event_interface_t *interface);
static nxt_int_t nxt_router_engine_conf_create(nxt_router_temp_conf_t *tmcf,
    nxt_router_engine_conf_t *recf);
static nxt_int_t nxt_router_engine_conf_update(nxt_router_temp_conf_t *tmcf,
    nxt_router_engine_conf_t *recf);
static nxt_int_t nxt_router_engine_conf_delete(nxt_router_temp_conf_t *tmcf,
    nxt_router_engine_conf_t *recf);
static nxt_int_t nxt_router_engine_joints_create(nxt_router_temp_conf_t *tmcf,
    nxt_router_engine_conf_t *recf, nxt_queue_t *sockets,
    nxt_work_handler_t handler);
static nxt_int_t nxt_router_engine_quit(nxt_router_temp_conf_t *tmcf,
    nxt_router_engine_conf_t *recf);
static nxt_int_t nxt_router_engine_joints_delete(nxt_router_temp_conf_t *tmcf,
    nxt_router_engine_conf_t *recf, nxt_queue_t *sockets);

static nxt_int_t nxt_router_threads_create(nxt_task_t *task, nxt_runtime_t *rt,
    nxt_router_temp_conf_t *tmcf);
static nxt_int_t nxt_router_thread_create(nxt_task_t *task, nxt_runtime_t *rt,
    nxt_event_engine_t *engine);
static void nxt_router_apps_sort(nxt_task_t *task, nxt_router_t *router,
    nxt_router_temp_conf_t *tmcf);

static void nxt_router_engines_post(nxt_router_t *router,
    nxt_router_temp_conf_t *tmcf);
static void nxt_router_engine_post(nxt_event_engine_t *engine,
    nxt_work_t *jobs);

static void nxt_router_thread_start(void *data);
static void nxt_router_rt_add_port(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_listen_socket_create(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_listen_socket_update(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_listen_socket_delete(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_worker_thread_quit(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_listen_socket_close(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_thread_exit_handler(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_req_headers_ack_handler(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, nxt_request_rpc_data_t *req_rpc_data);
static void nxt_router_listen_socket_release(nxt_task_t *task,
    nxt_socket_conf_t *skcf);

static void nxt_router_access_log_writer(nxt_task_t *task,
    nxt_http_request_t *r, nxt_router_access_log_t *access_log);
static u_char *nxt_router_access_log_date(u_char *buf, nxt_realtime_t *now,
    struct tm *tm, size_t size, const char *format);
static void nxt_router_access_log_open(nxt_task_t *task,
    nxt_router_temp_conf_t *tmcf);
static void nxt_router_access_log_ready(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, void *data);
static void nxt_router_access_log_error(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, void *data);
static void nxt_router_access_log_release(nxt_task_t *task,
    nxt_thread_spinlock_t *lock, nxt_router_access_log_t *access_log);
static void nxt_router_access_log_reopen_completion(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_access_log_reopen_ready(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, void *data);
static void nxt_router_access_log_reopen_error(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, void *data);

static void nxt_router_app_port_ready(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, void *data);
static nxt_int_t nxt_router_app_shared_port_send(nxt_task_t *task,
    nxt_port_t *app_port);
static void nxt_router_app_port_error(nxt_task_t *task,
    nxt_port_recv_msg_t *msg, void *data);

static void nxt_router_app_use(nxt_task_t *task, nxt_app_t *app, int i);
static void nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app);

static void nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app,
    nxt_port_t *port, nxt_apr_action_t action);
static void nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app,
    nxt_request_rpc_data_t *req_rpc_data);
static void nxt_router_http_request_error(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_http_request_done(nxt_task_t *task, void *obj,
    void *data);

static void nxt_router_app_prepare_request(nxt_task_t *task,
    nxt_request_rpc_data_t *req_rpc_data);
static nxt_buf_t *nxt_router_prepare_msg(nxt_task_t *task,
    nxt_http_request_t *r, nxt_app_t *app, const nxt_str_t *prefix);

static void nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data);
static void nxt_router_adjust_idle_timer(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_app_idle_timeout(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_app_joint_release_handler(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_free_app(nxt_task_t *task, void *obj, void *data);

static const nxt_http_request_state_t  nxt_http_request_send_state;
static void nxt_http_request_send_body(nxt_task_t *task, void *obj, void *data);

static void nxt_router_app_joint_use(nxt_task_t *task,
    nxt_app_joint_t *app_joint, int i);

static void nxt_router_http_request_release_post(nxt_task_t *task,
    nxt_http_request_t *r);
static void nxt_router_http_request_release(nxt_task_t *task, void *obj,
    void *data);
static void nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg);
static void nxt_router_get_port_handler(nxt_task_t *task,
    nxt_port_recv_msg_t *msg);
static void nxt_router_get_mmap_handler(nxt_task_t *task,
    nxt_port_recv_msg_t *msg);

extern const nxt_http_request_state_t  nxt_http_websocket;

static nxt_router_t  *nxt_router;

static const nxt_str_t http_prefix = nxt_string("HTTP_");
static const nxt_str_t empty_prefix = nxt_string("");

static const nxt_str_t  *nxt_app_msg_prefix[] = {
    &empty_prefix,
    &empty_prefix,
    &http_prefix,
    &http_prefix,
    &http_prefix,
    &empty_prefix,
};


static const nxt_port_handlers_t  nxt_router_process_port_handlers = {
    .quit         = nxt_signal_quit_handler,
    .new_port     = nxt_router_new_port_handler,
    .get_port     = nxt_router_get_port_handler,
    .change_file  = nxt_port_change_log_file_handler,
    .mmap         = nxt_port_mmap_handler,
    .get_mmap     = nxt_router_get_mmap_handler,
    .data         = nxt_router_conf_data_handler,
    .app_restart  = nxt_router_app_restart_handler,
    .remove_pid   = nxt_router_remove_pid_handler,
    .access_log   = nxt_router_access_log_reopen_handler,
    .rpc_ready    = nxt_port_rpc_handler,
    .rpc_error    = nxt_port_rpc_handler,
    .oosm         = nxt_router_oosm_handler,
};


const nxt_process_init_t  nxt_router_process = {
    .name           = "router",
    .type           = NXT_PROCESS_ROUTER,
    .prefork        = nxt_router_prefork,
    .restart        = 1,
    .setup          = nxt_process_core_setup,
    .start          = nxt_router_start,
    .port_handlers  = &nxt_router_process_port_handlers,
    .signals        = nxt_process_signals,
};


/* Queues of nxt_socket_conf_t */
nxt_queue_t  creating_sockets;
nxt_queue_t  pending_sockets;
nxt_queue_t  updating_sockets;
nxt_queue_t  keeping_sockets;
nxt_queue_t  deleting_sockets;


static nxt_int_t
nxt_router_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp)
{
    nxt_runtime_stop_app_processes(task, task->thread->runtime);

    return NXT_OK;
}


static nxt_int_t
nxt_router_start(nxt_task_t *task, nxt_process_data_t *data)
{
    nxt_int_t      ret;
    nxt_port_t     *controller_port;
    nxt_router_t   *router;
    nxt_runtime_t  *rt;

    rt = task->thread->runtime;

    nxt_log(task, NXT_LOG_INFO, "router started");

#if (NXT_TLS)
    rt->tls = nxt_service_get(rt->services, "SSL/TLS", "OpenSSL");
    if (nxt_slow_path(rt->tls == NULL)) {
        return NXT_ERROR;
    }

    ret = rt->tls->library_init(task);
    if (nxt_slow_path(ret != NXT_OK)) {
        return ret;
    }
#endif

    ret = nxt_http_init(task);
    if (nxt_slow_path(ret != NXT_OK)) {
        return ret;
    }

    router = nxt_zalloc(sizeof(nxt_router_t));
    if (nxt_slow_path(router == NULL)) {
        return NXT_ERROR;
    }

    nxt_queue_init(&router->engines);
    nxt_queue_init(&router->sockets);
    nxt_queue_init(&router->apps);

    nxt_router = router;

    controller_port = rt->port_by_type[NXT_PROCESS_CONTROLLER];
    if (controller_port != NULL) {
        nxt_router_greet_controller(task, controller_port);
    }

    return NXT_OK;
}


static void
nxt_router_greet_controller(nxt_task_t *task, nxt_port_t *controller_port)
{
    nxt_port_socket_write(task, controller_port, NXT_PORT_MSG_PROCESS_READY,
                          -1, 0, 0, NULL);
}


static void
nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port,
    void *data)
{
    size_t               size;
    uint32_t             stream;
    nxt_int_t            ret;
    nxt_app_t            *app;
    nxt_buf_t            *b;
    nxt_port_t           *dport;
    nxt_runtime_t        *rt;
    nxt_app_joint_rpc_t  *app_joint_rpc;

    app = data;

    nxt_thread_mutex_lock(&app->mutex);

    dport = app->proto_port;

    nxt_thread_mutex_unlock(&app->mutex);

    if (dport != NULL) {
        nxt_debug(task, "app '%V' %p start process", &app->name, app);

        b = NULL;

    } else {
        if (app->proto_port_requests > 0) {
            nxt_debug(task, "app '%V' %p wait for prototype process",
                      &app->name, app);

            app->proto_port_requests++;

            goto skip;
        }

        nxt_debug(task, "app '%V' %p start prototype process", &app->name, app);

        rt = task->thread->runtime;
        dport = rt->port_by_type[NXT_PROCESS_MAIN];

        size = app->name.length + 1 + app->conf.length;

        b = nxt_buf_mem_alloc(task->thread->engine->mem_pool, size, 0);
        if (nxt_slow_path(b == NULL)) {
            goto failed;
        }

        nxt_buf_cpystr(b, &app->name);
        *b->mem.free++ = '\0';
        nxt_buf_cpystr(b, &app->conf);
    }

    app_joint_rpc = nxt_port_rpc_register_handler_ex(task, port,
                                                     nxt_router_app_port_ready,
                                                     nxt_router_app_port_error,
                                                   sizeof(nxt_app_joint_rpc_t));
    if (nxt_slow_path(app_joint_rpc == NULL)) {
        goto failed;
    }

    stream = nxt_port_rpc_ex_stream(app_joint_rpc);

    ret = nxt_port_socket_write(task, dport, NXT_PORT_MSG_START_PROCESS,
                                -1, stream, port->id, b);
    if (nxt_slow_path(ret != NXT_OK)) {
        nxt_port_rpc_cancel(task, port, stream);

        goto failed;
    }

    app_joint_rpc->app_joint = app->joint;
    app_joint_rpc->generation = app->generation;
    app_joint_rpc->proto = (b != NULL);

    if (b != NULL) {
        app->proto_port_requests++;

        b = NULL;
    }

    nxt_router_app_joint_use(task, app->joint, 1);

failed:

    if (b != NULL) {
        nxt_mp_free(b->data, b);
    }

skip:

    nxt_router_app_use(task, app, -1);
}


static void
nxt_router_app_joint_use(nxt_task_t *task, nxt_app_joint_t *app_joint, int i)
{
    app_joint->use_count += i;

    if (app_joint->use_count == 0) {
        nxt_assert(app_joint->app == NULL);

        nxt_free(app_joint);
    }
}


static nxt_int_t
nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app)
{
    nxt_int_t      res;
    nxt_port_t     *router_port;
    nxt_runtime_t  *rt;

    nxt_debug(task, "app '%V' start process", &app->name);

    rt = task->thread->runtime;
    router_port = rt->port_by_type[NXT_PROCESS_ROUTER];

    nxt_router_app_use(task, app, 1);

    res = nxt_port_post(task, router_port, nxt_router_start_app_process_handler,
                        app);

    if (res == NXT_OK) {
        return res;
    }

    nxt_thread_mutex_lock(&app->mutex);

    app->pending_processes--;

    nxt_thread_mutex_unlock(&app->mutex);

    nxt_router_app_use(task, app, -1);

    return NXT_ERROR;
}


nxt_inline nxt_bool_t
nxt_router_msg_cancel(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data)
{
    nxt_buf_t       *b, *next;
    nxt_bool_t      cancelled;
    nxt_port_t      *app_port;
    nxt_msg_info_t  *msg_info;

    msg_info = &req_rpc_data->msg_info;

    if (msg_info->buf == NULL) {
        return 0;
    }

    app_port = req_rpc_data->app_port;

    if (app_port != NULL && app_port->id == NXT_SHARED_PORT_ID) {
        cancelled = nxt_app_queue_cancel(app_port->queue,
                                         msg_info->tracking_cookie,
                                         req_rpc_data->stream);

        if (cancelled) {
            nxt_debug(task, "stream #%uD: cancelled by router",
                      req_rpc_data->stream);
        }

    } else {
        cancelled = 0;
    }

    for (b = msg_info->buf; b != NULL; b = next) {
        next = b->next;
        b->next = NULL;

        if (b->is_port_mmap_sent) {
            b->is_port_mmap_sent = cancelled == 0;
        }

        b->completion_handler(task, b, b->parent);
    }

    msg_info->buf = NULL;

    return cancelled;
}


nxt_inline nxt_bool_t
nxt_queue_chk_remove(nxt_queue_link_t *lnk)
{
    if (lnk->next != NULL) {
        nxt_queue_remove(lnk);

        lnk->next = NULL;

        return 1;
    }

    return 0;
}


nxt_inline void
nxt_request_rpc_data_unlink(nxt_task_t *task,
    nxt_request_rpc_data_t *req_rpc_data)
{
    nxt_app_t           *app;
    nxt_bool_t          unlinked;
    nxt_http_request_t  *r;

    nxt_router_msg_cancel(task, req_rpc_data);

    app = req_rpc_data->app;

    if (req_rpc_data->app_port != NULL) {
        nxt_router_app_port_release(task, app, req_rpc_data->app_port,
                                    req_rpc_data->apr_action);

        req_rpc_data->app_port = NULL;
    }

    r = req_rpc_data->request;

    if (r != NULL) {
        r->timer_data = NULL;

        nxt_router_http_request_release_post(task, r);

        r->req_rpc_data = NULL;
        req_rpc_data->request = NULL;

        if (app != NULL) {
            unlinked = 0;

            nxt_thread_mutex_lock(&app->mutex);

            if (r->app_link.next != NULL) {
                nxt_queue_remove(&r->app_link);
                r->app_link.next = NULL;

                unlinked = 1;
            }

            nxt_thread_mutex_unlock(&app->mutex);

            if (unlinked) {
                nxt_mp_release(r->mem_pool);
            }
        }
    }

    if (app != NULL) {
        nxt_router_app_use(task, app, -1);

        req_rpc_data->app = NULL;
    }

    if (req_rpc_data->msg_info.body_fd != -1) {
        nxt_fd_close(req_rpc_data->msg_info.body_fd);

        req_rpc_data->msg_info.body_fd = -1;
    }

    if (req_rpc_data->rpc_cancel) {
        req_rpc_data->rpc_cancel = 0;

        nxt_port_rpc_cancel(task, task->thread->engine->port,
                            req_rpc_data->stream);
    }
}


static void
nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg)
{
    nxt_int_t      res;
    nxt_app_t      *app;
    nxt_port_t     *port, *main_app_port;
    nxt_runtime_t  *rt;

    nxt_port_new_port_handler(task, msg);

    port = msg->u.new_port;

    if (port != NULL && port->type == NXT_PROCESS_CONTROLLER) {
        nxt_router_greet_controller(task, msg->u.new_port);
    }

    if (port != NULL && port->type == NXT_PROCESS_PROTOTYPE)  {
        nxt_port_rpc_handler(task, msg);

        return;
    }

    if (port == NULL || port->type != NXT_PROCESS_APP) {

        if (msg->port_msg.stream == 0) {
            return;
        }

        msg->port_msg.type = _NXT_PORT_MSG_RPC_ERROR;

    } else {
        if (msg->fd[1] != -1) {
            res = nxt_router_port_queue_map(task, port, msg->fd[1]);
            if (nxt_slow_path(res != NXT_OK)) {
                return;
            }

            nxt_fd_close(msg->fd[1]);
            msg->fd[1] = -1;
        }
    }

    if (msg->port_msg.stream != 0) {
        nxt_port_rpc_handler(task, msg);
        return;
    }

    nxt_debug(task, "new port id %d (%d)", port->id, port->type);

    /*
     * Port with "id == 0" is application 'main' port and it always
     * should come with non-zero stream.
     */
    nxt_assert(port->id != 0);

    /* Find 'main' app port and get app reference. */
    rt = task->thread->runtime;

    /*
     * It is safe to access 'runtime->ports' hash because 'NEW_PORT'
     * sent to main port (with id == 0) and processed in main thread.
     */
    main_app_port = nxt_port_hash_find(&rt->ports, port->pid, 0);
    nxt_assert(main_app_port != NULL);

    app = main_app_port->app;

    if (nxt_fast_path(app != NULL)) {
        nxt_thread_mutex_lock(&app->mutex);

        /* TODO here should be find-and-add code because there can be
           port waiters in port_hash */
        nxt_port_hash_add(&app->