nxt_unit.c

/*
 * Copyright (C) NGINX, Inc.
 */

#include "nxt_main.h"
#include "nxt_port_memory_int.h"
#include "nxt_socket_msg.h"
#include "nxt_port_queue.h"
#include "nxt_app_queue.h"

#include "nxt_unit.h"
#include "nxt_unit_request.h"
#include "nxt_unit_response.h"
#include "nxt_unit_websocket.h"

#include "nxt_websocket.h"

#if (NXT_HAVE_MEMFD_CREATE)
#include <linux/memfd.h>
#endif

#define NXT_UNIT_MAX_PLAIN_SIZE  1024
#define NXT_UNIT_LOCAL_BUF_SIZE  \
    (NXT_UNIT_MAX_PLAIN_SIZE + sizeof(nxt_port_msg_t))

enum {
    NXT_QUIT_NORMAL   = 0,
    NXT_QUIT_GRACEFUL = 1,
};

typedef struct nxt_unit_impl_s                  nxt_unit_impl_t;
typedef struct nxt_unit_mmap_s                  nxt_unit_mmap_t;
typedef struct nxt_unit_mmaps_s                 nxt_unit_mmaps_t;
typedef struct nxt_unit_process_s               nxt_unit_process_t;
typedef struct nxt_unit_mmap_buf_s              nxt_unit_mmap_buf_t;
typedef struct nxt_unit_recv_msg_s              nxt_unit_recv_msg_t;
typedef struct nxt_unit_read_buf_s              nxt_unit_read_buf_t;
typedef struct nxt_unit_ctx_impl_s              nxt_unit_ctx_impl_t;
typedef struct nxt_unit_port_impl_s             nxt_unit_port_impl_t;
typedef struct nxt_unit_request_info_impl_s     nxt_unit_request_info_impl_t;
typedef struct nxt_unit_websocket_frame_impl_s  nxt_unit_websocket_frame_impl_t;

static nxt_unit_impl_t *nxt_unit_create(nxt_unit_init_t *init);
static int nxt_unit_ctx_init(nxt_unit_impl_t *lib,
    nxt_unit_ctx_impl_t *ctx_impl, void *data);
nxt_inline void nxt_unit_ctx_use(nxt_unit_ctx_t *ctx);
nxt_inline void nxt_unit_ctx_release(nxt_unit_ctx_t *ctx);
nxt_inline void nxt_unit_lib_use(nxt_unit_impl_t *lib);
nxt_inline void nxt_unit_lib_release(nxt_unit_impl_t *lib);
nxt_inline void nxt_unit_mmap_buf_insert(nxt_unit_mmap_buf_t **head,
    nxt_unit_mmap_buf_t *mmap_buf);
nxt_inline void nxt_unit_mmap_buf_insert_tail(nxt_unit_mmap_buf_t **prev,
    nxt_unit_mmap_buf_t *mmap_buf);
nxt_inline void nxt_unit_mmap_buf_unlink(nxt_unit_mmap_buf_t *mmap_buf);
static int nxt_unit_read_env(nxt_unit_port_t *ready_port,
    nxt_unit_port_t *router_port, nxt_unit_port_t *read_port,
    int *shared_port_fd, int *shared_queue_fd,
    int *log_fd, uint32_t *stream, uint32_t *shm_limit,
    uint32_t *request_limit);
static int nxt_unit_ready(nxt_unit_ctx_t *ctx, int ready_fd, uint32_t stream,
    int queue_fd);
static int nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf,
    nxt_unit_request_info_t **preq);
static int nxt_unit_process_new_port(nxt_unit_ctx_t *ctx,
    nxt_unit_recv_msg_t *recv_msg);
static int nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx);
static int nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx,
    nxt_unit_recv_msg_t *recv_msg, nxt_unit_request_info_t **preq);
static int nxt_unit_process_req_body(nxt_unit_ctx_t *ctx,
    nxt_unit_recv_msg_t *recv_msg);
static int nxt_unit_request_check_response_port(nxt_unit_request_info_t *req,
    nxt_unit_port_id_t *port_id);
static int nxt_unit_send_req_headers_ack(nxt_unit_request_info_t *req);
static int nxt_unit_process_websocket(nxt_unit_ctx_t *ctx,
    nxt_unit_recv_msg_t *recv_msg);
static int nxt_unit_process_shm_ack(nxt_unit_ctx_t *ctx);
static nxt_unit_request_info_impl_t *nxt_unit_request_info_get(
    nxt_unit_ctx_t *ctx);
static void nxt_unit_request_info_release(nxt_unit_request_info_t *req);
static void nxt_unit_request_info_free(nxt_unit_request_info_impl_t *req);
static nxt_unit_websocket_frame_impl_t *nxt_unit_websocket_frame_get(
    nxt_unit_ctx_t *ctx);
static void nxt_unit_websocket_frame_release(nxt_unit_websocket_frame_t *ws);
static void nxt_unit_websocket_frame_free(nxt_unit_ctx_t *ctx,
    nxt_unit_websocket_frame_impl_t *ws);
static nxt_unit_mmap_buf_t *nxt_unit_mmap_buf_get(nxt_unit_ctx_t *ctx);
static void nxt_unit_mmap_buf_release(nxt_unit_mmap_buf_t *mmap_buf);
static int nxt_unit_mmap_buf_send(nxt_unit_request_info_t *req,
    nxt_unit_mmap_buf_t *mmap_buf, int last);
static void nxt_unit_mmap_buf_free(nxt_unit_mmap_buf_t *mmap_buf);
static void nxt_unit_free_outgoing_buf(nxt_unit_mmap_buf_t *mmap_buf);
static nxt_unit_read_buf_t *nxt_unit_read_buf_get(nxt_unit_ctx_t *ctx);
static nxt_unit_read_buf_t *nxt_unit_read_buf_get_impl(
    nxt_unit_ctx_impl_t *ctx_impl);
static void nxt_unit_read_buf_release(nxt_unit_ctx_t *ctx,
    nxt_unit_read_buf_t *rbuf);
static nxt_unit_mmap_buf_t *nxt_unit_request_preread(
    nxt_unit_request_info_t *req, size_t size);
static ssize_t nxt_unit_buf_read(nxt_unit_buf_t **b, uint64_t *len, void *dst,
    size_t size);
static nxt_port_mmap_header_t *nxt_unit_mmap_get(nxt_unit_ctx_t *ctx,
    nxt_unit_port_t *port, nxt_chunk_id_t *c, int *n, int min_n);
static int nxt_unit_send_oosm(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port);
static int nxt_unit_wait_shm_ack(nxt_unit_ctx_t *ctx);
static nxt_unit_mmap_t *nxt_unit_mmap_at(nxt_unit_mmaps_t *mmaps, uint32_t i);
static nxt_port_mmap_header_t *nxt_unit_new_mmap(nxt_unit_ctx_t *ctx,
    nxt_unit_port_t *port, int n);
static int nxt_unit_shm_open(nxt_unit_ctx_t *ctx, size_t size);
static int nxt_unit_send_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    int fd);
static int nxt_unit_get_outgoing_buf(nxt_unit_ctx_t *ctx,
    nxt_unit_port_t *port, uint32_t size,
    uint32_t min_size, nxt_unit_mmap_buf_t *mmap_buf, char *local_buf);
static int nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd);

static void nxt_unit_awake_ctx(nxt_unit_ctx_t *ctx,
    nxt_unit_ctx_impl_t *ctx_impl);
static int nxt_unit_mmaps_init(nxt_unit_mmaps_t *mmaps);
nxt_inline void nxt_unit_process_use(nxt_unit_process_t *process);
nxt_inline void nxt_unit_process_release(nxt_unit_process_t *process);
static void nxt_unit_mmaps_destroy(nxt_unit_mmaps_t *mmaps);
static int nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx,
    nxt_unit_mmaps_t *mmaps, pid_t pid, uint32_t id,
    nxt_port_mmap_header_t **hdr, nxt_unit_read_buf_t *rbuf);
static int nxt_unit_mmap_read(nxt_unit_ctx_t *ctx,
    nxt_unit_recv_msg_t *recv_msg, nxt_unit_read_buf_t *rbuf);
static int nxt_unit_get_mmap(nxt_unit_ctx_t *ctx, pid_t pid, uint32_t id);
static void nxt_unit_mmap_release(nxt_unit_ctx_t *ctx,
    nxt_port_mmap_header_t *hdr, void *start, uint32_t size);
static int nxt_unit_send_shm_ack(nxt_unit_ctx_t *ctx, pid_t pid);

static nxt_unit_process_t *nxt_unit_process_get(nxt_unit_ctx_t *ctx, pid_t pid);
static nxt_unit_process_t *nxt_unit_process_find(nxt_unit_impl_t *lib,
    pid_t pid, int remove);
static nxt_unit_process_t *nxt_unit_process_pop_first(nxt_unit_impl_t *lib);
static int nxt_unit_run_once_impl(nxt_unit_ctx_t *ctx);
static int nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf);
static int nxt_unit_chk_ready(nxt_unit_ctx_t *ctx);
static int nxt_unit_process_pending_rbuf(nxt_unit_ctx_t *ctx);
static void nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx);
nxt_inline int nxt_unit_is_read_queue(nxt_unit_read_buf_t *rbuf);
nxt_inline int nxt_unit_is_read_socket(nxt_unit_read_buf_t *rbuf);
nxt_inline int nxt_unit_is_shm_ack(nxt_unit_read_buf_t *rbuf);
nxt_inline int nxt_unit_is_quit(nxt_unit_read_buf_t *rbuf);
static int nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx,
    nxt_unit_port_t *port);
static void nxt_unit_ctx_free(nxt_unit_ctx_impl_t *ctx_impl);
static nxt_unit_port_t *nxt_unit_create_port(nxt_unit_ctx_t *ctx);

static int nxt_unit_send_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *dst,
    nxt_unit_port_t *port, int queue_fd);

nxt_inline void nxt_unit_port_use(nxt_unit_port_t *port);
nxt_inline void nxt_unit_port_release(nxt_unit_port_t *port);
static nxt_unit_port_t *nxt_unit_add_port(nxt_unit_ctx_t *ctx,
    nxt_unit_port_t *port, void *queue);
static void nxt_unit_process_awaiting_req(nxt_unit_ctx_t *ctx,
    nxt_queue_t *awaiting_req);
static void nxt_unit_remove_port(nxt_unit_impl_t *lib, nxt_unit_ctx_t *ctx,
    nxt_unit_port_id_t *port_id);
static nxt_unit_port_t *nxt_unit_remove_port_unsafe(nxt_unit_impl_t *lib,
    nxt_unit_port_id_t *port_id);
static void nxt_unit_remove_pid(nxt_unit_impl_t *lib, pid_t pid);
static void nxt_unit_remove_process(nxt_unit_impl_t *lib,
    nxt_unit_process_t *process);
static void nxt_unit_quit(nxt_unit_ctx_t *ctx, uint8_t quit_param);
static int nxt_unit_get_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id);
static ssize_t nxt_unit_port_send(nxt_unit_ctx_t *ctx,
    nxt_unit_port_t *port, const void *buf, size_t buf_size,
    const nxt_send_oob_t *oob);
static ssize_t nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, int fd,
    const void *buf, size_t buf_size, const nxt_send_oob_t *oob);
static int nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    nxt_unit_read_buf_t *rbuf);
nxt_inline void nxt_unit_rbuf_cpy(nxt_unit_read_buf_t *dst,
    nxt_unit_read_buf_t *src);
static int nxt_unit_shared_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    nxt_unit_read_buf_t *rbuf);
static int nxt_unit_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    nxt_unit_read_buf_t *rbuf);
static int nxt_unit_port_queue_recv(nxt_unit_port_t *port,
    nxt_unit_read_buf_t *rbuf);
static int nxt_unit_app_queue_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    nxt_unit_read_buf_t *rbuf);
nxt_inline int nxt_unit_close(int fd);
static int nxt_unit_fd_blocking(int fd);

static int nxt_unit_port_hash_add(nxt_lvlhsh_t *port_hash,
    nxt_unit_port_t *port);
static nxt_unit_port_t *nxt_unit_port_hash_find(nxt_lvlhsh_t *port_hash,
    nxt_unit_port_id_t *port_id, int remove);

static int nxt_unit_request_hash_add(nxt_unit_ctx_t *ctx,
    nxt_unit_request_info_t *req);
static nxt_unit_request_info_t *nxt_unit_request_hash_find(
    nxt_unit_ctx_t *ctx, uint32_t stream, int remove);

static char * nxt_unit_snprint_prefix(char *p, char *end, pid_t pid,
    int level);
static void *nxt_unit_lvlhsh_alloc(void *data, size_t size);
static void nxt_unit_lvlhsh_free(void *data, void *p);
static int nxt_unit_memcasecmp(const void *p1, const void *p2, size_t length);


struct nxt_unit_mmap_buf_s {
    nxt_unit_buf_t           buf;

    nxt_unit_mmap_buf_t      *next;
    nxt_unit_mmap_buf_t      **prev;

    nxt_port_mmap_header_t   *hdr;
    nxt_unit_request_info_t  *req;
    nxt_unit_ctx_impl_t      *ctx_impl;
    char                     *free_ptr;
    char                     *plain_ptr;
};


struct nxt_unit_recv_msg_s {
    uint32_t                 stream;
    nxt_pid_t                pid;
    nxt_port_id_t            reply_port;

    uint8_t                  last;      /* 1 bit */
    uint8_t                  mmap;      /* 1 bit */

    void                     *start;
    uint32_t                 size;

    int                      fd[2];

    nxt_unit_mmap_buf_t      *incoming_buf;
};


typedef enum {
    NXT_UNIT_RS_START           = 0,
    NXT_UNIT_RS_RESPONSE_INIT,
    NXT_UNIT_RS_RESPONSE_HAS_CONTENT,
    NXT_UNIT_RS_RESPONSE_SENT,
    NXT_UNIT_RS_RELEASED,
} nxt_unit_req_state_t;


struct nxt_unit_request_info_impl_s {
    nxt_unit_request_info_t  req;

    uint32_t                 stream;

    nxt_unit_mmap_buf_t      *outgoing_buf;
    nxt_unit_mmap_buf_t      *incoming_buf;

    nxt_unit_req_state_t     state;
    uint8_t                  websocket;
    uint8_t                  in_hash;

    /*  for nxt_unit_ctx_impl_t.free_req or active_req */
    nxt_queue_link_t         link;
    /*  for nxt_unit_port_impl_t.awaiting_req */
    nxt_queue_link_t         port_wait_link;

    char                     extra_data[];
};


struct nxt_unit_websocket_frame_impl_s {
    nxt_unit_websocket_frame_t  ws;

    nxt_unit_mmap_buf_t         *buf;

    nxt_queue_link_t            link;

    nxt_unit_ctx_impl_t         *ctx_impl;
};


struct nxt_unit_read_buf_s {
    nxt_queue_link_t              link;
    nxt_unit_ctx_impl_t           *ctx_impl;
    ssize_t                       size;
    nxt_recv_oob_t                oob;
    char                          buf[16384];
};


struct nxt_unit_ctx_impl_s {
    nxt_unit_ctx_t                ctx;

    nxt_atomic_t                  use_count;
    nxt_atomic_t                  wait_items;

    pthread_mutex_t               mutex;

    nxt_unit_port_t               *read_port;

    nxt_queue_link_t              link;

    nxt_unit_mmap_buf_t           *free_buf;

    /*  of nxt_unit_request_info_impl_t */
    nxt_queue_t                   free_req;

    /*  of nxt_unit_websocket_frame_impl_t */
    nxt_queue_t                   free_ws;

    /*  of nxt_unit_request_info_impl_t */
    nxt_queue_t                   active_req;

    /*  of nxt_unit_request_info_impl_t */
    nxt_lvlhsh_t                  requests;

    /*  of nxt_unit_request_info_impl_t */
    nxt_queue_t                   ready_req;

    /*  of nxt_unit_read_buf_t */
    nxt_queue_t                   pending_rbuf;

    /*  of nxt_unit_read_buf_t */
    nxt_queue_t                   free_rbuf;

    uint8_t                       online;       /* 1 bit */
    uint8_t                       ready;        /* 1 bit */
    uint8_t                       quit_param;

    nxt_unit_mmap_buf_t           ctx_buf[2];
    nxt_unit_read_buf_t           ctx_read_buf;

    nxt_unit_request_info_impl_t  req;
};


struct nxt_unit_mmap_s {
    nxt_port_mmap_header_t   *hdr;
    pthread_t                src_thread;

    /*  of nxt_unit_read_buf_t */
    nxt_queue_t              awaiting_rbuf;
};


struct nxt_unit_mmaps_s {
    pthread_mutex_t          mutex;
    uint32_t                 size;
    uint32_t                 cap;
    nxt_atomic_t             allocated_chunks;
    nxt_unit_mmap_t          *elts;
};


struct nxt_unit_impl_s {
    nxt_unit_t               unit;
    nxt_unit_callbacks_t     callbacks;

    nxt_atomic_t             use_count;
    nxt_atomic_t             request_count;

    uint32_t                 request_data_size;
    uint32_t                 shm_mmap_limit;
    uint32_t                 request_limit;

    pthread_mutex_t          mutex;

    nxt_lvlhsh_t             processes;        /* of nxt_unit_process_t */
    nxt_lvlhsh_t             ports;            /* of nxt_unit_port_impl_t */

    nxt_unit_port_t          *router_port;
    nxt_unit_port_t          *shared_port;

    nxt_queue_t              contexts;         /* of nxt_unit_ctx_impl_t */

    nxt_unit_mmaps_t         incoming;
    nxt_unit_mmaps_t         outgoing;

    pid_t                    pid;
    int                      log_fd;

    nxt_unit_ctx_impl_t      main_ctx;
};


struct nxt_unit_port_impl_s {
    nxt_unit_port_t          port;

    nxt_atomic_t             use_count;

    /*  for nxt_unit_process_t.ports */
    nxt_queue_link_t         link;
    nxt_unit_process_t       *process;

    /*  of nxt_unit_request_info_impl_t */
    nxt_queue_t              awaiting_req;

    int                      ready;

    void                     *queue;

    int                      from_socket;
    nxt_unit_read_buf_t      *socket_rbuf;
};


struct nxt_unit_process_s {
    pid_t                    pid;

    nxt_queue_t              ports;            /* of nxt_unit_port_impl_t */

    nxt_unit_impl_t          *lib;

    nxt_atomic_t             use_count;

    uint32_t                 next_port_id;
};


/* Explicitly using 32 bit types to avoid possible alignment. */
typedef struct {
    int32_t   pid;
    uint32_t  id;
} nxt_unit_port_hash_id_t;


static pid_t  nxt_unit_pid;


nxt_unit_ctx_t *
nxt_unit_init(nxt_unit_init_t *init)
{
    int              rc, queue_fd, shared_queue_fd;
    void             *mem;
    uint32_t         ready_stream, shm_limit, request_limit;
    nxt_unit_ctx_t   *ctx;
    nxt_unit_impl_t  *lib;
    nxt_unit_port_t  ready_port, router_port, read_port, shared_port;

    nxt_unit_pid = getpid();

    lib = nxt_unit_create(init);
    if (nxt_slow_path(lib == NULL)) {
        return NULL;
    }

    queue_fd = -1;
    mem = MAP_FAILED;
    shared_port.out_fd = -1;
    shared_port.data = NULL;

    if (init->ready_port.id.pid != 0
        && init->ready_stream != 0
        && init->read_port.id.pid != 0)
    {
        ready_port = init->ready_port;
        ready_stream = init->ready_stream;
        router_port = init->router_port;
        read_port = init->read_port;
        lib->log_fd = init->log_fd;

        nxt_unit_port_id_init(&ready_port.id, ready_port.id.pid,
                              ready_port.id.id);
        nxt_unit_port_id_init(&router_port.id, router_port.id.pid,
                              router_port.id.id);
        nxt_unit_port_id_init(&read_port.id, read_port.id.pid,
                              read_port.id.id);

        shared_port.in_fd = init->shared_port_fd;
        shared_queue_fd = init->shared_queue_fd;

    } else {
        rc = nxt_unit_read_env(&ready_port, &router_port, &read_port,
                               &shared_port.in_fd, &shared_queue_fd,
                               &lib->log_fd, &ready_stream, &shm_limit,
                               &request_limit);
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            goto fail;
        }

        lib->shm_mmap_limit = (shm_limit + PORT_MMAP_DATA_SIZE - 1)
                                / PORT_MMAP_DATA_SIZE;
        lib->request_limit = request_limit;
    }

    if (nxt_slow_path(lib->shm_mmap_limit < 1)) {
        lib->shm_mmap_limit = 1;
    }

    lib->pid = read_port.id.pid;
    nxt_unit_pid = lib->pid;

    ctx = &lib->main_ctx.ctx;

    rc = nxt_unit_fd_blocking(router_port.out_fd);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        goto fail;
    }

    lib->router_port = nxt_unit_add_port(ctx, &router_port, NULL);
    if (nxt_slow_path(lib->router_port == NULL)) {
        nxt_unit_alert(NULL, "failed to add router_port");

        goto fail;
    }

    queue_fd = nxt_unit_shm_open(ctx, sizeof(nxt_port_queue_t));
    if (nxt_slow_path(queue_fd == -1)) {
        goto fail;
    }

    mem = mmap(NULL, sizeof(nxt_port_queue_t),
               PROT_READ | PROT_WRITE, MAP_SHARED, queue_fd, 0);
    if (nxt_slow_path(mem == MAP_FAILED)) {
        nxt_unit_alert(ctx, "mmap(%d) failed: %s (%d)", queue_fd,
                       strerror(errno), errno);

        goto fail;
    }

    nxt_port_queue_init(mem);

    rc = nxt_unit_fd_blocking(read_port.in_fd);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        goto fail;
    }

    lib->main_ctx.read_port = nxt_unit_add_port(ctx, &read_port, mem);
    if (nxt_slow_path(lib->main_ctx.read_port == NULL)) {
        nxt_unit_alert(NULL, "failed to add read_port");

        goto fail;
    }

    rc = nxt_unit_fd_blocking(ready_port.out_fd);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        goto fail;
    }

    nxt_unit_port_id_init(&shared_port.id, read_port.id.pid,
                          NXT_UNIT_SHARED_PORT_ID);

    mem = mmap(NULL, sizeof(nxt_app_queue_t), PROT_READ | PROT_WRITE,
               MAP_SHARED, shared_queue_fd, 0);
    if (nxt_slow_path(mem == MAP_FAILED)) {
        nxt_unit_alert(ctx, "mmap(%d) failed: %s (%d)", shared_queue_fd,
                       strerror(errno), errno);

        goto fail;
    }

    nxt_unit_close(shared_queue_fd);

    lib->shared_port = nxt_unit_add_port(ctx, &shared_port, mem);
    if (nxt_slow_path(lib->shared_port == NULL)) {
        nxt_unit_alert(NULL, "failed to add shared_port");

        goto fail;
    }

    rc = nxt_unit_ready(ctx, ready_port.out_fd, ready_stream, queue_fd);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        nxt_unit_alert(NULL, "failed to send READY message");

        goto fail;
    }

    nxt_unit_close(ready_port.out_fd);
    nxt_unit_close(queue_fd);

    return ctx;

fail:

    if (mem != MAP_FAILED) {
        munmap(mem, sizeof(nxt_port_queue_t));
    }

    if (queue_fd != -1) {
        nxt_unit_close(queue_fd);
    }

    nxt_unit_ctx_release(&lib->main_ctx.ctx);

    return NULL;
}


static nxt_unit_impl_t *
nxt_unit_create(nxt_unit_init_t *init)
{
    int              rc;
    nxt_unit_impl_t  *lib;

    if (nxt_slow_path(init->callbacks.request_handler == NULL)) {
        nxt_unit_alert(NULL, "request_handler is NULL");

        return NULL;
    }

    lib = nxt_unit_malloc(NULL,
                          sizeof(nxt_unit_impl_t) + init->request_data_size);
    if (nxt_slow_path(lib == NULL)) {
        nxt_unit_alert(NULL, "failed to allocate unit struct");

        return NULL;
    }

    rc = pthread_mutex_init(&lib->mutex, NULL);
    if (nxt_slow_path(rc != 0)) {
        nxt_unit_alert(NULL, "failed to initialize mutex (%d)", rc);

        goto out_unit_free;
    }

    lib->unit.data = init->data;
    lib->callbacks = init->callbacks;

    lib->request_data_size = init->request_data_size;
    lib->shm_mmap_limit = (init->shm_limit + PORT_MMAP_DATA_SIZE - 1)
                            / PORT_MMAP_DATA_SIZE;
    lib->request_limit = init->request_limit;

    lib->processes.slot = NULL;
    lib->ports.slot = NULL;

    lib->log_fd = STDERR_FILENO;

    nxt_queue_init(&lib->contexts);

    lib->use_count = 0;
    lib->request_count = 0;
    lib->router_port = NULL;
    lib->shared_port = NULL;

    rc = nxt_unit_ctx_init(lib, &lib->main_ctx, init->ctx_data);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        goto out_mutex_destroy;
    }

    rc = nxt_unit_mmaps_init(&lib->incoming);
    if (nxt_slow_path(rc != 0)) {
        nxt_unit_alert(NULL, "failed to initialize mutex (%d)", rc);

        goto out_ctx_free;
    }

    rc = nxt_unit_mmaps_init(&lib->outgoing);
    if (nxt_slow_path(rc != 0)) {
        nxt_unit_alert(NULL, "failed to initialize mutex (%d)", rc);

        goto out_mmaps_destroy;
    }

    return lib;

out_mmaps_destroy:
    nxt_unit_mmaps_destroy(&lib->incoming);

out_ctx_free:
    nxt_unit_ctx_free(&lib->main_ctx);

out_mutex_destroy:
    pthread_mutex_destroy(&lib->mutex);

out_unit_free:
    nxt_unit_free(NULL, lib);

    return NULL;
}


static int
nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl,
    void *data)
{
    int  rc;

    ctx_impl->ctx.data = data;
    ctx_impl->ctx.unit = &lib->unit;

    rc = pthread_mutex_init(&ctx_impl->mutex, NULL);
    if (nxt_slow_path(rc != 0)) {
        nxt_unit_alert(NULL, "failed to initialize mutex (%d)", rc);

        return NXT_UNIT_ERROR;
    }

    nxt_unit_lib_use(lib);

    pthread_mutex_lock(&lib->mutex);

    nxt_queue_insert_tail(&lib->contexts, &ctx_impl->link);

    pthread_mutex_unlock(&lib->mutex);

    ctx_impl->use_count = 1;
    ctx_impl->wait_items = 0;
    ctx_impl->online = 1;
    ctx_impl->ready = 0;
    ctx_impl->quit_param = NXT_QUIT_GRACEFUL;

    nxt_queue_init(&ctx_impl->free_req);
    nxt_queue_init(&ctx_impl->free_ws);
    nxt_queue_init(&ctx_impl->active_req);
    nxt_queue_init(&ctx_impl->ready_req);
    nxt_queue_init(&ctx_impl->pending_rbuf);
    nxt_queue_init(&ctx_impl->free_rbuf);

    ctx_impl->free_buf = NULL;
    nxt_unit_mmap_buf_insert(&ctx_impl->free_buf, &ctx_impl->ctx_buf[1]);
    nxt_unit_mmap_buf_insert(&ctx_impl->free_buf, &ctx_impl->ctx_buf[0]);

    nxt_queue_insert_tail(&ctx_impl->free_req, &ctx_impl->req.link);
    nxt_queue_insert_tail(&ctx_impl->free_rbuf, &ctx_impl->ctx_read_buf.link);

    ctx_impl->ctx_read_buf.ctx_impl = ctx_impl;

    ctx_impl->req.req.ctx = &ctx_impl->ctx;
    ctx_impl->req.req.unit = &lib->unit;

    ctx_impl->read_port = NULL;
    ctx_impl->requests.slot = 0;

    return NXT_UNIT_OK;
}


nxt_inline void
nxt_unit_ctx_use(nxt_unit_ctx_t *ctx)
{
    nxt_unit_ctx_impl_t  *ctx_impl;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    nxt_atomic_fetch_add(&ctx_impl->use_count, 1);
}


nxt_inline void
nxt_unit_ctx_release(nxt_unit_ctx_t *ctx)
{
    long                 c;
    nxt_unit_ctx_impl_t  *ctx_impl;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    c = nxt_atomic_fetch_add(&ctx_impl->use_count, -1);

    if (c == 1) {
        nxt_unit_ctx_free(ctx_impl);
    }
}


nxt_inline void
nxt_unit_lib_use(nxt_unit_impl_t *lib)
{
    nxt_atomic_fetch_add(&lib->use_count, 1);
}


nxt_inline void
nxt_unit_lib_release(nxt_unit_impl_t *lib)
{
    long                c;
    nxt_unit_process_t  *process;

    c = nxt_atomic_fetch_add(&lib->use_count, -1);

    if (c == 1) {
        for ( ;; ) {
            pthread_mutex_lock(&lib->mutex);

            process = nxt_unit_process_pop_first(lib);
            if (process == NULL) {
                pthread_mutex_unlock(&lib->mutex);

                break;
            }

            nxt_unit_remove_process(lib, process);
        }

        pthread_mutex_destroy(&lib->mutex);

        if (nxt_fast_path(lib->router_port != NULL)) {
            nxt_unit_port_release(lib->router_port);
        }

        if (nxt_fast_path(lib->shared_port != NULL)) {
            nxt_unit_port_release(lib->shared_port);
        }

        nxt_unit_mmaps_destroy(&lib->incoming);
        nxt_unit_mmaps_destroy(&lib->outgoing);

        nxt_unit_free(NULL, lib);
    }
}


nxt_inline void
nxt_unit_mmap_buf_insert(nxt_unit_mmap_buf_t **head,
    nxt_unit_mmap_buf_t *mmap_buf)
{
    mmap_buf->next = *head;

    if (mmap_buf->next != NULL) {
        mmap_buf->next->prev = &mmap_buf->next;
    }

    *head = mmap_buf;
    mmap_buf->prev = head;
}


nxt_inline void
nxt_unit_mmap_buf_insert_tail(nxt_unit_mmap_buf_t **prev,
    nxt_unit_mmap_buf_t *mmap_buf)
{
    while (*prev != NULL) {
        prev = &(*prev)->next;
    }

    nxt_unit_mmap_buf_insert(prev, mmap_buf);
}


nxt_inline void
nxt_unit_mmap_buf_unlink(nxt_unit_mmap_buf_t *mmap_buf)
{
    nxt_unit_mmap_buf_t  **prev;

    prev = mmap_buf->prev;

    if (mmap_buf->next != NULL) {
        mmap_buf->next->prev = prev;
    }

    if (prev != NULL) {
        *prev = mmap_buf->next;
    }
}


static int
nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *router_port,
    nxt_unit_port_t *read_port, int *shared_port_fd, int *shared_queue_fd,
    int *log_fd, uint32_t *stream,
    uint32_t *shm_limit, uint32_t *request_limit)
{
    int       rc;
    int       ready_fd, router_fd, read_in_fd, read_out_fd;
    char      *unit_init, *version_end, *vars;
    size_t    version_length;
    int64_t   ready_pid, router_pid, read_pid;
    uint32_t  ready_stream, router_id, ready_id, read_id;

    unit_init = getenv(NXT_UNIT_INIT_ENV);
    if (nxt_slow_path(unit_init == NULL)) {
        nxt_unit_alert(NULL, "%s is not in the current environment",
                       NXT_UNIT_INIT_ENV);

        return NXT_UNIT_ERROR;
    }

    version_end = strchr(unit_init, ';');
    if (nxt_slow_path(version_end == NULL)) {
        nxt_unit_alert(NULL, "Unit version not found in %s=\"%s\"",
                       NXT_UNIT_INIT_ENV, unit_init);

        return NXT_UNIT_ERROR;
    }

    version_length = version_end - unit_init;

    rc = version_length != nxt_length(NXT_VERSION)
         || memcmp(unit_init, NXT_VERSION, nxt_length(NXT_VERSION));

    if (nxt_slow_path(rc != 0)) {
        nxt_unit_alert(NULL, "versions mismatch: the Unit daemon has version "
                       "%.*s, while the app was compiled with libunit %s",
                       (int) version_length, unit_init, NXT_VERSION);

        return NXT_UNIT_ERROR;
    }

    vars = version_end + 1;

    rc = sscanf(vars,
                "%"PRIu32";"
                "%"PRId64",%"PRIu32",%d;"
                "%"PRId64",%"PRIu32",%d;"
                "%"PRId64",%"PRIu32",%d,%d;"
                "%d,%d;"
                "%d,%"PRIu32",%"PRIu32,
                &ready_stream,
                &ready_pid, &ready_id, &ready_fd,
                &router_pid, &router_id, &router_fd,
                &read_pid, &read_id, &read_in_fd, &read_out_fd,
                shared_port_fd, shared_queue_fd,
                log_fd, shm_limit, request_limit);

    if (nxt_slow_path(rc == EOF)) {
        nxt_unit_alert(NULL, "sscanf(%s) failed: %s (%d) for %s env",
                       vars, strerror(errno), errno, NXT_UNIT_INIT_ENV);

        return NXT_UNIT_ERROR;
    }

    if (nxt_slow_path(rc != 16)) {
        nxt_unit_alert(NULL, "invalid number of variables in %s env: "
                       "found %d of %d in %s", NXT_UNIT_INIT_ENV, rc, 16, vars);

        return NXT_UNIT_ERROR;
    }

    nxt_unit_debug(NULL, "%s='%s'", NXT_UNIT_INIT_ENV, unit_init);

    nxt_unit_port_id_init(&ready_port->id, (pid_t) ready_pid, ready_id);

    ready_port->in_fd = -1;
    ready_port->out_fd = ready_fd;
    ready_port->data = NULL;

    nxt_unit_port_id_init(&router_port->id, (pid_t) router_pid, router_id);

    router_port->in_fd = -1;
    router_port->out_fd = router_fd;
    router_port->data = NULL;

    nxt_unit_port_id_init(&read_port->id, (pid_t) read_pid, read_id);

    read_port->in_fd = read_in_fd;
    read_port->out_fd = read_out_fd;
    read_port->data = NULL;

    *stream = ready_stream;

    return NXT_UNIT_OK;
}


static int
nxt_unit_ready(nxt_unit_ctx_t *ctx, int ready_fd, uint32_t stream, int queue_fd)
{
    ssize_t          res;
    nxt_send_oob_t   oob;
    nxt_port_msg_t   msg;
    nxt_unit_impl_t  *lib;
    int              fds[2] = {queue_fd, -1};

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    msg.stream = stream;
    msg.pid = lib->pid;
    msg.reply_port = 0;
    msg.type = _NXT_PORT_MSG_PROCESS_READY;
    msg.last = 1;
    msg.mmap = 0;
    msg.nf = 0;
    msg.mf = 0;

    nxt_socket_msg_oob_init(&oob, fds);

    res = nxt_unit_sendmsg(ctx, ready_fd, &msg, sizeof(msg), &oob);
    if (res != sizeof(msg)) {
        return NXT_UNIT_ERROR;
    }

    return NXT_UNIT_OK;
}


static int
nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf,
    nxt_unit_request_info_t **preq)
{
    int                  rc;
    pid_t                pid;
    uint8_t              quit_param;
    nxt_port_msg_t       *port_msg;
    nxt_unit_impl_t      *lib;
    nxt_unit_recv_msg_t  recv_msg;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    recv_msg.incoming_buf = NULL;
    recv_msg.fd[0] = -1;
    recv_msg.fd[1] = -1;

    rc = nxt_socket_msg_oob_get_fds(&rbuf->oob, recv_msg.fd);
    if (nxt_slow_path(rc != NXT_OK)) {
        nxt_unit_alert(ctx, "failed to receive file descriptor over cmsg");
        rc = NXT_UNIT_ERROR;
        goto done;
    }

    if (nxt_slow_path(rbuf->size < (ssize_t) sizeof(nxt_port_msg_t))) {
        if (nxt_slow_path(rbuf->size == 0)) {
            nxt_unit_debug(ctx, "read port closed");

            nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL);
            rc = NXT_UNIT_OK;
            goto done;
        }

        nxt_unit_alert(ctx, "message too small (%d bytes)", (int) rbuf->size);

        rc = NXT_UNIT_ERROR;
        goto done;
    }

    port_msg = (nxt_port_msg_t *) rbuf->buf;

    nxt_unit_debug(ctx, "#%"PRIu32": process message %d fd[0] %d fd[1] %d",
                   port_msg->stream, (int) port_msg->type,
                   recv_msg.fd[0], recv_msg.fd[1]);

    recv_msg.stream = port_msg->stream;
    recv_msg.pid = port_msg->pid;
    recv_msg.reply_port = port_msg->reply_port;
    recv_msg.last = port_msg->last;
    recv_msg.mmap = port_msg->mmap;

    recv_msg.start = port_msg + 1;
    recv_msg.size = rbuf->size - sizeof(nxt_port_msg_t);

    if (nxt_slow_path(port_msg->type >= NXT_PORT_MSG_MAX)) {
        nxt_unit_alert(ctx, "#%"PRIu32": unknown message type (%d)",
                       port_msg->stream, (int) port_msg->type);
        rc = NXT_UNIT_ERROR;
        goto done;
    }

    /* Fragmentation is unsupported. */
    if (nxt_slow_path(port_msg->nf != 0 || port_msg->mf != 0)) {
        nxt_unit_alert(ctx, "#%"PRIu32": fragmented message type (%d)",
                       port_msg->stream, (int) port_msg->type);
        rc = NXT_UNIT_ERROR;
        goto done;
    }

    if (port_msg->mmap) {
        rc = nxt_unit_mmap_read(ctx, &recv_msg, rbuf);

        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            if (rc == NXT_UNIT_AGAIN) {
                recv_msg.fd[0] = -1;
                recv_msg.fd[1] = -1;
            }

            goto done;
        }
    }

    switch (port_msg->type) {

    case _NXT_PORT_MSG_RPC_READY:
        rc = NXT_UNIT_OK;
        break;

    case _NXT_PORT_MSG_QUIT:
        if (recv_msg.size == sizeof(quit_param)) {
            memcpy(&quit_param, recv_msg.start, sizeof(quit_param));

        } else {
            quit_param = NXT_QUIT_NORMAL;
        }

        nxt_unit_debug(ctx, "#%"PRIu32": %squit", port_msg->stream,
                       (quit_param == NXT_QUIT_GRACEFUL ? "graceful " : ""));

        nxt_unit_quit(ctx, quit_param);

        rc = NXT_UNIT_OK;
        break;

    case _NXT_PORT_MSG_NEW_PORT:
        rc = nxt_unit_process_new_port(ctx, &recv_msg);
        break;

    case _NXT_PORT_MSG_PORT_ACK:
        rc = nxt_unit_ctx_ready(ctx);
        break;

    case _NXT_PORT_MSG_CHANGE_FILE:
        nxt_unit_debug(ctx, "#%"PRIu32": change_file: fd %d",
                       port_msg->stream, recv_msg.fd[0]);

        if (dup2(recv_msg.fd[0], lib->log_fd) == -1) {
            nxt_unit_alert(ctx, "#%"PRIu32": dup2(%d, %d) failed: %s (%d)",
                           port_msg->stream, recv_msg.fd[0], lib->log_fd,
                           strerror(errno), errno);

            rc = NXT_UNIT_ERROR;
            goto done;
        }

        rc = NXT_UNIT_OK;
        break;

    case _NXT_PORT_MSG_MMAP:
        if (nxt_slow_path(recv_msg.fd[0] < 0)) {
            nxt_unit_alert(ctx, "#%"PRIu32": invalid fd %d for mmap",
                           port_msg->stream, recv_msg.fd[0]);

            rc = NXT_UNIT_ERROR;
            goto done;
        }

        rc = nxt_unit_incoming_mmap(ctx, port_msg->pid, recv_msg.fd[0]);
        break;

    case _NXT_PORT_MSG_REQ_HEADERS:
        rc = nxt_unit_process_req_headers(ctx, &recv_msg, preq);
        break;

    case _NXT_PORT_MSG_REQ_BODY:
        rc = nxt_unit_process_req_body(ctx, &recv_msg);
        break;

    case _NXT_PORT_MSG_WEBSOCKET:
        rc = nxt_unit_process_websocket(ctx, &recv_msg);
        break;

    case _NXT_PORT_MSG_REMOVE_PID:
        if (nxt_slow_path(recv_msg.size != sizeof(pid))) {
            nxt_unit_alert(ctx, "#%"PRIu32": remove_pid: invalid message size "
                           "(%d != %d)", port_msg->stream, (int) recv_msg.size,
                           (int) sizeof(pid));

            rc = NXT_UNIT_ERROR;
            goto done;
        }

        memcpy(&pid, recv_msg.start, sizeof(pid));

        nxt_unit_debug(ctx, "#%"PRIu32": remove_pid: %d",
                       port_msg->stream, (int) pid);

        nxt_unit_remove_pid(lib, pid);

        rc = NXT_UNIT_OK;
        break;

    case _NXT_PORT_MSG_SHM_ACK:
        rc = nxt_unit_process_shm_ack(ctx);
        break;

    default:
        nxt_unit_alert(ctx, "#%"PRIu32": ignore message type: %d",
                       port_msg->stream, (int) port_msg->type);

        rc = NXT_UNIT_ERROR;
        goto done;
    }

done:

    if (recv_msg.fd[0] != -1) {
        nxt_unit_close(recv_msg.fd[0]);
    }

    if (recv_msg.fd[1] != -1) {
        nxt_unit_close(recv_msg.fd[1]);
    }

    while (recv_msg.incoming_buf != NULL) {
        nxt_unit_mmap_buf_free(recv_msg.incoming_buf);
    }

    if (nxt_fast_path(rc != NXT_UNIT_AGAIN)) {
#if (NXT_DEBUG)
        memset(rbuf->buf, 0xAC, rbuf->size);
#endif
        nxt_unit_read_buf_release(ctx, rbuf);
    }

    return rc;
}


static int
nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg)
{
    void                     *mem;
    nxt_unit_port_t          new_port, *port;
    nxt_port_msg_new_port_t  *new_port_msg;

    if (nxt_slow_path(recv_msg->size != sizeof(nxt_port_msg_new_port_t))) {
        nxt_unit_warn(ctx, "#%"PRIu32": new_port: "
                      "invalid message size (%d)",
                      recv_msg->stream, (int) recv_msg->size);

        return NXT_UNIT_ERROR;
    }

    if (nxt_slow_path(recv_msg->fd[0] < 0)) {
        nxt_unit_alert(ctx, "#%"PRIu32": invalid fd %d for new port",
                       recv_msg->stream, recv_msg->fd[0]);

        return NXT_UNIT_ERROR;
    }

    new_port_msg = recv_msg->start;

    nxt_unit_debug(ctx, "#%"PRIu32": new_port: port{%d,%d} fd[0] %d fd[1] %d",
                   recv_msg->stream, (int) new_port_msg->pid,
                   (int) new_port_msg->id, recv_msg->fd[0], recv_msg->fd[1]);

    if (nxt_slow_path(nxt_unit_fd_blocking(recv_msg->fd[0]) != NXT_UNIT_OK)) {
        return NXT_UNIT_ERROR;
    }

    nxt_unit_port_id_init(&new_port.id, new_port_msg->pid, new_port_msg->id);

    new_port.in_fd = -1;
    new_port.out_fd = recv_msg->fd[0];

    mem = mmap(NULL, sizeof(nxt_port_queue_t), PROT_READ | PROT_WRITE,
               MAP_SHARED, recv_msg->fd[1], 0);

    if (nxt_slow_path(mem == MAP_FAILED)) {
        nxt_unit_alert(ctx, "mmap(%d) failed: %s (%d)", recv_msg->fd[1],
                       strerror(errno), errno);

        return NXT_UNIT_ERROR;
    }

    new_port.data = NULL;

    recv_msg->fd[0] = -1;

    port = nxt_unit_add_port(ctx, &new_port, mem);
    if (nxt_slow_path(port == NULL)) {
        return NXT_UNIT_ERROR;
    }

    nxt_unit_port_release(port);

    return NXT_UNIT_OK;
}


static int
nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx)
{
    nxt_unit_impl_t      *lib;
    nxt_unit_ctx_impl_t  *ctx_impl;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    if (nxt_slow_path(ctx_impl->ready)) {
        return NXT_UNIT_OK;
    }

    ctx_impl->ready = 1;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    /* Call ready_handler() only for main context. */
    if (&lib->main_ctx == ctx_impl && lib->callbacks.ready_handler != NULL) {
        return lib->callbacks.ready_handler(ctx);
    }

    if (&lib->main_ctx != ctx_impl) {
        /* Check if the main context is already stopped or quit. */
        if (nxt_slow_path(!lib->main_ctx.ready)) {
            ctx_impl->ready = 0;

            nxt_unit_quit(ctx, lib->main_ctx.quit_param);

            return NXT_UNIT_OK;
        }

        if (lib->callbacks.add_port != NULL) {
            lib->callbacks.add_port(ctx, lib->shared_port);
        }
    }

    return NXT_UNIT_OK;
}


static int
nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg,
    nxt_unit_request_info_t **preq)
{
    int                           res;
    nxt_unit_impl_t               *lib;
    nxt_unit_port_id_t            port_id;
    nxt_unit_request_t            *r;
    nxt_unit_mmap_buf_t           *b;
    nxt_unit_request_info_t       *req;
    nxt_unit_request_info_impl_t  *req_impl;

    if (nxt_slow_path(recv_msg->mmap == 0)) {
        nxt_unit_warn(ctx, "#%"PRIu32": data is not in shared memory",
                      recv_msg->stream);

        return NXT_UNIT_ERROR;
    }

    if (nxt_slow_path(recv_msg->size < sizeof(nxt_unit_request_t))) {
        nxt_unit_warn(ctx, "#%"PRIu32": data too short: %d while at least "
                      "%d expected", recv_msg->stream, (int) recv_msg->size,
                      (int) sizeof(nxt_unit_request_t));

        return NXT_UNIT_ERROR;
    }

    req_impl = nxt_unit_request_info_get(ctx);
    if (nxt_slow_path(req_impl == NULL)) {
        nxt_unit_warn(ctx, "#%"PRIu32": request info allocation failed",
                      recv_msg->stream);

        return NXT_UNIT_ERROR;
    }

    req = &req_impl->req;

    req->request = recv_msg->start;

    b = recv_msg->incoming_buf;

    req->request_buf = &b->buf;
    req->response = NULL;
    req->response_buf = NULL;

    r = req->request;

    req->content_length = r->content_length;

    req->content_buf = req->request_buf;
    req->content_buf->free = nxt_unit_sptr_get(&r->preread_content);

    req_impl->stream = recv_msg->stream;

    req_impl->outgoing_buf = NULL;

    for (b = recv_msg->incoming_buf; b != NULL; b = b->next) {
        b->req = req;
    }

    /* "Move" incoming buffer list to req_impl. */
    req_impl->incoming_buf = recv_msg->incoming_buf;
    req_impl->incoming_buf->prev = &req_impl->incoming_buf;
    recv_msg->incoming_buf = NULL;

    req->content_fd = recv_msg->fd[0];
    recv_msg->fd[0] = -1;

    req->response_max_fields = 0;
    req_impl->state = NXT_UNIT_RS_START;
    req_impl->websocket = 0;
    req_impl->in_hash = 0;

    nxt_unit_debug(ctx, "#%"PRIu32": %.*s %.*s (%d)", recv_msg->stream,
                   (int) r->method_length,
                   (char *) nxt_unit_sptr_get(&r->method),
                   (int) r->target_length,
                   (char *) nxt_unit_sptr_get(&r->target),
                   (int) r->content_length);

    nxt_unit_port_id_init(&port_id, recv_msg->pid, recv_msg->reply_port);

    res = nxt_unit_request_check_response_port(req, &port_id);
    if (nxt_slow_path(res == NXT_UNIT_ERROR)) {
        return NXT_UNIT_ERROR;
    }

    if (nxt_fast_path(res == NXT_UNIT_OK)) {
        res = nxt_unit_send_req_headers_ack(req);
        if (nxt_slow_path(res == NXT_UNIT_ERROR)) {
            nxt_unit_request_done(req, NXT_UNIT_ERROR);

            return NXT_UNIT_ERROR;
        }

        lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

        if (req->content_length
            > (uint64_t) (req->content_buf->end - req->content_buf->free))
        {
            res = nxt_unit_request_hash_add(ctx, req);
            if (nxt_slow_path(res != NXT_UNIT_OK)) {
                nxt_unit_req_warn(req, "failed to add request to hash");

                nxt_unit_request_done(req, NXT_UNIT_ERROR);

                return NXT_UNIT_ERROR;
            }

            /*
             * If application have separate data handler, we may start
             * request processing and process data when it is arrived.
             */
            if (lib->callbacks.data_handler == NULL) {
                return NXT_UNIT_OK;
            }
        }

        if (preq == NULL) {
            lib->callbacks.request_handler(req);

        } else {
            *preq = req;
        }
    }

    return NXT_UNIT_OK;
}


static int
nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg)
{
    uint64_t                 l;
    nxt_unit_impl_t          *lib;
    nxt_unit_mmap_buf_t      *b;
    nxt_unit_request_info_t  *req;

    req = nxt_unit_request_hash_find(ctx, recv_msg->stream, recv_msg->last);
    if (req == NULL) {
        return NXT_UNIT_OK;
    }

    l = req->content_buf->end - req->content_buf->free;

    for (b = recv_msg->incoming_buf; b != NULL; b = b->next) {
        b->req = req;
        l += b->buf.end - b->buf.free;
    }

    if (recv_msg->incoming_buf != NULL) {
        b = nxt_container_of(req->content_buf, nxt_unit_mmap_buf_t, buf);

        while (b->next != NULL) {
            b = b->next;
        }

        /* "Move" incoming buffer list to req_impl. */
        b->next = recv_msg->incoming_buf;
        b->next->prev = &b->next;

        recv_msg->incoming_buf = NULL;
    }

    req->content_fd = recv_msg->fd[0];
    recv_msg->fd[0] = -1;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    if (lib->callbacks.data_handler != NULL) {
        lib->callbacks.data_handler(req);

        return NXT_UNIT_OK;
    }

    if (req->content_fd != -1 || l == req->content_length) {
        lib->callbacks.request_handler(req);
    }

    return NXT_UNIT_OK;
}


static int
nxt_unit_request_check_response_port(nxt_unit_request_info_t *req,
    nxt_unit_port_id_t *port_id)
{
    int                           res;
    nxt_unit_ctx_t                *ctx;
    nxt_unit_impl_t               *lib;
    nxt_unit_port_t               *port;
    nxt_unit_process_t            *process;
    nxt_unit_ctx_impl_t           *ctx_impl;
    nxt_unit_port_impl_t          *port_impl;
    nxt_unit_request_info_impl_t  *req_impl;

    ctx = req->ctx;
    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);
    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    pthread_mutex_lock(&lib->mutex);

    port = nxt_unit_port_hash_find(&lib->ports, port_id, 0);

    if (nxt_fast_path(port != NULL)) {
        port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port);
        req->response_port = port;

        if (nxt_fast_path(port_impl->ready)) {
            pthread_mutex_unlock(&lib->mutex);

            nxt_unit_debug(ctx, "check_response_port: found port{%d,%d}",
                           (int) port->id.pid, (int) port->id.id);

            return NXT_UNIT_OK;
        }

        nxt_unit_debug(ctx, "check_response_port: "
                       "port{%d,%d} already requested",
                       (int) port->id.pid, (int) port->id.id);

        req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

        nxt_queue_insert_tail(&port_impl->awaiting_req,
                              &req_impl->port_wait_link);

        pthread_mutex_unlock(&lib->mutex);

        nxt_atomic_fetch_add(&ctx_impl->wait_items, 1);

        return NXT_UNIT_AGAIN;
    }

    port_impl = nxt_unit_malloc(ctx, sizeof(nxt_unit_port_impl_t));
    if (nxt_slow_path(port_impl == NULL)) {
        nxt_unit_alert(ctx, "check_response_port: malloc(%d) failed",
                       (int) sizeof(nxt_unit_port_impl_t));

        pthread_mutex_unlock(&lib->mutex);

        return NXT_UNIT_ERROR;
    }

    port = &port_impl->port;

    port->id = *port_id;
    port->in_fd = -1;
    port->out_fd = -1;
    port->data = NULL;

    res = nxt_unit_port_hash_add(&lib->ports, port);
    if (nxt_slow_path(res != NXT_UNIT_OK)) {
        nxt_unit_alert(ctx, "check_response_port: %d,%d hash_add failed",
                       port->id.pid, port->id.id);

        pthread_mutex_unlock(&lib->mutex);

        nxt_unit_free(ctx, port);

        return NXT_UNIT_ERROR;
    }

    process = nxt_unit_process_find(lib, port_id->pid, 0);
    if (nxt_slow_path(process == NULL)) {
        nxt_unit_alert(ctx, "check_response_port: process %d not found",
                       port->id.pid);

        nxt_unit_port_hash_find(&lib->ports, port_id, 1);

        pthread_mutex_unlock(&lib->mutex);

        nxt_unit_free(ctx, port);

        return NXT_UNIT_ERROR;
    }

    nxt_queue_insert_tail(&process->ports, &port_impl->link);

    port_impl->process = process;
    port_impl->queue = NULL;
    port_impl->from_socket = 0;
    port_impl->socket_rbuf = NULL;

    nxt_queue_init(&port_impl->awaiting_req);

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    nxt_queue_insert_tail(&port_impl->awaiting_req, &req_impl->port_wait_link);

    port_impl->use_count = 2;
    port_impl->ready = 0;

    req->response_port = port;

    pthread_mutex_unlock(&lib->mutex);

    res = nxt_unit_get_port(ctx, port_id);
    if (nxt_slow_path(res == NXT_UNIT_ERROR)) {
        return NXT_UNIT_ERROR;
    }

    nxt_atomic_fetch_add(&ctx_impl->wait_items, 1);

    return NXT_UNIT_AGAIN;
}


static int
nxt_unit_send_req_headers_ack(nxt_unit_request_info_t *req)
{
    ssize_t                       res;
    nxt_port_msg_t                msg;
    nxt_unit_impl_t               *lib;
    nxt_unit_ctx_impl_t           *ctx_impl;
    nxt_unit_request_info_impl_t  *req_impl;

    lib = nxt_container_of(req->ctx->unit, nxt_unit_impl_t, unit);
    ctx_impl = nxt_container_of(req->ctx, nxt_unit_ctx_impl_t, ctx);
    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    memset(&msg, 0, sizeof(nxt_port_msg_t));

    msg.stream = req_impl->stream;
    msg.pid = lib->pid;
    msg.reply_port = ctx_impl->read_port->id.id;
    msg.type = _NXT_PORT_MSG_REQ_HEADERS_ACK;

    res = nxt_unit_port_send(req->ctx, req->response_port,
                             &msg, sizeof(msg), NULL);
    if (nxt_slow_path(res != sizeof(msg))) {
        return NXT_UNIT_ERROR;
    }

    return NXT_UNIT_OK;
}


static int
nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg)
{
    size_t                           hsize;
    nxt_unit_impl_t                  *lib;
    nxt_unit_mmap_buf_t              *b;
    nxt_unit_callbacks_t             *cb;
    nxt_unit_request_info_t          *req;
    nxt_unit_request_info_impl_t     *req_impl;
    nxt_unit_websocket_frame_impl_t  *ws_impl;

    req = nxt_unit_request_hash_find(ctx, recv_msg->stream, recv_msg->last);
    if (nxt_slow_path(req == NULL)) {
        return NXT_UNIT_OK;
    }

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);
    cb = &lib->callbacks;

    if (cb->websocket_handler && recv_msg->size >= 2) {
        ws_impl = nxt_unit_websocket_frame_get(ctx);
        if (nxt_slow_path(ws_impl == NULL)) {
            nxt_unit_warn(ctx, "#%"PRIu32": websocket frame allocation failed",
                          req_impl->stream);

            return NXT_UNIT_ERROR;
        }

        ws_impl->ws.req = req;

        ws_impl->buf = NULL;

        if (recv_msg->mmap) {
            for (b = recv_msg->incoming_buf; b != NULL; b = b->next) {
                b->req = req;
            }

            /* "Move" incoming buffer list to ws_impl. */
            ws_impl->buf = recv_msg->incoming_buf;
            ws_impl->buf->prev = &ws_impl->buf;
            recv_msg->incoming_buf = NULL;

            b = ws_impl->buf;

        } else {
            b = nxt_unit_mmap_buf_get(ctx);
            if (nxt_slow_path(b == NULL)) {
                nxt_unit_alert(ctx, "#%"PRIu32": failed to allocate buf",
                               req_impl->stream);

                nxt_unit_websocket_frame_release(&ws_impl->ws);

                return NXT_UNIT_ERROR;
            }

            b->req = req;
            b->buf.start = recv_msg->start;
            b->buf.free = b->buf.start;
            b->buf.end = b->buf.start + recv_msg->size;

            nxt_unit_mmap_buf_insert(&ws_impl->buf, b);
        }

        ws_impl->ws.header = (void *) b->buf.start;
        ws_impl->ws.payload_len = nxt_websocket_frame_payload_len(
            ws_impl->ws.header);

        hsize = nxt_websocket_frame_header_size(ws_impl->ws.header);

        if (ws_impl->ws.header->mask) {
            ws_impl->ws.mask = (uint8_t *) b->buf.start + hsize - 4;

        } else {
            ws_impl->ws.mask = NULL;
        }

        b->buf.free += hsize;

        ws_impl->ws.content_buf = &b->buf;
        ws_impl->ws.content_length = ws_impl->ws.payload_len;

        nxt_unit_req_debug(req, "websocket_handler: opcode=%d, "
                           "payload_len=%"PRIu64,
                            ws_impl->ws.header->opcode,
                            ws_impl->ws.payload_len);

        cb->websocket_handler(&ws_impl->ws);
    }

    if (recv_msg->last) {
        if (cb->close_handler) {
            nxt_unit_req_debug(req, "close_handler");

            cb->close_handler(req);

        } else {
            nxt_unit_request_done(req, NXT_UNIT_ERROR);
        }
    }

    return NXT_UNIT_OK;
}


static int
nxt_unit_process_shm_ack(nxt_unit_ctx_t *ctx)
{
    nxt_unit_impl_t       *lib;
    nxt_unit_callbacks_t  *cb;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);
    cb = &lib->callbacks;

    if (cb->shm_ack_handler != NULL) {
        cb->shm_ack_handler(ctx);
    }

    return NXT_UNIT_OK;
}


static nxt_unit_request_info_impl_t *
nxt_unit_request_info_get(nxt_unit_ctx_t *ctx)
{
    nxt_unit_impl_t               *lib;
    nxt_queue_link_t              *lnk;
    nxt_unit_ctx_impl_t           *ctx_impl;
    nxt_unit_request_info_impl_t  *req_impl;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    pthread_mutex_lock(&ctx_impl->mutex);

    if (nxt_queue_is_empty(&ctx_impl->free_req)) {
        pthread_mutex_unlock(&ctx_impl->mutex);

        req_impl = nxt_unit_malloc(ctx, sizeof(nxt_unit_request_info_impl_t)
                                        + lib->request_data_size);
        if (nxt_slow_path(req_impl == NULL)) {
            return NULL;
        }

        req_impl->req.unit = ctx->unit;
        req_impl->req.ctx = ctx;

        pthread_mutex_lock(&ctx_impl->mutex);

    } else {
        lnk = nxt_queue_first(&ctx_impl->free_req);
        nxt_queue_remove(lnk);

        req_impl = nxt_container_of(lnk, nxt_unit_request_info_impl_t, link);
    }

    nxt_queue_insert_tail(&ctx_impl->active_req, &req_impl->link);

    pthread_mutex_unlock(&ctx_impl->mutex);

    req_impl->req.data = lib->request_data_size ? req_impl->extra_data : NULL;

    return req_impl;
}


static void
nxt_unit_request_info_release(nxt_unit_request_info_t *req)
{
    nxt_unit_ctx_t                *ctx;
    nxt_unit_ctx_impl_t           *ctx_impl;
    nxt_unit_request_info_impl_t  *req_impl;

    ctx = req->ctx;
    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);
    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    req->response = NULL;
    req->response_buf = NULL;

    if (req_impl->in_hash) {
        nxt_unit_request_hash_find(req->ctx, req_impl->stream, 1);
    }

    while (req_impl->outgoing_buf != NULL) {
        nxt_unit_mmap_buf_free(req_impl->outgoing_buf);
    }

    while (req_impl->incoming_buf != NULL) {
        nxt_unit_mmap_buf_free(req_impl->incoming_buf);
    }

    if (req->content_fd != -1) {
        nxt_unit_close(req->content_fd);

        req->content_fd = -1;
    }

    if (req->response_port != NULL) {
        nxt_unit_port_release(req->response_port);

        req->response_port = NULL;
    }

    req_impl->state = NXT_UNIT_RS_RELEASED;

    pthread_mutex_lock(&ctx_impl->mutex);

    nxt_queue_remove(&req_impl->link);

    nxt_queue_insert_tail(&ctx_impl->free_req, &req_impl->link);

    pthread_mutex_unlock(&ctx_impl->mutex);

    if (nxt_slow_path(!nxt_unit_chk_ready(ctx))) {
        nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL);
    }
}


static void
nxt_unit_request_info_free(nxt_unit_request_info_impl_t *req_impl)
{
    nxt_unit_ctx_impl_t  *ctx_impl;

    ctx_impl = nxt_container_of(req_impl->req.ctx, nxt_unit_ctx_impl_t, ctx);

    nxt_queue_remove(&req_impl->link);

    if (req_impl != &ctx_impl->req) {
        nxt_unit_free(&ctx_impl->ctx, req_impl);
    }
}


static nxt_unit_websocket_frame_impl_t *
nxt_unit_websocket_frame_get(nxt_unit_ctx_t *ctx)
{
    nxt_queue_link_t                 *lnk;
    nxt_unit_ctx_impl_t              *ctx_impl;
    nxt_unit_websocket_frame_impl_t  *ws_impl;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    pthread_mutex_lock(&ctx_impl->mutex);

    if (nxt_queue_is_empty(&ctx_impl->free_ws)) {
        pthread_mutex_unlock(&ctx_impl->mutex);

        ws_impl = nxt_unit_malloc(ctx, sizeof(nxt_unit_websocket_frame_impl_t));
        if (nxt_slow_path(ws_impl == NULL)) {
            return NULL;
        }

    } else {
        lnk = nxt_queue_first(&ctx_impl->free_ws);
        nxt_queue_remove(lnk);

        pthread_mutex_unlock(&ctx_impl->mutex);

        ws_impl = nxt_container_of(lnk, nxt_unit_websocket_frame_impl_t, link);
    }

    ws_impl->ctx_impl = ctx_impl;

    return ws_impl;
}


static void
nxt_unit_websocket_frame_release(nxt_unit_websocket_frame_t *ws)
{
    nxt_unit_websocket_frame_impl_t  *ws_impl;

    ws_impl = nxt_container_of(ws, nxt_unit_websocket_frame_impl_t, ws);

    while (ws_impl->buf != NULL) {
        nxt_unit_mmap_buf_free(ws_impl->buf);
    }

    ws->req = NULL;

    pthread_mutex_lock(&ws_impl->ctx_impl->mutex);

    nxt_queue_insert_tail(&ws_impl->ctx_impl->free_ws, &ws_impl->link);

    pthread_mutex_unlock(&ws_impl->ctx_impl->mutex);
}


static void
nxt_unit_websocket_frame_free(nxt_unit_ctx_t *ctx,
    nxt_unit_websocket_frame_impl_t *ws_impl)
{
    nxt_queue_remove(&ws_impl->link);

    nxt_unit_free(ctx, ws_impl);
}


uint16_t
nxt_unit_field_hash(const char *name, size_t name_length)
{
    u_char      ch;
    uint32_t    hash;
    const char  *p, *end;

    hash = 159406; /* Magic value copied from nxt_http_parse.c */
    end = name + name_length;

    for (p = name; p < end; p++) {
        ch = *p;
        hash = (hash << 4) + hash + nxt_lowcase(ch);
    }

    hash = (hash >> 16) ^ hash;

    return hash;
}


void
nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req)
{
    char                *name;
    uint32_t            i, j;
    nxt_unit_field_t    *fields, f;
    nxt_unit_request_t  *r;

    static const nxt_str_t  content_length = nxt_string("content-length");
    static const nxt_str_t  content_type = nxt_string("content-type");
    static const nxt_str_t  cookie = nxt_string("cookie");

    nxt_unit_req_debug(req, "group_dup_fields");

    r = req->request;
    fields = r->fields;

    for (i = 0; i < r->fields_count; i++) {
        name = nxt_unit_sptr_get(&fields[i].name);

        switch (fields[i].hash) {
        case NXT_UNIT_HASH_CONTENT_LENGTH:
            if (fields[i].name_length == content_length.length
                && nxt_unit_memcasecmp(name, content_length.start,
                                       content_length.length) == 0)
            {
                r->content_length_field = i;
            }

            break;

        case NXT_UNIT_HASH_CONTENT_TYPE:
            if (fields[i].name_length == content_type.length
                && nxt_unit_memcasecmp(name, content_type.start,
                                       content_type.length) == 0)
            {
                r->content_type_field = i;
            }

            break;

        case NXT_UNIT_HASH_COOKIE:
            if (fields[i].name_length == cookie.length
                && nxt_unit_memcasecmp(name, cookie.start,
                                       cookie.length) == 0)
            {
                r->cookie_field = i;
            }

            break;
        }

        for (j = i + 1; j < r->fields_count; j++) {
            if (fields[i].hash != fields[j].hash
                || fields[i].name_length != fields[j].name_length
                || nxt_unit_memcasecmp(name,
                                       nxt_unit_sptr_get(&fields[j].name),
                                       fields[j].name_length) != 0)
            {
                continue;
            }

            f = fields[j];
            f.value.offset += (j - (i + 1)) * sizeof(f);

            while (j > i + 1) {
                fields[j] = fields[j - 1];
                fields[j].name.offset -= sizeof(f);
                fields[j].value.offset -= sizeof(f);
                j--;
            }

            fields[j] = f;

            /* Assign the same name pointer for further grouping simplicity. */
            nxt_unit_sptr_set(&fields[j].name, name);

            i++;
        }
    }
}


int
nxt_unit_response_init(nxt_unit_request_info_t *req,
    uint16_t status, uint32_t max_fields_count, uint32_t max_fields_size)
{
    uint32_t                      buf_size;
    nxt_unit_buf_t                *buf;
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    if (nxt_slow_path(req_impl->state >= NXT_UNIT_RS_RESPONSE_SENT)) {
        nxt_unit_req_warn(req, "init: response already sent");

        return NXT_UNIT_ERROR;
    }

    nxt_unit_req_debug(req, "init: %d, max fields %d/%d", (int) status,
                       (int) max_fields_count, (int) max_fields_size);

    if (nxt_slow_path(req_impl->state >= NXT_UNIT_RS_RESPONSE_INIT)) {
        nxt_unit_req_debug(req, "duplicate response init");
    }

    /*
     * Each field name and value 0-terminated by libunit,
     * this is the reason of '+ 2' below.
     */
    buf_size = sizeof(nxt_unit_response_t)
               + max_fields_count * (sizeof(nxt_unit_field_t) + 2)
               + max_fields_size;

    if (nxt_slow_path(req->response_buf != NULL)) {
        buf = req->response_buf;

        if (nxt_fast_path(buf_size <= (uint32_t) (buf->end - buf->start))) {
            goto init_response;
        }

        nxt_unit_buf_free(buf);

        req->response_buf = NULL;
        req->response = NULL;
        req->response_max_fields = 0;

        req_impl->state = NXT_UNIT_RS_START;
    }

    buf = nxt_unit_response_buf_alloc(req, buf_size);
    if (nxt_slow_path(buf == NULL)) {
        return NXT_UNIT_ERROR;
    }

init_response:

    memset(buf->start, 0, sizeof(nxt_unit_response_t));

    req->response_buf = buf;

    req->response = (nxt_unit_response_t *) buf->start;
    req->response->status = status;

    buf->free = buf->start + sizeof(nxt_unit_response_t)
                + max_fields_count * sizeof(nxt_unit_field_t);

    req->response_max_fields = max_fields_count;
    req_impl->state = NXT_UNIT_RS_RESPONSE_INIT;

    return NXT_UNIT_OK;
}


int
nxt_unit_response_realloc(nxt_unit_request_info_t *req,
    uint32_t max_fields_count, uint32_t max_fields_size)
{
    char                          *p;
    uint32_t                      i, buf_size;
    nxt_unit_buf_t                *buf;
    nxt_unit_field_t              *f, *src;
    nxt_unit_response_t           *resp;
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) {
        nxt_unit_req_warn(req, "realloc: response not init");

        return NXT_UNIT_ERROR;
    }

    if (nxt_slow_path(req_impl->state >= NXT_UNIT_RS_RESPONSE_SENT)) {
        nxt_unit_req_warn(req, "realloc: response already sent");

        return NXT_UNIT_ERROR;
    }

    if (nxt_slow_path(max_fields_count < req->response->fields_count)) {
        nxt_unit_req_warn(req, "realloc: new max_fields_count is too small");

        return NXT_UNIT_ERROR;
    }

    /*
     * Each field name and value 0-terminated by libunit,
     * this is the reason of '+ 2' below.
     */
    buf_size = sizeof(nxt_unit_response_t)
               + max_fields_count * (sizeof(nxt_unit_field_t) + 2)
               + max_fields_size;

    nxt_unit_req_debug(req, "realloc %"PRIu32"", buf_size);

    buf = nxt_unit_response_buf_alloc(req, buf_size);
    if (nxt_slow_path(buf == NULL)) {
        nxt_unit_req_warn(req, "realloc: new buf allocation failed");
        return NXT_UNIT_ERROR;
    }

    resp = (nxt_unit_response_t *) buf->start;

    memset(resp, 0, sizeof(nxt_unit_response_t));

    resp->status = req->response->status;
    resp->content_length = req->response->content_length;

    p = buf->start + sizeof(nxt_unit_response_t)
        + max_fields_count * sizeof(nxt_unit_field_t);
    f = resp->fields;

    for (i = 0; i < req->response->fields_count; i++) {
        src = req->response->fields + i;

        if (nxt_slow_path(src->skip != 0)) {
            continue;
        }

        if (nxt_slow_path(src->name_length + src->value_length + 2
                          > (uint32_t) (buf->end - p)))
        {
            nxt_unit_req_warn(req, "realloc: not enough space for field"
                  " #%"PRIu32" (%p), (%"PRIu32" + %"PRIu32") required",
                  i, src, src->name_length, src->value_length);

            goto fail;
        }

        nxt_unit_sptr_set(&f->name, p);
        p = nxt_cpymem(p, nxt_unit_sptr_get(&src->name), src->name_length);
        *p++ = '\0';

        nxt_unit_sptr_set(&f->value, p);
        p = nxt_cpymem(p, nxt_unit_sptr_get(&src->value), src->value_length);
        *p++ = '\0';

        f->hash = src->hash;
        f->skip = 0;
        f->name_length = src->name_length;
        f->value_length = src->value_length;

        resp->fields_count++;
        f++;
    }

    if (req->response->piggyback_content_length > 0) {
        if (nxt_slow_path(req->response->piggyback_content_length
                          > (uint32_t) (buf->end - p)))
        {
            nxt_unit_req_warn(req, "realloc: not enought space for content"
                  " #%"PRIu32", %"PRIu32" required",
                  i, req->response->piggyback_content_length);

            goto fail;
        }

        resp->piggyback_content_length =
                                       req->response->piggyback_content_length;

        nxt_unit_sptr_set(&resp->piggyback_content, p);
        p = nxt_cpymem(p, nxt_unit_sptr_get(&req->response->piggyback_content),
                       req->response->piggyback_content_length);
    }

    buf->free = p;

    nxt_unit_buf_free(req->response_buf);

    req->response = resp;
    req->response_buf = buf;
    req->response_max_fields = max_fields_count;

    return NXT_UNIT_OK;

fail:

    nxt_unit_buf_free(buf);

    return NXT_UNIT_ERROR;
}


int
nxt_unit_response_is_init(nxt_unit_request_info_t *req)
{
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    return req_impl->state >= NXT_UNIT_RS_RESPONSE_INIT;
}


int
nxt_unit_response_add_field(nxt_unit_request_info_t *req,
    const char *name, uint8_t name_length,
    const char *value, uint32_t value_length)
{
    nxt_unit_buf_t                *buf;
    nxt_unit_field_t              *f;
    nxt_unit_response_t           *resp;
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    if (nxt_slow_path(req_impl->state != NXT_UNIT_RS_RESPONSE_INIT)) {
        nxt_unit_req_warn(req, "add_field: response not initialized or "
                          "already sent");

        return NXT_UNIT_ERROR;
    }

    resp = req->response;

    if (nxt_slow_path(resp->fields_count >= req->response_max_fields)) {
        nxt_unit_req_warn(req, "add_field: too many response fields (%d)",
                          (int) resp->fields_count);

        return NXT_UNIT_ERROR;
    }

    buf = req->response_buf;

    if (nxt_slow_path(name_length + value_length + 2
                      > (uint32_t) (buf->end - buf->free)))
    {
        nxt_unit_req_warn(req, "add_field: response buffer overflow");

        return NXT_UNIT_ERROR;
    }

    nxt_unit_req_debug(req, "add_field #%"PRIu32": %.*s: %.*s",
                       resp->fields_count,
                       (int) name_length, name,
                       (int) value_length, value);

    f = resp->fields + resp->fields_count;

    nxt_unit_sptr_set(&f->name, buf->free);
    buf->free = nxt_cpymem(buf->free, name, name_length);
    *buf->free++ = '\0';

    nxt_unit_sptr_set(&f->value, buf->free);
    buf->free = nxt_cpymem(buf->free, value, value_length);
    *buf->free++ = '\0';

    f->hash = nxt_unit_field_hash(name, name_length);
    f->skip = 0;
    f->name_length = name_length;
    f->value_length = value_length;

    resp->fields_count++;

    return NXT_UNIT_OK;
}


int
nxt_unit_response_add_content(nxt_unit_request_info_t *req,
    const void* src, uint32_t size)
{
    nxt_unit_buf_t                *buf;
    nxt_unit_response_t           *resp;
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) {
        nxt_unit_req_warn(req, "add_content: response not initialized yet");

        return NXT_UNIT_ERROR;
    }

    if (nxt_slow_path(req_impl->state >= NXT_UNIT_RS_RESPONSE_SENT)) {
        nxt_unit_req_warn(req, "add_content: response already sent");

        return NXT_UNIT_ERROR;
    }

    buf = req->response_buf;

    if (nxt_slow_path(size > (uint32_t) (buf->end - buf->free))) {
        nxt_unit_req_warn(req, "add_content: buffer overflow");

        return NXT_UNIT_ERROR;
    }

    resp = req->response;

    if (resp->piggyback_content_length == 0) {
        nxt_unit_sptr_set(&resp->piggyback_content, buf->free);
        req_impl->state = NXT_UNIT_RS_RESPONSE_HAS_CONTENT;
    }

    resp->piggyback_content_length += size;

    buf->free = nxt_cpymem(buf->free, src, size);

    return NXT_UNIT_OK;
}


int
nxt_unit_response_send(nxt_unit_request_info_t *req)
{
    int                           rc;
    nxt_unit_mmap_buf_t           *mmap_buf;
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) {
        nxt_unit_req_warn(req, "send: response is not initialized yet");

        return NXT_UNIT_ERROR;
    }

    if (nxt_slow_path(req_impl->state >= NXT_UNIT_RS_RESPONSE_SENT)) {
        nxt_unit_req_warn(req, "send: response already sent");

        return NXT_UNIT_ERROR;
    }

    if (req->request->websocket_handshake && req->response->status == 101) {
        nxt_unit_response_upgrade(req);
    }

    nxt_unit_req_debug(req, "send: %"PRIu32" fields, %d bytes",
                       req->response->fields_count,
                       (int) (req->response_buf->free
                              - req->response_buf->start));

    mmap_buf = nxt_container_of(req->response_buf, nxt_unit_mmap_buf_t, buf);

    rc = nxt_unit_mmap_buf_send(req, mmap_buf, 0);
    if (nxt_fast_path(rc == NXT_UNIT_OK)) {
        req->response = NULL;
        req->response_buf = NULL;
        req_impl->state = NXT_UNIT_RS_RESPONSE_SENT;

        nxt_unit_mmap_buf_free(mmap_buf);
    }

    return rc;
}


int
nxt_unit_response_is_sent(nxt_unit_request_info_t *req)
{
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    return req_impl->state >= NXT_UNIT_RS_RESPONSE_SENT;
}


nxt_unit_buf_t *
nxt_unit_response_buf_alloc(nxt_unit_request_info_t *req, uint32_t size)
{
    int                           rc;
    nxt_unit_mmap_buf_t           *mmap_buf;
    nxt_unit_request_info_impl_t  *req_impl;

    if (nxt_slow_path(size > PORT_MMAP_DATA_SIZE)) {
        nxt_unit_req_warn(req, "response_buf_alloc: "
                          "requested buffer (%"PRIu32") too big", size);

        return NULL;
    }

    nxt_unit_req_debug(req, "response_buf_alloc: %"PRIu32, size);

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    mmap_buf = nxt_unit_mmap_buf_get(req->ctx);
    if (nxt_slow_path(mmap_buf == NULL)) {
        nxt_unit_req_alert(req, "response_buf_alloc: failed to allocate buf");

        return NULL;
    }

    mmap_buf->req = req;

    nxt_unit_mmap_buf_insert_tail(&req_impl->outgoing_buf, mmap_buf);

    rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port,
                                   size, size, mmap_buf,
                                   NULL);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        nxt_unit_mmap_buf_release(mmap_buf);

        nxt_unit_req_alert(req, "response_buf_alloc: failed to get out buf");

        return NULL;
    }

    return &mmap_buf->buf;
}


static nxt_unit_mmap_buf_t *
nxt_unit_mmap_buf_get(nxt_unit_ctx_t *ctx)
{
    nxt_unit_mmap_buf_t  *mmap_buf;
    nxt_unit_ctx_impl_t  *ctx_impl;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    pthread_mutex_lock(&ctx_impl->mutex);

    if (ctx_impl->free_buf == NULL) {
        pthread_mutex_unlock(&ctx_impl->mutex);

        mmap_buf = nxt_unit_malloc(ctx, sizeof(nxt_unit_mmap_buf_t));
        if (nxt_slow_path(mmap_buf == NULL)) {
            return NULL;
        }

    } else {
        mmap_buf = ctx_impl->free_buf;

        nxt_unit_mmap_buf_unlink(mmap_buf);

        pthread_mutex_unlock(&ctx_impl->mutex);
    }

    mmap_buf->ctx_impl = ctx_impl;

    mmap_buf->hdr = NULL;
    mmap_buf->free_ptr = NULL;

    return mmap_buf;
}


static void
nxt_unit_mmap_buf_release(nxt_unit_mmap_buf_t *mmap_buf)
{
    nxt_unit_mmap_buf_unlink(mmap_buf);

    pthread_mutex_lock(&mmap_buf->ctx_impl->mutex);

    nxt_unit_mmap_buf_insert(&mmap_buf->ctx_impl->free_buf, mmap_buf);

    pthread_mutex_unlock(&mmap_buf->ctx_impl->mutex);
}


int
nxt_unit_request_is_websocket_handshake(nxt_unit_request_info_t *req)
{
    return req->request->websocket_handshake;
}


int
nxt_unit_response_upgrade(nxt_unit_request_info_t *req)
{
    int                           rc;
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    if (nxt_slow_path(req_impl->websocket != 0)) {
        nxt_unit_req_debug(req, "upgrade: already upgraded");

        return NXT_UNIT_OK;
    }

    if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) {
        nxt_unit_req_warn(req, "upgrade: response is not initialized yet");

        return NXT_UNIT_ERROR;
    }

    if (nxt_slow_path(req_impl->state >= NXT_UNIT_RS_RESPONSE_SENT)) {
        nxt_unit_req_warn(req, "upgrade: response already sent");

        return NXT_UNIT_ERROR;
    }

    rc = nxt_unit_request_hash_add(req->ctx, req);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        nxt_unit_req_warn(req, "upgrade: failed to add request to hash");

        return NXT_UNIT_ERROR;
    }

    req_impl->websocket = 1;

    req->response->status = 101;

    return NXT_UNIT_OK;
}


int
nxt_unit_response_is_websocket(nxt_unit_request_info_t *req)
{
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    return req_impl->websocket;
}


nxt_unit_request_info_t *
nxt_unit_get_request_info_from_data(void *data)
{
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(data, nxt_unit_request_info_impl_t, extra_data);

    return &req_impl->req;
}


int
nxt_unit_buf_send(nxt_unit_buf_t *buf)
{
    int                           rc;
    nxt_unit_mmap_buf_t           *mmap_buf;
    nxt_unit_request_info_t       *req;
    nxt_unit_request_info_impl_t  *req_impl;

    mmap_buf = nxt_container_of(buf, nxt_unit_mmap_buf_t, buf);

    req = mmap_buf->req;
    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    nxt_unit_req_debug(req, "buf_send: %d bytes",
                       (int) (buf->free - buf->start));

    if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) {
        nxt_unit_req_warn(req, "buf_send: response not initialized yet");

        return NXT_UNIT_ERROR;
    }

    if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_SENT)) {
        nxt_unit_req_warn(req, "buf_send: headers not sent yet");

        return NXT_UNIT_ERROR;
    }

    if (nxt_fast_path(buf->free > buf->start)) {
        rc = nxt_unit_mmap_buf_send(req, mmap_buf, 0);
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            return rc;
        }
    }

    nxt_unit_mmap_buf_free(mmap_buf);

    return NXT_UNIT_OK;
}


static void
nxt_unit_buf_send_done(nxt_unit_buf_t *buf)
{
    int                      rc;
    nxt_unit_mmap_buf_t      *mmap_buf;
    nxt_unit_request_info_t  *req;

    mmap_buf = nxt_container_of(buf, nxt_unit_mmap_buf_t, buf);

    req = mmap_buf->req;

    rc = nxt_unit_mmap_buf_send(req, mmap_buf, 1);
    if (nxt_slow_path(rc == NXT_UNIT_OK)) {
        nxt_unit_mmap_buf_free(mmap_buf);

        nxt_unit_request_info_release(req);

    } else {
        nxt_unit_request_done(req, rc);
    }
}


static int
nxt_unit_mmap_buf_send(nxt_unit_request_info_t *req,
    nxt_unit_mmap_buf_t *mmap_buf, int last)
{
    struct {
        nxt_port_msg_t       msg;
        nxt_port_mmap_msg_t  mmap_msg;
    } m;

    int                           rc;
    u_char                        *last_used, *first_free;
    ssize_t                       res;
    nxt_chunk_id_t                first_free_chunk;
    nxt_unit_buf_t                *buf;
    nxt_unit_impl_t               *lib;
    nxt_port_mmap_header_t        *hdr;
    nxt_unit_request_info_impl_t  *req_impl;

    lib = nxt_container_of(req->ctx->unit, nxt_unit_impl_t, unit);
    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    buf = &mmap_buf->buf;
    hdr = mmap_buf->hdr;

    m.mmap_msg.size = buf->free - buf->start;

    m.msg.stream = req_impl->stream;
    m.msg.pid = lib->pid;
    m.msg.reply_port = 0;
    m.msg.type = _NXT_PORT_MSG_DATA;
    m.msg.last = last != 0;
    m.msg.mmap = hdr != NULL && m.mmap_msg.size > 0;
    m.msg.nf = 0;
    m.msg.mf = 0;

    rc = NXT_UNIT_ERROR;

    if (m.msg.mmap) {
        m.mmap_msg.mmap_id = hdr->id;
        m.mmap_msg.chunk_id = nxt_port_mmap_chunk_id(hdr,
                                                     (u_char *) buf->start);

        nxt_unit_debug(req->ctx, "#%"PRIu32": send mmap: (%d,%d,%d)",
                       req_impl->stream,
                       (int) m.mmap_msg.mmap_id,
                       (int) m.mmap_msg.chunk_id,
                       (int) m.mmap_msg.size);

        res = nxt_unit_port_send(req->ctx, req->response_port, &m, sizeof(m),
                                 NULL);
        if (nxt_slow_path(res != sizeof(m))) {
            goto free_buf;
        }

        last_used = (u_char *) buf->free - 1;
        first_free_chunk = nxt_port_mmap_chunk_id(hdr, last_used) + 1;

        if (buf->end - buf->free >= PORT_MMAP_CHUNK_SIZE) {
            first_free = nxt_port_mmap_chunk_start(hdr, first_free_chunk);

            buf->start = (char *) first_free;
            buf->free = buf->start;

            if (buf->end < buf->start) {
                buf->end = buf->start;
            }

        } else {
            buf->start = NULL;
            buf->free = NULL;
            buf->end = NULL;

            mmap_buf->hdr = NULL;
        }

        nxt_atomic_fetch_add(&lib->outgoing.allocated_chunks,
                            (int) m.mmap_msg.chunk_id - (int) first_free_chunk);

        nxt_unit_debug(req->ctx, "allocated_chunks %d",
                       (int) lib->outgoing.allocated_chunks);

    } else {
        if (nxt_slow_path(mmap_buf->plain_ptr == NULL
                          || mmap_buf->plain_ptr > buf->start - sizeof(m.msg)))
        {
            nxt_unit_alert(req->ctx,
                           "#%"PRIu32": failed to send plain memory buffer"
                           ": no space reserved for message header",
                           req_impl->stream);

            goto free_buf;
        }

        memcpy(buf->start - sizeof(m.msg), &m.msg, sizeof(m.msg));

        nxt_unit_debug(req->ctx, "#%"PRIu32": send plain: %d",
                       req_impl->stream,
                       (int) (sizeof(m.msg) + m.mmap_msg.size));

        res = nxt_unit_port_send(req->ctx, req->response_port,
                                 buf->start - sizeof(m.msg),
                                 m.mmap_msg.size + sizeof(m.msg), NULL);

        if (nxt_slow_path(res != (ssize_t) (m.mmap_msg.size + sizeof(m.msg)))) {
            goto free_buf;
        }
    }

    rc = NXT_UNIT_OK;

free_buf:

    nxt_unit_free_outgoing_buf(mmap_buf);

    return rc;
}


void
nxt_unit_buf_free(nxt_unit_buf_t *buf)
{
    nxt_unit_mmap_buf_free(nxt_container_of(buf, nxt_unit_mmap_buf_t, buf));
}


static void
nxt_unit_mmap_buf_free(nxt_unit_mmap_buf_t *mmap_buf)
{
    nxt_unit_free_outgoing_buf(mmap_buf);

    nxt_unit_mmap_buf_release(mmap_buf);
}


static void
nxt_unit_free_outgoing_buf(nxt_unit_mmap_buf_t *mmap_buf)
{
    if (mmap_buf->hdr != NULL) {
        nxt_unit_mmap_release(&mmap_buf->ctx_impl->ctx,
                              mmap_buf->hdr, mmap_buf->buf.start,
                              mmap_buf->buf.end - mmap_buf->buf.start);

        mmap_buf->hdr = NULL;

        return;
    }

    if (mmap_buf->free_ptr != NULL) {
        nxt_unit_free(&mmap_buf->ctx_impl->ctx, mmap_buf->free_ptr);

        mmap_buf->free_ptr = NULL;
    }
}


static nxt_unit_read_buf_t *
nxt_unit_read_buf_get(nxt_unit_ctx_t *ctx)
{
    nxt_unit_ctx_impl_t  *ctx_impl;
    nxt_unit_read_buf_t  *rbuf;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    pthread_mutex_lock(&ctx_impl->mutex);

    rbuf = nxt_unit_read_buf_get_impl(ctx_impl);

    pthread_mutex_unlock(&ctx_impl->mutex);

    rbuf->oob.size = 0;

    return rbuf;
}


static nxt_unit_read_buf_t *
nxt_unit_read_buf_get_impl(nxt_unit_ctx_impl_t *ctx_impl)
{
    nxt_queue_link_t     *link;
    nxt_unit_read_buf_t  *rbuf;

    if (!nxt_queue_is_empty(&ctx_impl->free_rbuf)) {
        link = nxt_queue_first(&ctx_impl->free_rbuf);
        nxt_queue_remove(link);

        rbuf = nxt_container_of(link, nxt_unit_read_buf_t, link);

        return rbuf;
    }

    rbuf = nxt_unit_malloc(&ctx_impl->ctx, sizeof(nxt_unit_read_buf_t));

    if (nxt_fast_path(rbuf != NULL)) {
        rbuf->ctx_impl = ctx_impl;
    }

    return rbuf;
}


static void
nxt_unit_read_buf_release(nxt_unit_ctx_t *ctx,
    nxt_unit_read_buf_t *rbuf)
{
    nxt_unit_ctx_impl_t  *ctx_impl;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    pthread_mutex_lock(&ctx_impl->mutex);

    nxt_queue_insert_head(&ctx_impl->free_rbuf, &rbuf->link);

    pthread_mutex_unlock(&ctx_impl->mutex);
}


nxt_unit_buf_t *
nxt_unit_buf_next(nxt_unit_buf_t *buf)
{
    nxt_unit_mmap_buf_t  *mmap_buf;

    mmap_buf = nxt_container_of(buf, nxt_unit_mmap_buf_t, buf);

    if (mmap_buf->next == NULL) {
        return NULL;
    }

    return &mmap_buf->next->buf;
}


uint32_t
nxt_unit_buf_max(void)
{
    return PORT_MMAP_DATA_SIZE;
}


uint32_t
nxt_unit_buf_min(void)
{
    return PORT_MMAP_CHUNK_SIZE;
}


int
nxt_unit_response_write(nxt_unit_request_info_t *req, const void *start,
    size_t size)
{
    ssize_t  res;

    res = nxt_unit_response_write_nb(req, start, size, size);

    return res < 0 ? -res : NXT_UNIT_OK;
}


ssize_t
nxt_unit_response_write_nb(nxt_unit_request_info_t *req, const void *start,
    size_t size, size_t min_size)
{
    int                           rc;
    ssize_t                       sent;
    uint32_t                      part_size, min_part_size, buf_size;
    const char                    *part_start;
    nxt_unit_mmap_buf_t           mmap_buf;
    nxt_unit_request_info_impl_t  *req_impl;
    char                          local_buf[NXT_UNIT_LOCAL_BUF_SIZE];

    nxt_unit_req_debug(req, "write: %d", (int) size);

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    part_start = start;
    sent = 0;

    if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) {
        nxt_unit_req_alert(req, "write: response not initialized yet");

        return -NXT_UNIT_ERROR;
    }

    /* Check if response is not send yet. */
    if (nxt_slow_path(req->response_buf != NULL)) {
        part_size = req->response_buf->end - req->response_buf->free;
        part_size = nxt_min(size, part_size);

        rc = nxt_unit_response_add_content(req, part_start, part_size);
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            return -rc;
        }

        rc = nxt_unit_response_send(req);
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            return -rc;
        }

        size -= part_size;
        part_start += part_size;
        sent += part_size;

        min_size -= nxt_min(min_size, part_size);
    }

    while (size > 0) {
        part_size = nxt_min(size, PORT_MMAP_DATA_SIZE);
        min_part_size = nxt_min(min_size, part_size);
        min_part_size = nxt_min(min_part_size, PORT_MMAP_CHUNK_SIZE);

        rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port, part_size,
                                       min_part_size, &mmap_buf, local_buf);
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            return -rc;
        }

        buf_size = mmap_buf.buf.end - mmap_buf.buf.free;
        if (nxt_slow_path(buf_size == 0)) {
            return sent;
        }
        part_size = nxt_min(buf_size, part_size);

        mmap_buf.buf.free = nxt_cpymem(mmap_buf.buf.free,
                                       part_start, part_size);

        rc = nxt_unit_mmap_buf_send(req, &mmap_buf, 0);
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            return -rc;
        }

        size -= part_size;
        part_start += part_size;
        sent += part_size;

        min_size -= nxt_min(min_size, part_size);
    }

    return sent;
}


int
nxt_unit_response_write_cb(nxt_unit_request_info_t *req,
    nxt_unit_read_info_t *read_info)
{
    int                           rc;
    ssize_t                       n;
    uint32_t                      buf_size;
    nxt_unit_buf_t                *buf;
    nxt_unit_mmap_buf_t           mmap_buf;
    nxt_unit_request_info_impl_t  *req_impl;
    char                          local_buf[NXT_UNIT_LOCAL_BUF_SIZE];

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) {
        nxt_unit_req_alert(req, "write: response not initialized yet");

        return NXT_UNIT_ERROR;
    }

    /* Check if response is not send yet. */
    if (nxt_slow_path(req->response_buf != NULL)) {

        /* Enable content in headers buf. */
        rc = nxt_unit_response_add_content(req, "", 0);
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            nxt_unit_req_error(req, "Failed to add piggyback content");

            return rc;
        }

        buf = req->response_buf;

        while (buf->end - buf->free > 0) {
            n = read_info->read(read_info, buf->free, buf->end - buf->free);
            if (nxt_slow_path(n < 0)) {
                nxt_unit_req_error(req, "Read error");

                return NXT_UNIT_ERROR;
            }

            /* Manually increase sizes. */
            buf->free += n;
            req->response->piggyback_content_length += n;

            if (read_info->eof) {
                break;
            }
        }

        rc = nxt_unit_response_send(req);
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            nxt_unit_req_error(req, "Failed to send headers with content");

            return rc;
        }

        if (read_info->eof) {
            return NXT_UNIT_OK;
        }
    }

    while (!read_info->eof) {
        nxt_unit_req_debug(req, "write_cb, alloc %"PRIu32"",
                           read_info->buf_size);

        buf_size = nxt_min(read_info->buf_size, PORT_MMAP_DATA_SIZE);

        rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port,
                                       buf_size, buf_size,
                                       &mmap_buf, local_buf);
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            return rc;
        }

        buf = &mmap_buf.buf;

        while (!read_info->eof && buf->end > buf->free) {
            n = read_info->read(read_info, buf->free, buf->end - buf->free);
            if (nxt_slow_path(n < 0)) {
                nxt_unit_req_error(req, "Read error");

                nxt_unit_free_outgoing_buf(&mmap_buf);

                return NXT_UNIT_ERROR;
            }

            buf->free += n;
        }

        rc = nxt_unit_mmap_buf_send(req, &mmap_buf, 0);
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            nxt_unit_req_error(req, "Failed to send content");

            return rc;
        }
    }

    return NXT_UNIT_OK;
}


ssize_t
nxt_unit_request_read(nxt_unit_request_info_t *req, void *dst, size_t size)
{
    ssize_t  buf_res, res;

    buf_res = nxt_unit_buf_read(&req->content_buf, &req->content_length,
                                dst, size);

    if (buf_res < (ssize_t) size && req->content_fd != -1) {
        res = read(req->content_fd, dst, size);
        if (nxt_slow_path(res < 0)) {
            nxt_unit_req_alert(req, "failed to read content: %s (%d)",
                               strerror(errno), errno);

            return res;
        }

        if (res < (ssize_t) size) {
            nxt_unit_close(req->content_fd);

            req->content_fd = -1;
        }

        req->content_length -= res;

        dst = nxt_pointer_to(dst, res);

    } else {
        res = 0;
    }

    return buf_res + res;
}


ssize_t
nxt_unit_request_readline_size(nxt_unit_request_info_t *req, size_t max_size)
{
    char                 *p;
    size_t               l_size, b_size;
    nxt_unit_buf_t       *b;
    nxt_unit_mmap_buf_t  *mmap_buf, *preread_buf;

    if (req->content_length == 0) {
        return 0;
    }

    l_size = 0;

    b = req->content_buf;

    while (b != NULL) {
        b_size = b->end - b->free;
        p = memchr(b->free, '\n', b_size);

        if (p != NULL) {
            p++;
            l_size += p - b->free;
            break;
        }

        l_size += b_size;

        if (max_size <= l_size) {
            break;
        }

        mmap_buf = nxt_container_of(b, nxt_unit_mmap_buf_t, buf);
        if (mmap_buf->next == NULL
            && req->content_fd != -1
            && l_size < req->content_length)
        {
            preread_buf = nxt_unit_request_preread(req, 16384);
            if (nxt_slow_path(preread_buf == NULL)) {
                return -1;
            }

            nxt_unit_mmap_buf_insert(&mmap_buf->next, preread_buf);
        }

        b = nxt_unit_buf_next(b);
    }

    return nxt_min(max_size, l_size);
}


static nxt_unit_mmap_buf_t *
nxt_unit_request_preread(nxt_unit_request_info_t *req, size_t size)
{
    ssize_t              res;
    nxt_unit_mmap_buf_t  *mmap_buf;

    if (req->content_fd == -1) {
        nxt_unit_req_alert(req, "preread: content_fd == -1");
        return NULL;
    }

    mmap_buf = nxt_unit_mmap_buf_get(req->ctx);
    if (nxt_slow_path(mmap_buf == NULL)) {
        nxt_unit_req_alert(req, "preread: failed to allocate buf");
        return NULL;
    }

    mmap_buf->free_ptr = nxt_unit_malloc(req->ctx, size);
    if (nxt_slow_path(mmap_buf->free_ptr == NULL)) {
        nxt_unit_req_alert(req, "preread: failed to allocate buf memory");
        nxt_unit_mmap_buf_release(mmap_buf);
        return NULL;
    }

    mmap_buf->plain_ptr = mmap_buf->free_ptr;

    mmap_buf->hdr = NULL;
    mmap_buf->buf.start = mmap_buf->free_ptr;
    mmap_buf->buf.free = mmap_buf->buf.start;
    mmap_buf->buf.end = mmap_buf->buf.start + size;

    res = read(req->content_fd, mmap_buf->free_ptr, size);
    if (res < 0) {
        nxt_unit_req_alert(req, "failed to read content: %s (%d)",
                           strerror(errno), errno);

        nxt_unit_mmap_buf_free(mmap_buf);

        return NULL;
    }

    if (res < (ssize_t) size) {
        nxt_unit_close(req->content_fd);

        req->content_fd = -1;
    }

    nxt_unit_req_debug(req, "preread: read %d", (int) res);

    mmap_buf->buf.end = mmap_buf->buf.free + res;

    return mmap_buf;
}


static ssize_t
nxt_unit_buf_read(nxt_unit_buf_t **b, uint64_t *len, void *dst, size_t size)
{
    u_char          *p;
    size_t          rest, copy, read;
    nxt_unit_buf_t  *buf, *last_buf;

    p = dst;
    rest = size;

    buf = *b;
    last_buf = buf;

    while (buf != NULL) {
        last_buf = buf;

        copy = buf->end - buf->free;
        copy = nxt_min(rest, copy);

        p = nxt_cpymem(p, buf->free, copy);

        buf->free += copy;
        rest -= copy;

        if (rest == 0) {
            if (buf->end == buf->free) {
                buf = nxt_unit_buf_next(buf);
            }

            break;
        }

        buf = nxt_unit_buf_next(buf);
    }

    *b = last_buf;

    read = size - rest;

    *len -= read;

    return read;
}


void
nxt_unit_request_done(nxt_unit_request_info_t *req, int rc)
{
    uint32_t                      size;
    nxt_port_msg_t                msg;
    nxt_unit_impl_t               *lib;
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

    nxt_unit_req_debug(req, "done: %d", rc);

    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        goto skip_response_send;
    }

    if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) {

        size = nxt_length("Content-Type") + nxt_length("text/plain");

        rc = nxt_unit_response_init(req, 200, 1, size);
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            goto skip_response_send;
        }

        rc = nxt_unit_response_add_field(req, "Content-Type",
                                   nxt_length("Content-Type"),
                                   "text/plain", nxt_length("text/plain"));
        if (nxt_slow_path(rc != NXT_UNIT_OK)) {
            goto skip_response_send;
        }
    }

    if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_SENT)) {

        req_impl->state = NXT_UNIT_RS_RESPONSE_SENT;

        nxt_unit_buf_send_done(req->response_buf);

        return;
    }

skip_response_send:

    lib = nxt_container_of(req->unit, nxt_unit_impl_t, unit);

    msg.stream = req_impl->stream;
    msg.pid = lib->pid;
    msg.reply_port = 0;
    msg.type = (rc == NXT_UNIT_OK) ? _NXT_PORT_MSG_DATA
                                   : _NXT_PORT_MSG_RPC_ERROR;
    msg.last = 1;
    msg.mmap = 0;
    msg.nf = 0;
    msg.mf = 0;

    (void) nxt_unit_port_send(req->ctx, req->response_port,
                              &msg, sizeof(msg), NULL);

    nxt_unit_request_info_release(req);
}


int
nxt_unit_websocket_send(nxt_unit_request_info_t *req, uint8_t opcode,
    uint8_t last, const void *start, size_t size)
{
    const struct iovec  iov = { (void *) start, size };

    return nxt_unit_websocket_sendv(req, opcode, last, &iov, 1);
}


int
nxt_unit_websocket_sendv(nxt_unit_request_info_t *req, uint8_t opcode,
    uint8_t last, const struct iovec *iov, int iovcnt)
{
    int                     i, rc;
    size_t                  l, copy;
    uint32_t                payload_len, buf_size, alloc_size;
    const uint8_t           *b;
    nxt_unit_buf_t          *buf;
    nxt_unit_mmap_buf_t     mmap_buf;
    nxt_websocket_header_t  *wh;
    char                    local_buf[NXT_UNIT_LOCAL_BUF_SIZE];

    payload_len = 0;

    for (i = 0; i < iovcnt; i++) {
        payload_len += iov[i].iov_len;
    }

    buf_size = 10 + payload_len;
    alloc_size = nxt_min(buf_size, PORT_MMAP_DATA_SIZE);

    rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port,
                                   alloc_size, alloc_size,
                                   &mmap_buf, local_buf);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        return rc;
    }

    buf = &mmap_buf.buf;

    buf->start[0] = 0;
    buf->start[1] = 0;

    buf_size -= buf->end - buf->start;

    wh = (void *) buf->free;

    buf->free = nxt_websocket_frame_init(wh, payload_len);
    wh->fin = last;
    wh->opcode = opcode;

    for (i = 0; i < iovcnt; i++) {
        b = iov[i].iov_base;
        l = iov[i].iov_len;

        while (l > 0) {
            copy = buf->end - buf->free;
            copy = nxt_min(l, copy);

            buf->free = nxt_cpymem(buf->free, b, copy);
            b += copy;
            l -= copy;

            if (l > 0) {
                if (nxt_fast_path(buf->free > buf->start)) {
                    rc = nxt_unit_mmap_buf_send(req, &mmap_buf, 0);

                    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
                        return rc;
                    }
                }

                alloc_size = nxt_min(buf_size, PORT_MMAP_DATA_SIZE);

                rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port,
                                               alloc_size, alloc_size,
                                               &mmap_buf, local_buf);
                if (nxt_slow_path(rc != NXT_UNIT_OK)) {
                    return rc;
                }

                buf_size -= buf->end - buf->start;
            }
        }
    }

    if (buf->free > buf->start) {
        rc = nxt_unit_mmap_buf_send(req, &mmap_buf, 0);
    }

    return rc;
}


ssize_t
nxt_unit_websocket_read(nxt_unit_websocket_frame_t *ws, void *dst,
    size_t size)
{
    ssize_t   res;
    uint8_t   *b;
    uint64_t  i, d;

    res = nxt_unit_buf_read(&ws->content_buf, &ws->content_length,
                            dst, size);

    if (ws->mask == NULL) {
        return res;
    }

    b = dst;
    d = (ws->payload_len - ws->content_length - res) % 4;

    for (i = 0; i < (uint64_t) res; i++) {
        b[i] ^= ws->mask[ (i + d) % 4 ];
    }

    return res;
}


int
nxt_unit_websocket_retain(nxt_unit_websocket_frame_t *ws)
{
    char                             *b;
    size_t                           size, hsize;
    nxt_unit_websocket_frame_impl_t  *ws_impl;

    ws_impl = nxt_container_of(ws, nxt_unit_websocket_frame_impl_t, ws);

    if (ws_impl->buf->free_ptr != NULL || ws_impl->buf->hdr != NULL) {
        return NXT_UNIT_OK;
    }

    size = ws_impl->buf->buf.end - ws_impl->buf->buf.start;

    b = nxt_unit_malloc(ws->req->ctx, size);
    if (nxt_slow_path(b == NULL)) {
        return NXT_UNIT_ERROR;
    }

    memcpy(b, ws_impl->buf->buf.start, size);

    hsize = nxt_websocket_frame_header_size(b);

    ws_impl->buf->buf.start = b;
    ws_impl->buf->buf.free = b + hsize;
    ws_impl->buf->buf.end = b + size;

    ws_impl->buf->free_ptr = b;

    ws_impl->ws.header = (nxt_websocket_header_t *) b;

    if (ws_impl->ws.header->mask) {
        ws_impl->ws.mask = (uint8_t *) b + hsize - 4;

    } else {
        ws_impl->ws.mask = NULL;
    }

    return NXT_UNIT_OK;
}


void
nxt_unit_websocket_done(nxt_unit_websocket_frame_t *ws)
{
    nxt_unit_websocket_frame_release(ws);
}


static nxt_port_mmap_header_t *
nxt_unit_mmap_get(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    nxt_chunk_id_t *c, int *n, int min_n)
{
    int                     res, nchunks, i;
    uint32_t                outgoing_size;
    nxt_unit_mmap_t         *mm, *mm_end;
    nxt_unit_impl_t         *lib;
    nxt_port_mmap_header_t  *hdr;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    pthread_mutex_lock(&lib->outgoing.mutex);

    if (nxt_slow_path(lib->outgoing.elts == NULL)) {
        goto skip;
    }

retry:

    outgoing_size = lib->outgoing.size;

    mm_end = lib->outgoing.elts + outgoing_size;

    for (mm = lib->outgoing.elts; mm < mm_end; mm++) {
        hdr = mm->hdr;

        if (hdr->sent_over != 0xFFFFu
            && (hdr->sent_over != port->id.id
                || mm->src_thread != pthread_self()))
        {
            continue;
        }

        *c = 0;

        while (nxt_port_mmap_get_free_chunk(hdr->free_map, c)) {
            nchunks = 1;

            while (nchunks < *n) {
                res = nxt_port_mmap_chk_set_chunk_busy(hdr->free_map,
                                                       *c + nchunks);

                if (res == 0) {
                    if (nchunks >= min_n) {
                        *n = nchunks;

                        goto unlock;
                    }

                    for (i = 0; i < nchunks; i++) {
                        nxt_port_mmap_set_chunk_free(hdr->free_map, *c + i);
                    }

                    *c += nchunks + 1;
                    nchunks = 0;
                    break;
                }

                nchunks++;
            }

            if (nchunks >= min_n) {
                *n = nchunks;

                goto unlock;
            }
        }

        hdr->oosm = 1;
    }

    if (outgoing_size >= lib->shm_mmap_limit) {
        /* Cannot allocate more shared memory. */
        pthread_mutex_unlock(&lib->outgoing.mutex);

        if (min_n == 0) {
            *n = 0;
        }

        if (nxt_slow_path(lib->outgoing.allocated_chunks + min_n
                          >= lib->shm_mmap_limit * PORT_MMAP_CHUNK_COUNT))
        {
            /* Memory allocated by application, but not send to router. */
            return NULL;
        }

        /* Notify router about OOSM condition. */

        res = nxt_unit_send_oosm(ctx, port);
        if (nxt_slow_path(res != NXT_UNIT_OK)) {
            return NULL;
        }

        /* Return if caller can handle OOSM condition. Non-blocking mode. */

        if (min_n == 0) {
            return NULL;
        }

        nxt_unit_debug(ctx, "oosm: waiting for ACK");

        res = nxt_unit_wait_shm_ack(ctx);
        if (nxt_slow_path(res != NXT_UNIT_OK)) {
            return NULL;
        }

        nxt_unit_debug(ctx, "oosm: retry");

        pthread_mutex_lock(&lib->outgoing.mutex);

        goto retry;
    }

skip:

    *c = 0;
    hdr = nxt_unit_new_mmap(ctx, port, *n);

unlock:

    nxt_atomic_fetch_add(&lib->outgoing.allocated_chunks, *n);

    nxt_unit_debug(ctx, "allocated_chunks %d",
                   (int) lib->outgoing.allocated_chunks);

    pthread_mutex_unlock(&lib->outgoing.mutex);

    return hdr;
}


static int
nxt_unit_send_oosm(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port)
{
    ssize_t          res;
    nxt_port_msg_t   msg;
    nxt_unit_impl_t  *lib;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    msg.stream = 0;
    msg.pid = lib->pid;
    msg.reply_port = 0;
    msg.type = _NXT_PORT_MSG_OOSM;
    msg.last = 0;
    msg.mmap = 0;
    msg.nf = 0;
    msg.mf = 0;

    res = nxt_unit_port_send(ctx, lib->router_port, &msg, sizeof(msg), NULL);
    if (nxt_slow_path(res != sizeof(msg))) {
        return NXT_UNIT_ERROR;
    }

    return NXT_UNIT_OK;
}


static int
nxt_unit_wait_shm_ack(nxt_unit_ctx_t *ctx)
{
    int                  res;
    nxt_unit_ctx_impl_t  *ctx_impl;
    nxt_unit_read_buf_t  *rbuf;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    while (1) {
        rbuf = nxt_unit_read_buf_get(ctx);
        if (nxt_slow_path(rbuf == NULL)) {
            return NXT_UNIT_ERROR;
        }

        do {
            res = nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf);
        } while (res == NXT_UNIT_AGAIN);

        if (res == NXT_UNIT_ERROR) {
            nxt_unit_read_buf_release(ctx, rbuf);

            return NXT_UNIT_ERROR;
        }

        if (nxt_unit_is_shm_ack(rbuf)) {
            nxt_unit_read_buf_release(ctx, rbuf);
            break;
        }

        pthread_mutex_lock(&ctx_impl->mutex);

        nxt_queue_insert_tail(&ctx_impl->pending_rbuf, &rbuf->link);

        pthread_mutex_unlock(&ctx_impl->mutex);

        if (nxt_unit_is_quit(rbuf)) {
            nxt_unit_debug(ctx, "oosm: quit received");

            return NXT_UNIT_ERROR;
        }
    }

    return NXT_UNIT_OK;
}


static nxt_unit_mmap_t *
nxt_unit_mmap_at(nxt_unit_mmaps_t *mmaps, uint32_t i)
{
    uint32_t         cap, n;
    nxt_unit_mmap_t  *e;

    if (nxt_fast_path(mmaps->size > i)) {
        return mmaps->elts + i;
    }

    cap = mmaps->cap;

    if (cap == 0) {
        cap = i + 1;
    }

    while (i + 1 > cap) {

        if (cap < 16) {
            cap = cap * 2;

        } else {
            cap = cap + cap / 2;
        }
    }

    if (cap != mmaps->cap) {

        e = realloc(mmaps->elts, cap * sizeof(nxt_unit_mmap_t));
        if (nxt_slow_path(e == NULL)) {
            return NULL;
        }

        mmaps->elts = e;

        for (n = mmaps->cap; n < cap; n++) {
            e = mmaps->elts + n;

            e->hdr = NULL;
            nxt_queue_init(&e->awaiting_rbuf);
        }

        mmaps->cap = cap;
    }

    if (i + 1 > mmaps->size) {
        mmaps->size = i + 1;
    }

    return mmaps->elts + i;
}


static nxt_port_mmap_header_t *
nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int n)
{
    int                     i, fd, rc;
    void                    *mem;
    nxt_unit_mmap_t         *mm;
    nxt_unit_impl_t         *lib;
    nxt_port_mmap_header_t  *hdr;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    mm = nxt_unit_mmap_at(&lib->outgoing, lib->outgoing.size);
    if (nxt_slow_path(mm == NULL)) {
        nxt_unit_alert(ctx, "failed to add mmap to outgoing array");

        return NULL;
    }

    fd = nxt_unit_shm_open(ctx, PORT_MMAP_SIZE);
    if (nxt_slow_path(fd == -1)) {
        goto remove_fail;
    }

    mem = mmap(NULL, PORT_MMAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    if (nxt_slow_path(mem == MAP_FAILED)) {
        nxt_unit_alert(ctx, "mmap(%d) failed: %s (%d)", fd,
                       strerror(errno), errno);

        nxt_unit_close(fd);

        goto remove_fail;
    }

    mm->hdr = mem;
    hdr = mem;

    memset(hdr->free_map, 0xFFU, sizeof(hdr->free_map));
    memset(hdr->free_tracking_map, 0xFFU, sizeof(hdr->free_tracking_map));

    hdr->id = lib->outgoing.size - 1;
    hdr->src_pid = lib->pid;
    hdr->dst_pid = port->id.pid;
    hdr->sent_over = port->id.id;
    mm->src_thread = pthread_self();

    /* Mark first n chunk(s) as busy */
    for (i = 0; i < n; i++) {
        nxt_port_mmap_set_chunk_busy(hdr->free_map, i);
    }

    /* Mark as busy chunk followed the last available chunk. */
    nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT);
    nxt_port_mmap_set_chunk_busy(hdr->free_tracking_map, PORT_MMAP_CHUNK_COUNT);

    pthread_mutex_unlock(&lib->outgoing.mutex);

    rc = nxt_unit_send_mmap(ctx, port, fd);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        munmap(mem, PORT_MMAP_SIZE);
        hdr = NULL;

    } else {
        nxt_unit_debug(ctx, "new mmap #%"PRIu32" created for %d -> %d",
                       hdr->id, (int) lib->pid, (int) port->id.pid);
    }

    nxt_unit_close(fd);

    pthread_mutex_lock(&lib->outgoing.mutex);

    if (nxt_fast_path(hdr != NULL)) {
        return hdr;
    }

remove_fail:

    lib->outgoing.size--;

    return NULL;
}


static int
nxt_unit_shm_open(nxt_unit_ctx_t *ctx, size_t size)
{
    int              fd;

#if (NXT_HAVE_MEMFD_CREATE || NXT_HAVE_SHM_OPEN)
    char             name[64];
    nxt_unit_impl_t  *lib;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);
    snprintf(name, sizeof(name), NXT_SHM_PREFIX "unit.%d.%p",
             lib->pid, (void *) (uintptr_t) pthread_self());
#endif

#if (NXT_HAVE_MEMFD_CREATE)

    fd = syscall(SYS_memfd_create, name, MFD_CLOEXEC);
    if (nxt_slow_path(fd == -1)) {
        nxt_unit_alert(ctx, "memfd_create(%s) failed: %s (%d)", name,
                       strerror(errno), errno);

        return -1;
    }

    nxt_unit_debug(ctx, "memfd_create(%s): %d", name, fd);

#elif (NXT_HAVE_SHM_OPEN_ANON)

    fd = shm_open(SHM_ANON, O_RDWR, 0600);
    if (nxt_slow_path(fd == -1)) {
        nxt_unit_alert(ctx, "shm_open(SHM_ANON) failed: %s (%d)",
                       strerror(errno), errno);

        return -1;
    }

#elif (NXT_HAVE_SHM_OPEN)

    /* Just in case. */
    shm_unlink(name);

    fd = shm_open(name, O_CREAT | O_EXCL | O_RDWR, 0600);
    if (nxt_slow_path(fd == -1)) {
        nxt_unit_alert(ctx, "shm_open(%s) failed: %s (%d)", name,
                       strerror(errno), errno);

        return -1;
    }

    if (nxt_slow_path(shm_unlink(name) == -1)) {
        nxt_unit_alert(ctx, "shm_unlink(%s) failed: %s (%d)", name,
                       strerror(errno), errno);
    }

#else

#error No working shared memory implementation.

#endif

    if (nxt_slow_path(ftruncate(fd, size) == -1)) {
        nxt_unit_alert(ctx, "ftruncate(%d) failed: %s (%d)", fd,
                       strerror(errno), errno);

        nxt_unit_close(fd);

        return -1;
    }

    return fd;
}


static int
nxt_unit_send_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int fd)
{
    ssize_t          res;
    nxt_send_oob_t   oob;
    nxt_port_msg_t   msg;
    nxt_unit_impl_t  *lib;
    int              fds[2] = {fd, -1};

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    msg.stream = 0;
    msg.pid = lib->pid;
    msg.reply_port = 0;
    msg.type = _NXT_PORT_MSG_MMAP;
    msg.last = 0;
    msg.mmap = 0;
    msg.nf = 0;
    msg.mf = 0;

    nxt_socket_msg_oob_init(&oob, fds);

    res = nxt_unit_port_send(ctx, port, &msg, sizeof(msg), &oob);
    if (nxt_slow_path(res != sizeof(msg))) {
        return NXT_UNIT_ERROR;
    }

    return NXT_UNIT_OK;
}


static int
nxt_unit_get_outgoing_buf(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    uint32_t size, uint32_t min_size,
    nxt_unit_mmap_buf_t *mmap_buf, char *local_buf)
{
    int                     nchunks, min_nchunks;
    nxt_chunk_id_t          c;
    nxt_port_mmap_header_t  *hdr;

    if (size <= NXT_UNIT_MAX_PLAIN_SIZE) {
        if (local_buf != NULL) {
            mmap_buf->free_ptr = NULL;
            mmap_buf->plain_ptr = local_buf;

        } else {
            mmap_buf->free_ptr = nxt_unit_malloc(ctx,
                                                 size + sizeof(nxt_port_msg_t));
            if (nxt_slow_path(mmap_buf->free_ptr == NULL)) {
                return NXT_UNIT_ERROR;
            }

            mmap_buf->plain_ptr = mmap_buf->free_ptr;
        }

        mmap_buf->hdr = NULL;
        mmap_buf->buf.start = mmap_buf->plain_ptr + sizeof(nxt_port_msg_t);
        mmap_buf->buf.free = mmap_buf->buf.start;
        mmap_buf->buf.end = mmap_buf->buf.start + size;

        nxt_unit_debug(ctx, "outgoing plain buffer allocation: (%p, %d)",
                       mmap_buf->buf.start, (int) size);

        return NXT_UNIT_OK;
    }

    nchunks = (size + PORT_MMAP_CHUNK_SIZE - 1) / PORT_MMAP_CHUNK_SIZE;
    min_nchunks = (min_size + PORT_MMAP_CHUNK_SIZE - 1) / PORT_MMAP_CHUNK_SIZE;

    hdr = nxt_unit_mmap_get(ctx, port, &c, &nchunks, min_nchunks);
    if (nxt_slow_path(hdr == NULL)) {
        if (nxt_fast_path(min_nchunks == 0 && nchunks == 0)) {
            mmap_buf->hdr = NULL;
            mmap_buf->buf.start = NULL;
            mmap_buf->buf.free = NULL;
            mmap_buf->buf.end = NULL;
            mmap_buf->free_ptr = NULL;

            return NXT_UNIT_OK;
        }

        return NXT_UNIT_ERROR;
    }

    mmap_buf->hdr = hdr;
    mmap_buf->buf.start = (char *) nxt_port_mmap_chunk_start(hdr, c);
    mmap_buf->buf.free = mmap_buf->buf.start;
    mmap_buf->buf.end = mmap_buf->buf.start + nchunks * PORT_MMAP_CHUNK_SIZE;
    mmap_buf->free_ptr = NULL;
    mmap_buf->ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    nxt_unit_debug(ctx, "outgoing mmap allocation: (%d,%d,%d)",
                  (int) hdr->id, (int) c,
                  (int) (nchunks * PORT_MMAP_CHUNK_SIZE));

    return NXT_UNIT_OK;
}


static int
nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd)
{
    int                     rc;
    void                    *mem;
    nxt_queue_t             awaiting_rbuf;
    struct stat             mmap_stat;
    nxt_unit_mmap_t         *mm;
    nxt_unit_impl_t         *lib;
    nxt_unit_ctx_impl_t     *ctx_impl;
    nxt_unit_read_buf_t     *rbuf;
    nxt_port_mmap_header_t  *hdr;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    nxt_unit_debug(ctx, "incoming_mmap: fd %d from process %d", fd, (int) pid);

    if (fstat(fd, &mmap_stat) == -1) {
        nxt_unit_alert(ctx, "incoming_mmap: fstat(%d) failed: %s (%d)", fd,
                       strerror(errno), errno);

        return NXT_UNIT_ERROR;
    }

    mem = mmap(NULL, mmap_stat.st_size, PROT_READ | PROT_WRITE,
               MAP_SHARED, fd, 0);
    if (nxt_slow_path(mem == MAP_FAILED)) {
        nxt_unit_alert(ctx, "incoming_mmap: mmap() failed: %s (%d)",
                       strerror(errno), errno);

        return NXT_UNIT_ERROR;
    }

    hdr = mem;

    if (nxt_slow_path(hdr->src_pid != pid)) {

        nxt_unit_alert(ctx, "incoming_mmap: unexpected pid in mmap header "
                       "detected: %d != %d or %d != %d", (int) hdr->src_pid,
                       (int) pid, (int) hdr->dst_pid, (int) lib->pid);

        munmap(mem, PORT_MMAP_SIZE);

        return NXT_UNIT_ERROR;
    }

    nxt_queue_init(&awaiting_rbuf);

    pthread_mutex_lock(&lib->incoming.mutex);

    mm = nxt_unit_mmap_at(&lib->incoming, hdr->id);
    if (nxt_slow_path(mm == NULL)) {
        nxt_unit_alert(ctx, "incoming_mmap: failed to add to incoming array");

        munmap(mem, PORT_MMAP_SIZE);

        rc = NXT_UNIT_ERROR;

    } else {
        mm->hdr = hdr;

        hdr->sent_over = 0xFFFFu;

        nxt_queue_add(&awaiting_rbuf, &mm->awaiting_rbuf);
        nxt_queue_init(&mm->awaiting_rbuf);

        rc = NXT_UNIT_OK;
    }

    pthread_mutex_unlock(&lib->incoming.mutex);

    nxt_queue_each(rbuf, &awaiting_rbuf, nxt_unit_read_buf_t, link) {

        ctx_impl = rbuf->ctx_impl;

        pthread_mutex_lock(&ctx_impl->mutex);

        nxt_queue_insert_head(&ctx_impl->pending_rbuf, &rbuf->link);

        pthread_mutex_unlock(&ctx_impl->mutex);

        nxt_atomic_fetch_add(&ctx_impl->wait_items, -1);

        nxt_unit_awake_ctx(ctx, ctx_impl);

    } nxt_queue_loop;

    return rc;
}


static void
nxt_unit_awake_ctx(nxt_unit_ctx_t *ctx, nxt_unit_ctx_impl_t *ctx_impl)
{
    nxt_port_msg_t  msg;

    if (nxt_fast_path(ctx == &ctx_impl->ctx)) {
        return;
    }

    if (nxt_slow_path(ctx_impl->read_port == NULL
                      || ctx_impl->read_port->out_fd == -1))
    {
        nxt_unit_alert(ctx, "target context read_port is NULL or not writable");

        return;
    }

    memset(&msg, 0, sizeof(nxt_port_msg_t));

    msg.type = _NXT_PORT_MSG_RPC_READY;

    (void) nxt_unit_port_send(ctx, ctx_impl->read_port,
                              &msg, sizeof(msg), NULL);
}


static int
nxt_unit_mmaps_init(nxt_unit_mmaps_t *mmaps)
{
    mmaps->size = 0;
    mmaps->cap = 0;
    mmaps->elts = NULL;
    mmaps->allocated_chunks = 0;

    return pthread_mutex_init(&mmaps->mutex, NULL);
}


nxt_inline void
nxt_unit_process_use(nxt_unit_process_t *process)
{
    nxt_atomic_fetch_add(&process->use_count, 1);
}


nxt_inline void
nxt_unit_process_release(nxt_unit_process_t *process)
{
    long c;

    c = nxt_atomic_fetch_add(&process->use_count, -1);

    if (c == 1) {
        nxt_unit_debug(NULL, "destroy process #%d", (int) process->pid);

        nxt_unit_free(NULL, process);
    }
}


static void
nxt_unit_mmaps_destroy(nxt_unit_mmaps_t *mmaps)
{
    nxt_unit_mmap_t  *mm, *end;

    if (mmaps->elts != NULL) {
        end = mmaps->elts + mmaps->size;

        for (mm = mmaps->elts; mm < end; mm++) {
            munmap(mm->hdr, PORT_MMAP_SIZE);
        }

        nxt_unit_free(NULL, mmaps->elts);
    }

    pthread_mutex_destroy(&mmaps->mutex);
}


static int
nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, nxt_unit_mmaps_t *mmaps,
    pid_t pid, uint32_t id, nxt_port_mmap_header_t **hdr,
    nxt_unit_read_buf_t *rbuf)
{
    int                  res, need_rbuf;
    nxt_unit_mmap_t      *mm;
    nxt_unit_ctx_impl_t  *ctx_impl;

    mm = nxt_unit_mmap_at(mmaps, id);
    if (nxt_slow_path(mm == NULL)) {
        nxt_unit_alert(ctx, "failed to allocate mmap");

        pthread_mutex_unlock(&mmaps->mutex);

        *hdr = NULL;

        return NXT_UNIT_ERROR;
    }

    *hdr = mm->hdr;

    if (nxt_fast_path(*hdr != NULL)) {
        return NXT_UNIT_OK;
    }

    need_rbuf = nxt_queue_is_empty(&mm->awaiting_rbuf);

    nxt_queue_insert_tail(&mm->awaiting_rbuf, &rbuf->link);

    pthread_mutex_unlock(&mmaps->mutex);

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    nxt_atomic_fetch_add(&ctx_impl->wait_items, 1);

    if (need_rbuf) {
        res = nxt_unit_get_mmap(ctx, pid, id);
        if (nxt_slow_path(res == NXT_UNIT_ERROR)) {
            return NXT_UNIT_ERROR;
        }
    }

    return NXT_UNIT_AGAIN;
}


static int
nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg,
    nxt_unit_read_buf_t *rbuf)
{
    int                     res;
    void                    *start;
    uint32_t                size;
    nxt_unit_impl_t         *lib;
    nxt_unit_mmaps_t        *mmaps;
    nxt_unit_mmap_buf_t     *b, **incoming_tail;
    nxt_port_mmap_msg_t     *mmap_msg, *end;
    nxt_port_mmap_header_t  *hdr;

    if (nxt_slow_path(recv_msg->size < sizeof(nxt_port_mmap_msg_t))) {
        nxt_unit_warn(ctx, "#%"PRIu32": mmap_read: too small message (%d)",
                      recv_msg->stream, (int) recv_msg->size);

        return NXT_UNIT_ERROR;
    }

    mmap_msg = recv_msg->start;
    end = nxt_pointer_to(recv_msg->start, recv_msg->size);

    incoming_tail = &recv_msg->incoming_buf;

    /* Allocating buffer structures. */
    for (; mmap_msg < end; mmap_msg++) {
        b = nxt_unit_mmap_buf_get(ctx);
        if (nxt_slow_path(b == NULL)) {
            nxt_unit_warn(ctx, "#%"PRIu32": mmap_read: failed to allocate buf",
                          recv_msg->stream);

            while (recv_msg->incoming_buf != NULL) {
                nxt_unit_mmap_buf_release(recv_msg->incoming_buf);
            }

            return NXT_UNIT_ERROR;
        }

        nxt_unit_mmap_buf_insert(incoming_tail, b);
        incoming_tail = &b->next;
    }

    b = recv_msg->incoming_buf;
    mmap_msg = recv_msg->start;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    mmaps = &lib->incoming;

    pthread_mutex_lock(&mmaps->mutex);

    for (; mmap_msg < end; mmap_msg++) {
        res = nxt_unit_check_rbuf_mmap(ctx, mmaps,
                                       recv_msg->pid, mmap_msg->mmap_id,
                                       &hdr, rbuf);

        if (nxt_slow_path(res != NXT_UNIT_OK)) {
            while (recv_msg->incoming_buf != NULL) {
                nxt_unit_mmap_buf_release(recv_msg->incoming_buf);
            }

            return res;
        }

        start = nxt_port_mmap_chunk_start(hdr, mmap_msg->chunk_id);
        size = mmap_msg->size;

        if (recv_msg->start == mmap_msg) {
            recv_msg->start = start;
            recv_msg->size = size;
        }

        b->buf.start = start;
        b->buf.free = start;
        b->buf.end = b->buf.start + size;
        b->hdr = hdr;

        b = b->next;

        nxt_unit_debug(ctx, "#%"PRIu32": mmap_read: [%p,%d] %d->%d,(%d,%d,%d)",
                       recv_msg->stream,
                       start, (int) size,
                       (int) hdr->src_pid, (int) hdr->dst_pid,
                       (int) hdr->id, (int) mmap_msg->chunk_id,
                       (int) mmap_msg->size);
    }

    pthread_mutex_unlock(&mmaps->mutex);

    return NXT_UNIT_OK;
}


static int
nxt_unit_get_mmap(nxt_unit_ctx_t *ctx, pid_t pid, uint32_t id)
{
    ssize_t              res;
    nxt_unit_impl_t      *lib;
    nxt_unit_ctx_impl_t  *ctx_impl;

    struct {
        nxt_port_msg_t           msg;
        nxt_port_msg_get_mmap_t  get_mmap;
    } m;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);
    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    memset(&m.msg, 0, sizeof(nxt_port_msg_t));

    m.msg.pid = lib->pid;
    m.msg.reply_port = ctx_impl->read_port->id.id;
    m.msg.type = _NXT_PORT_MSG_GET_MMAP;

    m.get_mmap.id = id;

    nxt_unit_debug(ctx, "get_mmap: %d %d", (int) pid, (int) id);

    res = nxt_unit_port_send(ctx, lib->router_port, &m, sizeof(m), NULL);
    if (nxt_slow_path(res != sizeof(m))) {
        return NXT_UNIT_ERROR;
    }

    return NXT_UNIT_OK;
}


static void
nxt_unit_mmap_release(nxt_unit_ctx_t *ctx, nxt_port_mmap_header_t *hdr,
    void *start, uint32_t size)
{
    int              freed_chunks;
    u_char           *p, *end;
    nxt_chunk_id_t   c;
    nxt_unit_impl_t  *lib;

    memset(start, 0xA5, size);

    p = start;
    end = p + size;
    c = nxt_port_mmap_chunk_id(hdr, p);
    freed_chunks = 0;

    while (p < end) {
        nxt_port_mmap_set_chunk_free(hdr->free_map, c);

        p += PORT_MMAP_CHUNK_SIZE;
        c++;
        freed_chunks++;
    }

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    if (hdr->src_pid == lib->pid && freed_chunks != 0) {
        nxt_atomic_fetch_add(&lib->outgoing.allocated_chunks, -freed_chunks);

        nxt_unit_debug(ctx, "allocated_chunks %d",
                       (int) lib->outgoing.allocated_chunks);
    }

    if (hdr->dst_pid == lib->pid
        && freed_chunks != 0
        && nxt_atomic_cmp_set(&hdr->oosm, 1, 0))
    {
        nxt_unit_send_shm_ack(ctx, hdr->src_pid);
    }
}


static int
nxt_unit_send_shm_ack(nxt_unit_ctx_t *ctx, pid_t pid)
{
    ssize_t          res;
    nxt_port_msg_t   msg;
    nxt_unit_impl_t  *lib;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    msg.stream = 0;
    msg.pid = lib->pid;
    msg.reply_port = 0;
    msg.type = _NXT_PORT_MSG_SHM_ACK;
    msg.last = 0;
    msg.mmap = 0;
    msg.nf = 0;
    msg.mf = 0;

    res = nxt_unit_port_send(ctx, lib->router_port, &msg, sizeof(msg), NULL);
    if (nxt_slow_path(res != sizeof(msg))) {
        return NXT_UNIT_ERROR;
    }

    return NXT_UNIT_OK;
}


static nxt_int_t
nxt_unit_lvlhsh_pid_test(nxt_lvlhsh_query_t *lhq, void *data)
{
    nxt_process_t  *process;

    process = data;

    if (lhq->key.length == sizeof(pid_t)
        && *(pid_t *) lhq->key.start == process->pid)
    {
        return NXT_OK;
    }

    return NXT_DECLINED;
}


static const nxt_lvlhsh_proto_t  lvlhsh_processes_proto  nxt_aligned(64) = {
    NXT_LVLHSH_DEFAULT,
    nxt_unit_lvlhsh_pid_test,
    nxt_unit_lvlhsh_alloc,
    nxt_unit_lvlhsh_free,
};


static inline void
nxt_unit_process_lhq_pid(nxt_lvlhsh_query_t *lhq, pid_t *pid)
{
    lhq->key_hash = nxt_murmur_hash2(pid, sizeof(*pid));
    lhq->key.length = sizeof(*pid);
    lhq->key.start = (u_char *) pid;
    lhq->proto = &lvlhsh_processes_proto;
}


static nxt_unit_process_t *
nxt_unit_process_get(nxt_unit_ctx_t *ctx, pid_t pid)
{
    nxt_unit_impl_t     *lib;
    nxt_unit_process_t  *process;
    nxt_lvlhsh_query_t  lhq;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    nxt_unit_process_lhq_pid(&lhq, &pid);

    if (nxt_lvlhsh_find(&lib->processes, &lhq) == NXT_OK) {
        process = lhq.value;
        nxt_unit_process_use(process);

        return process;
    }

    process = nxt_unit_malloc(ctx, sizeof(nxt_unit_process_t));
    if (nxt_slow_path(process == NULL)) {
        nxt_unit_alert(ctx, "failed to allocate process for #%d", (int) pid);

        return NULL;
    }

    process->pid = pid;
    process->use_count = 2;
    process->next_port_id = 0;
    process->lib = lib;

    nxt_queue_init(&process->ports);

    lhq.replace = 0;
    lhq.value = process;

    switch (nxt_lvlhsh_insert(&lib->processes, &lhq)) {

    case NXT_OK:
        break;

    default:
        nxt_unit_alert(ctx, "process %d insert failed", (int) pid);

        nxt_unit_free(ctx, process);
        process = NULL;
        break;
    }

    return process;
}


static nxt_unit_process_t *
nxt_unit_process_find(nxt_unit_impl_t *lib, pid_t pid, int remove)
{
    int                 rc;
    nxt_lvlhsh_query_t  lhq;

    nxt_unit_process_lhq_pid(&lhq, &pid);

    if (remove) {
        rc = nxt_lvlhsh_delete(&lib->processes, &lhq);

    } else {
        rc = nxt_lvlhsh_find(&lib->processes, &lhq);
    }

    if (rc == NXT_OK) {
        if (!remove) {
            nxt_unit_process_use(lhq.value);
        }

        return lhq.value;
    }

    return NULL;
}


static nxt_unit_process_t *
nxt_unit_process_pop_first(nxt_unit_impl_t *lib)
{
    return nxt_lvlhsh_retrieve(&lib->processes, &lvlhsh_processes_proto, NULL);
}


int
nxt_unit_run(nxt_unit_ctx_t *ctx)
{
    int                  rc;
    nxt_unit_ctx_impl_t  *ctx_impl;

    nxt_unit_ctx_use(ctx);

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    rc = NXT_UNIT_OK;

    while (nxt_fast_path(ctx_impl->online)) {
        rc = nxt_unit_run_once_impl(ctx);

        if (nxt_slow_path(rc == NXT_UNIT_ERROR)) {
            nxt_unit_quit(ctx, NXT_QUIT_NORMAL);
            break;
        }
    }

    nxt_unit_ctx_release(ctx);

    return rc;
}


int
nxt_unit_run_once(nxt_unit_ctx_t *ctx)
{
    int  rc;

    nxt_unit_ctx_use(ctx);

    rc = nxt_unit_run_once_impl(ctx);

    nxt_unit_ctx_release(ctx);

    return rc;
}


static int
nxt_unit_run_once_impl(nxt_unit_ctx_t *ctx)
{
    int                  rc;
    nxt_unit_read_buf_t  *rbuf;

    rbuf = nxt_unit_read_buf_get(ctx);
    if (nxt_slow_path(rbuf == NULL)) {
        return NXT_UNIT_ERROR;
    }

    rc = nxt_unit_read_buf(ctx, rbuf);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        nxt_unit_read_buf_release(ctx, rbuf);

        return rc;
    }

    rc = nxt_unit_process_msg(ctx, rbuf, NULL);
    if (nxt_slow_path(rc == NXT_UNIT_ERROR)) {
        return NXT_UNIT_ERROR;
    }

    rc = nxt_unit_process_pending_rbuf(ctx);
    if (nxt_slow_path(rc == NXT_UNIT_ERROR)) {
        return NXT_UNIT_ERROR;
    }

    nxt_unit_process_ready_req(ctx);

    return rc;
}


static int
nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf)
{
    int                   nevents, res, err;
    nxt_uint_t            nfds;
    nxt_unit_impl_t       *lib;
    nxt_unit_ctx_impl_t   *ctx_impl;
    nxt_unit_port_impl_t  *port_impl;
    struct pollfd         fds[2];

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    if (ctx_impl->wait_items > 0 || !nxt_unit_chk_ready(ctx)) {
        return nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf);
    }

    port_impl = nxt_container_of(ctx_impl->read_port, nxt_unit_port_impl_t,
                                 port);

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

retry:

    if (port_impl->from_socket == 0) {
        res = nxt_unit_port_queue_recv(ctx_impl->read_port, rbuf);
        if (res == NXT_UNIT_OK) {
            if (nxt_unit_is_read_socket(rbuf)) {
                port_impl->from_socket++;

                nxt_unit_debug(ctx, "port{%d,%d} dequeue 1 read_socket %d",
                               (int) ctx_impl->read_port->id.pid,
                               (int) ctx_impl->read_port->id.id,
                               port_impl->from_socket);

            } else {
                nxt_unit_debug(ctx, "port{%d,%d} dequeue %d",
                               (int) ctx_impl->read_port->id.pid,
                               (int) ctx_impl->read_port->id.id,
                               (int) rbuf->size);

                return NXT_UNIT_OK;
            }
        }
    }

    if (nxt_fast_path(nxt_unit_chk_ready(ctx))) {
        res = nxt_unit_app_queue_recv(ctx, lib->shared_port, rbuf);
        if (res == NXT_UNIT_OK) {
            return NXT_UNIT_OK;
        }

        fds[1].fd = lib->shared_port->in_fd;
        fds[1].events = POLLIN;

        nfds = 2;

    } else {
        nfds = 1;
    }

    fds[0].fd = ctx_impl->read_port->in_fd;
    fds[0].events = POLLIN;
    fds[0].revents = 0;

    fds[1].revents = 0;

    nevents = poll(fds, nfds, -1);
    if (nxt_slow_path(nevents == -1)) {
        err = errno;

        if (err == EINTR) {
            goto retry;
        }

        nxt_unit_alert(ctx, "poll(%d,%d) failed: %s (%d)",
                       fds[0].fd, fds[1].fd, strerror(err), err);

        rbuf->size = -1;

        return (err == EAGAIN) ? NXT_UNIT_AGAIN : NXT_UNIT_ERROR;
    }

    nxt_unit_debug(ctx, "poll(%d,%d): %d, revents [%04X, %04X]",
                   fds[0].fd, fds[1].fd, nevents, fds[0].revents,
                   fds[1].revents);

    if ((fds[0].revents & POLLIN) != 0) {
        res = nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf);
        if (res == NXT_UNIT_AGAIN) {
            goto retry;
        }

        return res;
    }

    if ((fds[1].revents & POLLIN) != 0) {
        res = nxt_unit_shared_port_recv(ctx, lib->shared_port, rbuf);
        if (res == NXT_UNIT_AGAIN) {
            goto retry;
        }

        return res;
    }

    nxt_unit_alert(ctx, "poll(%d,%d): %d unexpected revents [%04uXi, %04uXi]",
                   fds[0].fd, fds[1].fd, nevents, fds[0].revents,
                   fds[1].revents);

    return NXT_UNIT_ERROR;
}


static int
nxt_unit_chk_ready(nxt_unit_ctx_t *ctx)
{
    nxt_unit_impl_t      *lib;
    nxt_unit_ctx_impl_t  *ctx_impl;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);
    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    return (ctx_impl->ready
            && (lib->request_limit == 0
                || lib->request_count < lib->request_limit));
}


static int
nxt_unit_process_pending_rbuf(nxt_unit_ctx_t *ctx)
{
    int                  rc;
    nxt_queue_t          pending_rbuf;
    nxt_unit_ctx_impl_t  *ctx_impl;
    nxt_unit_read_buf_t  *rbuf;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    pthread_mutex_lock(&ctx_impl->mutex);

    if (nxt_queue_is_empty(&ctx_impl->pending_rbuf)) {
        pthread_mutex_unlock(&ctx_impl->mutex);

        return NXT_UNIT_OK;
    }

    nxt_queue_init(&pending_rbuf);

    nxt_queue_add(&pending_rbuf, &ctx_impl->pending_rbuf);
    nxt_queue_init(&ctx_impl->pending_rbuf);

    pthread_mutex_unlock(&ctx_impl->mutex);

    rc = NXT_UNIT_OK;

    nxt_queue_each(rbuf, &pending_rbuf, nxt_unit_read_buf_t, link) {

        if (nxt_fast_path(rc != NXT_UNIT_ERROR)) {
            rc = nxt_unit_process_msg(&ctx_impl->ctx, rbuf, NULL);

        } else {
            nxt_unit_read_buf_release(ctx, rbuf);
        }

    } nxt_queue_loop;

    if (!ctx_impl->ready) {
        nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL);
    }

    return rc;
}


static void
nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx)
{
    int                           res;
    nxt_queue_t                   ready_req;
    nxt_unit_impl_t               *lib;
    nxt_unit_ctx_impl_t           *ctx_impl;
    nxt_unit_request_info_t       *req;
    nxt_unit_request_info_impl_t  *req_impl;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    pthread_mutex_lock(&ctx_impl->mutex);

    if (nxt_queue_is_empty(&ctx_impl->ready_req)) {
        pthread_mutex_unlock(&ctx_impl->mutex);

        return;
    }

    nxt_queue_init(&ready_req);

    nxt_queue_add(&ready_req, &ctx_impl->ready_req);
    nxt_queue_init(&ctx_impl->ready_req);

    pthread_mutex_unlock(&ctx_impl->mutex);

    nxt_queue_each(req_impl, &ready_req,
                   nxt_unit_request_info_impl_t, port_wait_link)
    {
        lib = nxt_container_of(ctx_impl->ctx.unit, nxt_unit_impl_t, unit);

        req = &req_impl->req;

        res = nxt_unit_send_req_headers_ack(req);
        if (nxt_slow_path(res != NXT_UNIT_OK)) {
            nxt_unit_request_done(req, NXT_UNIT_ERROR);

            continue;
        }

        if (req->content_length
            > (uint64_t) (req->content_buf->end - req->content_buf->free))
        {
            res = nxt_unit_request_hash_add(ctx, req);
            if (nxt_slow_path(res != NXT_UNIT_OK)) {
                nxt_unit_req_warn(req, "failed to add request to hash");

                nxt_unit_request_done(req, NXT_UNIT_ERROR);

                continue;
            }

            /*
             * If application have separate data handler, we may start
             * request processing and process data when it is arrived.
             */
            if (lib->callbacks.data_handler == NULL) {
                continue;
            }
        }

        lib->callbacks.request_handler(&req_impl->req);

    } nxt_queue_loop;
}


int
nxt_unit_run_ctx(nxt_unit_ctx_t *ctx)
{
    int                  rc;
    nxt_unit_read_buf_t  *rbuf;
    nxt_unit_ctx_impl_t  *ctx_impl;

    nxt_unit_ctx_use(ctx);

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    rc = NXT_UNIT_OK;

    while (nxt_fast_path(ctx_impl->online)) {
        rbuf = nxt_unit_read_buf_get(ctx);
        if (nxt_slow_path(rbuf == NULL)) {
            rc = NXT_UNIT_ERROR;
            break;
        }

    retry:

        rc = nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf);
        if (rc == NXT_UNIT_AGAIN) {
            goto retry;
        }

        rc = nxt_unit_process_msg(ctx, rbuf, NULL);
        if (nxt_slow_path(rc == NXT_UNIT_ERROR)) {
            break;
        }

        rc = nxt_unit_process_pending_rbuf(ctx);
        if (nxt_slow_path(rc == NXT_UNIT_ERROR)) {
            break;
        }

        nxt_unit_process_ready_req(ctx);
    }

    nxt_unit_ctx_release(ctx);

    return rc;
}


nxt_inline int
nxt_unit_is_read_queue(nxt_unit_read_buf_t *rbuf)
{
    nxt_port_msg_t  *port_msg;

    if (nxt_fast_path(rbuf->size == (ssize_t) sizeof(nxt_port_msg_t))) {
        port_msg = (nxt_port_msg_t *) rbuf->buf;

        return port_msg->type == _NXT_PORT_MSG_READ_QUEUE;
    }

    return 0;
}


nxt_inline int
nxt_unit_is_read_socket(nxt_unit_read_buf_t *rbuf)
{
    if (nxt_fast_path(rbuf->size == 1)) {
        return rbuf->buf[0] == _NXT_PORT_MSG_READ_SOCKET;
    }

    return 0;
}


nxt_inline int
nxt_unit_is_shm_ack(nxt_unit_read_buf_t *rbuf)
{
    nxt_port_msg_t  *port_msg;

    if (nxt_fast_path(rbuf->size == (ssize_t) sizeof(nxt_port_msg_t))) {
        port_msg = (nxt_port_msg_t *) rbuf->buf;

        return port_msg->type == _NXT_PORT_MSG_SHM_ACK;
    }

    return 0;
}


nxt_inline int
nxt_unit_is_quit(nxt_unit_read_buf_t *rbuf)
{
    nxt_port_msg_t  *port_msg;

    if (nxt_fast_path(rbuf->size == (ssize_t) sizeof(nxt_port_msg_t))) {
        port_msg = (nxt_port_msg_t *) rbuf->buf;

        return port_msg->type == _NXT_PORT_MSG_QUIT;
    }

    return 0;
}


int
nxt_unit_run_shared(nxt_unit_ctx_t *ctx)
{
    int                  rc;
    nxt_unit_impl_t      *lib;
    nxt_unit_read_buf_t  *rbuf;

    nxt_unit_ctx_use(ctx);

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    rc = NXT_UNIT_OK;

    while (nxt_fast_path(nxt_unit_chk_ready(ctx))) {
        rbuf = nxt_unit_read_buf_get(ctx);
        if (nxt_slow_path(rbuf == NULL)) {
            rc = NXT_UNIT_ERROR;
            break;
        }

    retry:

        rc = nxt_unit_shared_port_recv(ctx, lib->shared_port, rbuf);
        if (rc == NXT_UNIT_AGAIN) {
            goto retry;
        }

        if (nxt_slow_path(rc == NXT_UNIT_ERROR)) {
            nxt_unit_read_buf_release(ctx, rbuf);
            break;
        }

        rc = nxt_unit_process_msg(ctx, rbuf, NULL);
        if (nxt_slow_path(rc == NXT_UNIT_ERROR)) {
            break;
        }
    }

    nxt_unit_ctx_release(ctx);

    return rc;
}


nxt_unit_request_info_t *
nxt_unit_dequeue_request(nxt_unit_ctx_t *ctx)
{
    int                      rc;
    nxt_unit_impl_t          *lib;
    nxt_unit_read_buf_t      *rbuf;
    nxt_unit_request_info_t  *req;

    nxt_unit_ctx_use(ctx);

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    req = NULL;

    if (nxt_slow_path(!nxt_unit_chk_ready(ctx))) {
        goto done;
    }

    rbuf = nxt_unit_read_buf_get(ctx);
    if (nxt_slow_path(rbuf == NULL)) {
        goto done;
    }

    rc = nxt_unit_app_queue_recv(ctx, lib->shared_port, rbuf);
    if (rc != NXT_UNIT_OK) {
        nxt_unit_read_buf_release(ctx, rbuf);
        goto done;
    }

    (void) nxt_unit_process_msg(ctx, rbuf, &req);

done:

    nxt_unit_ctx_release(ctx);

    return req;
}


int
nxt_unit_process_port_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port)
{
    int  rc;

    nxt_unit_ctx_use(ctx);

    rc = nxt_unit_process_port_msg_impl(ctx, port);

    nxt_unit_ctx_release(ctx);

    return rc;
}


static int
nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port)
{
    int                  rc;
    nxt_unit_impl_t      *lib;
    nxt_unit_read_buf_t  *rbuf;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    if (port == lib->shared_port && !nxt_unit_chk_ready(ctx)) {
        return NXT_UNIT_AGAIN;
    }

    rbuf = nxt_unit_read_buf_get(ctx);
    if (nxt_slow_path(rbuf == NULL)) {
        return NXT_UNIT_ERROR;
    }

    if (port == lib->shared_port) {
        rc = nxt_unit_shared_port_recv(ctx, port, rbuf);

    } else {
        rc = nxt_unit_ctx_port_recv(ctx, port, rbuf);
    }

    if (rc != NXT_UNIT_OK) {
        nxt_unit_read_buf_release(ctx, rbuf);
        return rc;
    }

    rc = nxt_unit_process_msg(ctx, rbuf, NULL);
    if (nxt_slow_path(rc == NXT_UNIT_ERROR)) {
        return NXT_UNIT_ERROR;
    }

    rc = nxt_unit_process_pending_rbuf(ctx);
    if (nxt_slow_path(rc == NXT_UNIT_ERROR)) {
        return NXT_UNIT_ERROR;
    }

    nxt_unit_process_ready_req(ctx);

    return rc;
}


void
nxt_unit_done(nxt_unit_ctx_t *ctx)
{
    nxt_unit_ctx_release(ctx);
}


nxt_unit_ctx_t *
nxt_unit_ctx_alloc(nxt_unit_ctx_t *ctx, void *data)
{
    int                   rc, queue_fd;
    void                  *mem;
    nxt_unit_impl_t       *lib;
    nxt_unit_port_t       *port;
    nxt_unit_ctx_impl_t   *new_ctx;
    nxt_unit_port_impl_t  *port_impl;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    new_ctx = nxt_unit_malloc(ctx, sizeof(nxt_unit_ctx_impl_t)
                                   + lib->request_data_size);
    if (nxt_slow_path(new_ctx == NULL)) {
        nxt_unit_alert(ctx, "failed to allocate context");

        return NULL;
    }

    rc = nxt_unit_ctx_init(lib, new_ctx, data);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        nxt_unit_free(ctx, new_ctx);

        return NULL;
    }

    queue_fd = -1;

    port = nxt_unit_create_port(&new_ctx->ctx);
    if (nxt_slow_path(port == NULL)) {
        goto fail;
    }

    new_ctx->read_port = port;

    queue_fd = nxt_unit_shm_open(&new_ctx->ctx, sizeof(nxt_port_queue_t));
    if (nxt_slow_path(queue_fd == -1)) {
        goto fail;
    }

    mem = mmap(NULL, sizeof(nxt_port_queue_t),
               PROT_READ | PROT_WRITE, MAP_SHARED, queue_fd, 0);
    if (nxt_slow_path(mem == MAP_FAILED)) {
        nxt_unit_alert(ctx, "mmap(%d) failed: %s (%d)", queue_fd,
                       strerror(errno), errno);

        goto fail;
    }

    nxt_port_queue_init(mem);

    port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port);
    port_impl->queue = mem;

    rc = nxt_unit_send_port(&new_ctx->ctx, lib->router_port, port, queue_fd);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        goto fail;
    }

    nxt_unit_close(queue_fd);

    return &new_ctx->ctx;

fail:

    if (queue_fd != -1) {
        nxt_unit_close(queue_fd);
    }

    nxt_unit_ctx_release(&new_ctx->ctx);

    return NULL;
}


static void
nxt_unit_ctx_free(nxt_unit_ctx_impl_t *ctx_impl)
{
    nxt_unit_impl_t                  *lib;
    nxt_unit_mmap_buf_t              *mmap_buf;
    nxt_unit_read_buf_t              *rbuf;
    nxt_unit_request_info_impl_t     *req_impl;
    nxt_unit_websocket_frame_impl_t  *ws_impl;

    lib = nxt_container_of(ctx_impl->ctx.unit, nxt_unit_impl_t, unit);

    nxt_queue_each(req_impl, &ctx_impl->active_req,
                   nxt_unit_request_info_impl_t, link)
    {
        nxt_unit_req_warn(&req_impl->req, "active request on ctx free");

        nxt_unit_request_done(&req_impl->req, NXT_UNIT_ERROR);

    } nxt_queue_loop;

    nxt_unit_mmap_buf_unlink(&ctx_impl->ctx_buf[0]);
    nxt_unit_mmap_buf_unlink(&ctx_impl->ctx_buf[1]);

    while (ctx_impl->free_buf != NULL) {
        mmap_buf = ctx_impl->free_buf;
        nxt_unit_mmap_buf_unlink(mmap_buf);
        nxt_unit_free(&ctx_impl->ctx, mmap_buf);
    }

    nxt_queue_each(req_impl, &ctx_impl->free_req,
                   nxt_unit_request_info_impl_t, link)
    {
        nxt_unit_request_info_free(req_impl);

    } nxt_queue_loop;

    nxt_queue_each(ws_impl, &ctx_impl->free_ws,
                   nxt_unit_websocket_frame_impl_t, link)
    {
        nxt_unit_websocket_frame_free(&ctx_impl->ctx, ws_impl);

    } nxt_queue_loop;

    nxt_queue_each(rbuf, &ctx_impl->free_rbuf, nxt_unit_read_buf_t, link)
    {
        if (rbuf != &ctx_impl->ctx_read_buf) {
            nxt_unit_free(&ctx_impl->ctx, rbuf);
        }
    } nxt_queue_loop;

    pthread_mutex_destroy(&ctx_impl->mutex);

    pthread_mutex_lock(&lib->mutex);

    nxt_queue_remove(&ctx_impl->link);

    pthread_mutex_unlock(&lib->mutex);

    if (nxt_fast_path(ctx_impl->read_port != NULL)) {
        nxt_unit_remove_port(lib, NULL, &ctx_impl->read_port->id);
        nxt_unit_port_release(ctx_impl->read_port);
    }

    if (ctx_impl != &lib->main_ctx) {
        nxt_unit_free(&lib->main_ctx.ctx, ctx_impl);
    }

    nxt_unit_lib_release(lib);
}


/* SOCK_SEQPACKET is disabled to test SOCK_DGRAM on all platforms. */
#if (0 || NXT_HAVE_AF_UNIX_SOCK_SEQPACKET)
#define NXT_UNIX_SOCKET  SOCK_SEQPACKET
#else
#define NXT_UNIX_SOCKET  SOCK_DGRAM
#endif


void
nxt_unit_port_id_init(nxt_unit_port_id_t *port_id, pid_t pid, uint16_t id)
{
    nxt_unit_port_hash_id_t  port_hash_id;

    port_hash_id.pid = pid;
    port_hash_id.id = id;

    port_id->pid = pid;
    port_id->hash = nxt_murmur_hash2(&port_hash_id, sizeof(port_hash_id));
    port_id->id = id;
}


static nxt_unit_port_t *
nxt_unit_create_port(nxt_unit_ctx_t *ctx)
{
    int                 rc, port_sockets[2];
    nxt_unit_impl_t     *lib;
    nxt_unit_port_t     new_port, *port;
    nxt_unit_process_t  *process;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    rc = socketpair(AF_UNIX, NXT_UNIX_SOCKET, 0, port_sockets);
    if (nxt_slow_path(rc != 0)) {
        nxt_unit_warn(ctx, "create_port: socketpair() failed: %s (%d)",
                      strerror(errno), errno);

        return NULL;
    }

#if (NXT_HAVE_SOCKOPT_SO_PASSCRED)
    int  enable_creds = 1;

    if (nxt_slow_path(setsockopt(port_sockets[0], SOL_SOCKET, SO_PASSCRED,
                        &enable_creds, sizeof(enable_creds)) == -1))
    {
        nxt_unit_warn(ctx, "failed to set SO_PASSCRED %s", strerror(errno));
        return NULL;
    }

    if (nxt_slow_path(setsockopt(port_sockets[1], SOL_SOCKET, SO_PASSCRED,
                        &enable_creds, sizeof(enable_creds)) == -1))
    {
        nxt_unit_warn(ctx, "failed to set SO_PASSCRED %s", strerror(errno));
        return NULL;
    }
#endif

    nxt_unit_debug(ctx, "create_port: new socketpair: %d->%d",
                   port_sockets[0], port_sockets[1]);

    pthread_mutex_lock(&lib->mutex);

    process = nxt_unit_process_get(ctx, lib->pid);
    if (nxt_slow_path(process == NULL)) {
        pthread_mutex_unlock(&lib->mutex);

        nxt_unit_close(port_sockets[0]);
        nxt_unit_close(port_sockets[1]);

        return NULL;
    }

    nxt_unit_port_id_init(&new_port.id, lib->pid, process->next_port_id++);

    new_port.in_fd = port_sockets[0];
    new_port.out_fd = port_sockets[1];
    new_port.data = NULL;

    pthread_mutex_unlock(&lib->mutex);

    nxt_unit_process_release(process);

    port = nxt_unit_add_port(ctx, &new_port, NULL);
    if (nxt_slow_path(port == NULL)) {
        nxt_unit_close(port_sockets[0]);
        nxt_unit_close(port_sockets[1]);
    }

    return port;
}


static int
nxt_unit_send_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *dst,
    nxt_unit_port_t *port, int queue_fd)
{
    ssize_t          res;
    nxt_send_oob_t   oob;
    nxt_unit_impl_t  *lib;
    int              fds[2] = { port->out_fd, queue_fd };

    struct {
        nxt_port_msg_t            msg;
        nxt_port_msg_new_port_t   new_port;
    } m;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    m.msg.stream = 0;
    m.msg.pid = lib->pid;
    m.msg.reply_port = 0;
    m.msg.type = _NXT_PORT_MSG_NEW_PORT;
    m.msg.last = 0;
    m.msg.mmap = 0;
    m.msg.nf = 0;
    m.msg.mf = 0;

    m.new_port.id = port->id.id;
    m.new_port.pid = port->id.pid;
    m.new_port.type = NXT_PROCESS_APP;
    m.new_port.max_size = 16 * 1024;
    m.new_port.max_share = 64 * 1024;

    nxt_socket_msg_oob_init(&oob, fds);

    res = nxt_unit_port_send(ctx, dst, &m, sizeof(m), &oob);

    return (res == sizeof(m)) ? NXT_UNIT_OK : NXT_UNIT_ERROR;
}


nxt_inline void nxt_unit_port_use(nxt_unit_port_t *port)
{
    nxt_unit_port_impl_t  *port_impl;

    port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port);

    nxt_atomic_fetch_add(&port_impl->use_count, 1);
}


nxt_inline void nxt_unit_port_release(nxt_unit_port_t *port)
{
    long                  c;
    nxt_unit_port_impl_t  *port_impl;

    port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port);

    c = nxt_atomic_fetch_add(&port_impl->use_count, -1);

    if (c == 1) {
        nxt_unit_debug(NULL, "destroy port{%d,%d} in_fd %d out_fd %d",
                       (int) port->id.pid, (int) port->id.id,
                       port->in_fd, port->out_fd);

        nxt_unit_process_release(port_impl->process);

        if (port->in_fd != -1) {
            nxt_unit_close(port->in_fd);

            port->in_fd = -1;
        }

        if (port->out_fd != -1) {
            nxt_unit_close(port->out_fd);

            port->out_fd = -1;
        }

        if (port_impl->queue != NULL) {
            munmap(port_impl->queue, (port->id.id == NXT_UNIT_SHARED_PORT_ID)
                                     ? sizeof(nxt_app_queue_t)
                                     : sizeof(nxt_port_queue_t));
        }

        nxt_unit_free(NULL, port_impl);
    }
}


static nxt_unit_port_t *
nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue)
{
    int                   rc, ready;
    nxt_queue_t           awaiting_req;
    nxt_unit_impl_t       *lib;
    nxt_unit_port_t       *old_port;
    nxt_unit_process_t    *process;
    nxt_unit_port_impl_t  *new_port, *old_port_impl;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    pthread_mutex_lock(&lib->mutex);

    old_port = nxt_unit_port_hash_find(&lib->ports, &port->id, 0);

    if (nxt_slow_path(old_port != NULL)) {
        nxt_unit_debug(ctx, "add_port: duplicate port{%d,%d} "
                            "in_fd %d out_fd %d queue %p",
                            port->id.pid, port->id.id,
                            port->in_fd, port->out_fd, queue);

        if (old_port->data == NULL) {
            old_port->data = port->data;
            port->data = NULL;
        }

        if (old_port->in_fd == -1) {
            old_port->in_fd = port->in_fd;
            port->in_fd = -1;
        }

        if (port->in_fd != -1) {
            nxt_unit_close(port->in_fd);
            port->in_fd = -1;
        }

        if (old_port->out_fd == -1) {
            old_port->out_fd = port->out_fd;
            port->out_fd = -1;
        }

        if (port->out_fd != -1) {
            nxt_unit_close(port->out_fd);
            port->out_fd = -1;
        }

        *port = *old_port;

        nxt_queue_init(&awaiting_req);

        old_port_impl = nxt_container_of(old_port, nxt_unit_port_impl_t, port);

        if (old_port_impl->queue == NULL) {
            old_port_impl->queue = queue;
        }

        ready = (port->in_fd != -1 || port->out_fd != -1);

        /*
         * Port can be market as 'ready' only after callbacks.add_port() call.
         * Otherwise, request may try to use the port before callback.
         */
        if (lib->callbacks.add_port == NULL && ready) {
            old_port_impl->ready = ready;

            if (!nxt_queue_is_empty(&old_port_impl->awaiting_req)) {
                nxt_queue_add(&awaiting_req, &old_port_impl->awaiting_req);
                nxt_queue_init(&old_port_impl->awaiting_req);
            }
        }

        pthread_mutex_unlock(&lib->mutex);

        if (lib->callbacks.add_port != NULL && ready) {
            lib->callbacks.add_port(ctx, old_port);

            pthread_mutex_lock(&lib->mutex);

            old_port_impl->ready = ready;

            if (!nxt_queue_is_empty(&old_port_impl->awaiting_req)) {
                nxt_queue_add(&awaiting_req, &old_port_impl->awaiting_req);
                nxt_queue_init(&old_port_impl->awaiting_req);
            }

            pthread_mutex_unlock(&lib->mutex);
        }

        nxt_unit_process_awaiting_req(ctx, &awaiting_req);

        return old_port;
    }

    new_port = NULL;
    ready = 0;

    nxt_unit_debug(ctx, "add_port: port{%d,%d} in_fd %d out_fd %d queue %p",
                   port->id.pid, port->id.id,
                   port->in_fd, port->out_fd, queue);

    process = nxt_unit_process_get(ctx, port->id.pid);
    if (nxt_slow_path(process == NULL)) {
        goto unlock;
    }

    if (port->id.id != NXT_UNIT_SHARED_PORT_ID
        && port->id.id >= process->next_port_id)
    {
        process->next_port_id = port->id.id + 1;
    }

    new_port = nxt_unit_malloc(ctx, sizeof(nxt_unit_port_impl_t));
    if (nxt_slow_path(new_port == NULL)) {
        nxt_unit_alert(ctx, "add_port: %d,%d malloc() failed",
                       port->id.pid, port->id.id);

        goto unlock;
    }

    new_port->port = *port;

    rc = nxt_unit_port_hash_add(&lib->ports, &new_port->port);
    if (nxt_slow_path(rc != NXT_UNIT_OK)) {
        nxt_unit_alert(ctx, "add_port: %d,%d hash_add failed",
                       port->id.pid, port->id.id);

        nxt_unit_free(ctx, new_port);

        new_port = NULL;

        goto unlock;
    }

    nxt_queue_insert_tail(&process->ports, &new_port->link);

    new_port->use_count = 2;
    new_port->process = process;
    new_port->queue = queue;
    new_port->from_socket = 0;
    new_port->socket_rbuf = NULL;

    nxt_queue_init(&new_port->awaiting_req);

    ready = (port->in_fd != -1 || port->out_fd != -1);

    if (lib->callbacks.add_port == NULL) {
        new_port->ready = ready;

    } else {
        new_port->ready = 0;
    }

    process = NULL;

unlock:

    pthread_mutex_unlock(&lib->mutex);

    if (nxt_slow_path(process != NULL)) {
        nxt_unit_process_release(process);
    }

    if (lib->callbacks.add_port != NULL && new_port != NULL && ready) {
        lib->callbacks.add_port(ctx, &new_port->port);

        nxt_queue_init(&awaiting_req);

        pthread_mutex_lock(&lib->mutex);

        new_port->ready = 1;

        if (!nxt_queue_is_empty(&new_port->awaiting_req)) {
            nxt_queue_add(&awaiting_req, &new_port->awaiting_req);
            nxt_queue_init(&new_port->awaiting_req);
        }

        pthread_mutex_unlock(&lib->mutex);

        nxt_unit_process_awaiting_req(ctx, &awaiting_req);
    }

    return (new_port == NULL) ? NULL : &new_port->port;
}


static void
nxt_unit_process_awaiting_req(nxt_unit_ctx_t *ctx, nxt_queue_t *awaiting_req)
{
    nxt_unit_ctx_impl_t           *ctx_impl;
    nxt_unit_request_info_impl_t  *req_impl;

    nxt_queue_each(req_impl, awaiting_req,
                   nxt_unit_request_info_impl_t, port_wait_link)
    {
        nxt_queue_remove(&req_impl->port_wait_link);

        ctx_impl = nxt_container_of(req_impl->req.ctx, nxt_unit_ctx_impl_t,
                                    ctx);

        pthread_mutex_lock(&ctx_impl->mutex);

        nxt_queue_insert_tail(&ctx_impl->ready_req,
                              &req_impl->port_wait_link);

        pthread_mutex_unlock(&ctx_impl->mutex);

        nxt_atomic_fetch_add(&ctx_impl->wait_items, -1);

        nxt_unit_awake_ctx(ctx, ctx_impl);

    } nxt_queue_loop;
}


static void
nxt_unit_remove_port(nxt_unit_impl_t *lib, nxt_unit_ctx_t *ctx,
    nxt_unit_port_id_t *port_id)
{
    nxt_unit_port_t       *port;
    nxt_unit_port_impl_t  *port_impl;

    pthread_mutex_lock(&lib->mutex);

    port = nxt_unit_remove_port_unsafe(lib, port_id);

    if (nxt_fast_path(port != NULL)) {
        port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port);

        nxt_queue_remove(&port_impl->link);
    }

    pthread_mutex_unlock(&lib->mutex);

    if (lib->callbacks.remove_port != NULL && port != NULL) {
        lib->callbacks.remove_port(&lib->unit, ctx, port);
    }

    if (nxt_fast_path(port != NULL)) {
        nxt_unit_port_release(port);
    }
}


static nxt_unit_port_t *
nxt_unit_remove_port_unsafe(nxt_unit_impl_t *lib, nxt_unit_port_id_t *port_id)
{
    nxt_unit_port_t  *port;

    port = nxt_unit_port_hash_find(&lib->ports, port_id, 1);
    if (nxt_slow_path(port == NULL)) {
        nxt_unit_debug(NULL, "remove_port: port{%d,%d} not found",
                       (int) port_id->pid, (int) port_id->id);

        return NULL;
    }

    nxt_unit_debug(NULL, "remove_port: port{%d,%d}, fds %d,%d, data %p",
                   (int) port_id->pid, (int) port_id->id,
                   port->in_fd, port->out_fd, port->data);

    return port;
}


static void
nxt_unit_remove_pid(nxt_unit_impl_t *lib, pid_t pid)
{
    nxt_unit_process_t  *process;

    pthread_mutex_lock(&lib->mutex);

    process = nxt_unit_process_find(lib, pid, 1);
    if (nxt_slow_path(process == NULL)) {
        nxt_unit_debug(NULL, "remove_pid: process %d not found", (int) pid);

        pthread_mutex_unlock(&lib->mutex);

        return;
    }

    nxt_unit_remove_process(lib, process);

    if (lib->callbacks.remove_pid != NULL) {
        lib->callbacks.remove_pid(&lib->unit, pid);
    }
}


static void
nxt_unit_remove_process(nxt_unit_impl_t *lib, nxt_unit_process_t *process)
{
    nxt_queue_t           ports;
    nxt_unit_port_impl_t  *port;

    nxt_queue_init(&ports);

    nxt_queue_add(&ports, &process->ports);

    nxt_queue_each(port, &ports, nxt_unit_port_impl_t, link) {

        nxt_unit_remove_port_unsafe(lib, &port->port.id);

    } nxt_queue_loop;

    pthread_mutex_unlock(&lib->mutex);

    nxt_queue_each(port, &ports, nxt_unit_port_impl_t, link) {

        nxt_queue_remove(&port->link);

        if (lib->callbacks.remove_port != NULL) {
            lib->callbacks.remove_port(&lib->unit, NULL, &port->port);
        }

        nxt_unit_port_release(&port->port);

    } nxt_queue_loop;

    nxt_unit_process_release(process);
}


static void
nxt_unit_quit(nxt_unit_ctx_t *ctx, uint8_t quit_param)
{
    nxt_bool_t                    skip_graceful_broadcast, quit;
    nxt_unit_impl_t               *lib;
    nxt_unit_ctx_impl_t           *ctx_impl;
    nxt_unit_callbacks_t          *cb;
    nxt_unit_request_info_t       *req;
    nxt_unit_request_info_impl_t  *req_impl;

    struct {
        nxt_port_msg_t            msg;
        uint8_t                   quit_param;
    } nxt_packed m;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);
    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    nxt_unit_debug(ctx, "quit: %d/%d/%d", (int) quit_param, ctx_impl->ready,
                   ctx_impl->online);

    if (nxt_slow_path(!ctx_impl->online)) {
        return;
    }

    skip_graceful_broadcast = quit_param == NXT_QUIT_GRACEFUL
                              && !ctx_impl->ready;

    cb = &lib->callbacks;

    if (nxt_fast_path(ctx_impl->ready)) {
        ctx_impl->ready = 0;

        if (cb->remove_port != NULL) {
            cb->remove_port(&lib->unit, ctx, lib->shared_port);
        }
    }

    if (quit_param == NXT_QUIT_GRACEFUL) {
        pthread_mutex_lock(&ctx_impl->mutex);

        quit = nxt_queue_is_empty(&ctx_impl->active_req)
               && nxt_queue_is_empty(&ctx_impl->pending_rbuf)
               && ctx_impl->wait_items == 0;

        pthread_mutex_unlock(&ctx_impl->mutex);

    } else {
        quit = 1;
        ctx_impl->quit_param = NXT_QUIT_GRACEFUL;
    }

    if (quit) {
        ctx_impl->online = 0;

        if (cb->quit != NULL) {
            cb->quit(ctx);
        }

        nxt_queue_each(req_impl, &ctx_impl->active_req,
                       nxt_unit_request_info_impl_t, link)
        {
            req = &req_impl->req;

            nxt_unit_req_warn(req, "active request on ctx quit");

            if (cb->close_handler) {
                nxt_unit_req_debug(req, "close_handler");

                cb->close_handler(req);

            } else {
                nxt_unit_request_done(req, NXT_UNIT_ERROR);
            }

        } nxt_queue_loop;

        if (nxt_fast_path(ctx_impl->read_port != NULL)) {
            nxt_unit_remove_port(lib, ctx, &ctx_impl->read_port->id);
        }
    }

    if (ctx != &lib->main_ctx.ctx || skip_graceful_broadcast) {
        return;
    }

    memset(&m.msg, 0, sizeof(nxt_port_msg_t));

    m.msg.pid = lib->pid;
    m.msg.type = _NXT_PORT_MSG_QUIT;
    m.quit_param = quit_param;

    pthread_mutex_lock(&lib->mutex);

    nxt_queue_each(ctx_impl, &lib->contexts, nxt_unit_ctx_impl_t, link) {

        if (ctx == &ctx_impl->ctx
            || ctx_impl->read_port == NULL
            || ctx_impl->read_port->out_fd == -1)
        {
            continue;
        }

        (void) nxt_unit_port_send(ctx, ctx_impl->read_port,
                                  &m, sizeof(m), NULL);

    } nxt_queue_loop;

    pthread_mutex_unlock(&lib->mutex);
}


static int
nxt_unit_get_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id)
{
    ssize_t              res;
    nxt_unit_impl_t      *lib;
    nxt_unit_ctx_impl_t  *ctx_impl;

    struct {
        nxt_port_msg_t           msg;
        nxt_port_msg_get_port_t  get_port;
    } m;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);
    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    memset(&m.msg, 0, sizeof(nxt_port_msg_t));

    m.msg.pid = lib->pid;
    m.msg.reply_port = ctx_impl->read_port->id.id;
    m.msg.type = _NXT_PORT_MSG_GET_PORT;

    m.get_port.id = port_id->id;
    m.get_port.pid = port_id->pid;

    nxt_unit_debug(ctx, "get_port: %d %d", (int) port_id->pid,
                   (int) port_id->id);

    res = nxt_unit_port_send(ctx, lib->router_port, &m, sizeof(m), NULL);
    if (nxt_slow_path(res != sizeof(m))) {
        return NXT_UNIT_ERROR;
    }

    return NXT_UNIT_OK;
}


static ssize_t
nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    const void *buf, size_t buf_size, const nxt_send_oob_t *oob)
{
    int                   notify;
    ssize_t               ret;
    nxt_int_t             rc;
    nxt_port_msg_t        msg;
    nxt_unit_impl_t       *lib;
    nxt_unit_port_impl_t  *port_impl;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port);
    if (port_impl->queue != NULL && (oob == NULL || oob->size == 0)
        && buf_size <= NXT_PORT_QUEUE_MSG_SIZE)
    {
        rc = nxt_port_queue_send(port_impl->queue, buf, buf_size, &notify);
        if (nxt_slow_path(rc != NXT_OK)) {
            nxt_unit_alert(ctx, "port_send: port %d,%d queue overflow",
                           (int) port->id.pid, (int) port->id.id);

            return -1;
        }

        nxt_unit_debug(ctx, "port{%d,%d} enqueue %d notify %d",
                       (int) port->id.pid, (int) port->id.id,
                       (int) buf_size, notify);

        if (notify) {
            memcpy(&msg, buf, sizeof(nxt_port_msg_t));

            msg.type = _NXT_PORT_MSG_READ_QUEUE;

            if (lib->callbacks.port_send == NULL) {
                ret = nxt_unit_sendmsg(ctx, port->out_fd, &msg,
                                       sizeof(nxt_port_msg_t), NULL);

                nxt_unit_debug(ctx, "port{%d,%d} send %d read_queue",
                               (int) port->id.pid, (int) port->id.id,
                               (int) ret);

            } else {
                ret = lib->callbacks.port_send(ctx, port, &msg,
                                               sizeof(nxt_port_msg_t), NULL, 0);

                nxt_unit_debug(ctx, "port{%d,%d} sendcb %d read_queue",
                               (int) port->id.pid, (int) port->id.id,
                               (int) ret);
            }

        }

        return buf_size;
    }

    if (port_impl->queue != NULL) {
        msg.type = _NXT_PORT_MSG_READ_SOCKET;

        rc = nxt_port_queue_send(port_impl->queue, &msg.type, 1, &notify);
        if (nxt_slow_path(rc != NXT_OK)) {
            nxt_unit_alert(ctx, "port_send: port %d,%d queue overflow",
                           (int) port->id.pid, (int) port->id.id);

            return -1;
        }

        nxt_unit_debug(ctx, "port{%d,%d} enqueue 1 read_socket notify %d",
                       (int) port->id.pid, (int) port->id.id, notify);
    }

    if (lib->callbacks.port_send != NULL) {
        ret = lib->callbacks.port_send(ctx, port, buf, buf_size,
                                       oob != NULL ? oob->buf : NULL,
                                       oob != NULL ? oob->size : 0);

        nxt_unit_debug(ctx, "port{%d,%d} sendcb %d",
                       (int) port->id.pid, (int) port->id.id,
                       (int) ret);

    } else {
        ret = nxt_unit_sendmsg(ctx, port->out_fd, buf, buf_size, oob);

        nxt_unit_debug(ctx, "port{%d,%d} sendmsg %d",
                       (int) port->id.pid, (int) port->id.id,
                       (int) ret);
    }

    return ret;
}


static ssize_t
nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, int fd,
    const void *buf, size_t buf_size, const nxt_send_oob_t *oob)
{
    int            err;
    ssize_t        n;
    struct iovec   iov[1];

    iov[0].iov_base = (void *) buf;
    iov[0].iov_len = buf_size;

retry:

    n = nxt_sendmsg(fd, iov, 1, oob);

    if (nxt_slow_path(n == -1)) {
        err = errno;

        if (err == EINTR) {
            goto retry;
        }

        /*
         * FIXME: This should be "alert" after router graceful shutdown
         * implementation.
         */
        nxt_unit_warn(ctx, "sendmsg(%d, %d) failed: %s (%d)",
                      fd, (int) buf_size, strerror(err), err);

    } else {
        nxt_unit_debug(ctx, "sendmsg(%d, %d, %d): %d", fd, (int) buf_size,
                       (oob != NULL ? (int) oob->size : 0), (int) n);
    }

    return n;
}


static int
nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    nxt_unit_read_buf_t *rbuf)
{
    int                   res, read;
    nxt_unit_port_impl_t  *port_impl;

    port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port);

    read = 0;

retry:

    if (port_impl->from_socket > 0) {
        if (port_impl->socket_rbuf != NULL
            && port_impl->socket_rbuf->size > 0)
        {
            port_impl->from_socket--;

            nxt_unit_rbuf_cpy(rbuf, port_impl->socket_rbuf);
            port_impl->socket_rbuf->size = 0;

            nxt_unit_debug(ctx, "port{%d,%d} use suspended message %d",
                           (int) port->id.pid, (int) port->id.id,
                           (int) rbuf->size);

            return NXT_UNIT_OK;
        }

    } else {
        res = nxt_unit_port_queue_recv(port, rbuf);

        if (res == NXT_UNIT_OK) {
            if (nxt_unit_is_read_socket(rbuf)) {
                port_impl->from_socket++;

                nxt_unit_debug(ctx, "port{%d,%d} dequeue 1 read_socket %d",
                               (int) port->id.pid, (int) port->id.id,
                               port_impl->from_socket);

                goto retry;
            }

            nxt_unit_debug(ctx, "port{%d,%d} dequeue %d",
                           (int) port->id.pid, (int) port->id.id,
                           (int) rbuf->size);

            return NXT_UNIT_OK;
        }
    }

    if (read) {
        return NXT_UNIT_AGAIN;
    }

    res = nxt_unit_port_recv(ctx, port, rbuf);
    if (nxt_slow_path(res == NXT_UNIT_ERROR)) {
        return NXT_UNIT_ERROR;
    }

    read = 1;

    if (nxt_unit_is_read_queue(rbuf)) {
        nxt_unit_debug(ctx, "port{%d,%d} recv %d read_queue",
                       (int) port->id.pid, (int) port->id.id, (int) rbuf->size);

        goto retry;
    }

    nxt_unit_debug(ctx, "port{%d,%d} recvmsg %d",
                   (int) port->id.pid, (int) port->id.id,
                   (int) rbuf->size);

    if (res == NXT_UNIT_AGAIN) {
        return NXT_UNIT_AGAIN;
    }

    if (port_impl->from_socket > 0) {
        port_impl->from_socket--;

        return NXT_UNIT_OK;
    }

    nxt_unit_debug(ctx, "port{%d,%d} suspend message %d",
                   (int) port->id.pid, (int) port->id.id,
                   (int) rbuf->size);

    if (port_impl->socket_rbuf == NULL) {
        port_impl->socket_rbuf = nxt_unit_read_buf_get(ctx);

        if (nxt_slow_path(port_impl->socket_rbuf == NULL)) {
            return NXT_UNIT_ERROR;
        }

        port_impl->socket_rbuf->size = 0;
    }

    if (port_impl->socket_rbuf->size > 0) {
        nxt_unit_alert(ctx, "too many port socket messages");

        return NXT_UNIT_ERROR;
    }

    nxt_unit_rbuf_cpy(port_impl->socket_rbuf, rbuf);

    rbuf->oob.size = 0;

    goto retry;
}


nxt_inline void
nxt_unit_rbuf_cpy(nxt_unit_read_buf_t *dst, nxt_unit_read_buf_t *src)
{
    memcpy(dst->buf, src->buf, src->size);
    dst->size = src->size;
    dst->oob.size = src->oob.size;
    memcpy(dst->oob.buf, src->oob.buf, src->oob.size);
}


static int
nxt_unit_shared_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    nxt_unit_read_buf_t *rbuf)
{
    int                   res;
    nxt_unit_port_impl_t  *port_impl;

    port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port);

retry:

    res = nxt_unit_app_queue_recv(ctx, port, rbuf);

    if (res == NXT_UNIT_OK) {
        return NXT_UNIT_OK;
    }

    if (res == NXT_UNIT_AGAIN) {
        res = nxt_unit_port_recv(ctx, port, rbuf);
        if (nxt_slow_path(res == NXT_UNIT_ERROR)) {
            return NXT_UNIT_ERROR;
        }

        if (nxt_unit_is_read_queue(rbuf)) {
            nxt_app_queue_notification_received(port_impl->queue);

            nxt_unit_debug(ctx, "port{%d,%d} recv %d read_queue",
                           (int) port->id.pid, (int) port->id.id, (int) rbuf->size);

            goto retry;
        }
    }

    return res;
}


static int
nxt_unit_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    nxt_unit_read_buf_t *rbuf)
{
    int              fd, err;
    size_t           oob_size;
    struct iovec     iov[1];
    nxt_unit_impl_t  *lib;

    lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

    if (lib->callbacks.port_recv != NULL) {
        oob_size = sizeof(rbuf->oob.buf);

        rbuf->size = lib->callbacks.port_recv(ctx, port,
                                              rbuf->buf, sizeof(rbuf->buf),
                                              rbuf->oob.buf, &oob_size);

        nxt_unit_debug(ctx, "port{%d,%d} recvcb %d",
                       (int) port->id.pid, (int) port->id.id, (int) rbuf->size);

        if (nxt_slow_path(rbuf->size < 0)) {
            return NXT_UNIT_ERROR;
        }

        rbuf->oob.size = oob_size;
        return NXT_UNIT_OK;
    }

    iov[0].iov_base = rbuf->buf;
    iov[0].iov_len = sizeof(rbuf->buf);

    fd = port->in_fd;

retry:

    rbuf->size = nxt_recvmsg(fd, iov, 1, &rbuf->oob);

    if (nxt_slow_path(rbuf->size == -1)) {
        err = errno;

        if (err == EINTR) {
            goto retry;
        }

        if (err == EAGAIN) {
            nxt_unit_debug(ctx, "recvmsg(%d) failed: %s (%d)",
                           fd, strerror(err), err);

            return NXT_UNIT_AGAIN;
        }

        nxt_unit_alert(ctx, "recvmsg(%d) failed: %s (%d)",
                       fd, strerror(err), err);

        return NXT_UNIT_ERROR;
    }

    nxt_unit_debug(ctx, "recvmsg(%d): %d", fd, (int) rbuf->size);

    return NXT_UNIT_OK;
}


static int
nxt_unit_port_queue_recv(nxt_unit_port_t *port, nxt_unit_read_buf_t *rbuf)
{
    nxt_unit_port_impl_t  *port_impl;

    port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port);

    rbuf->size = nxt_port_queue_recv(port_impl->queue, rbuf->buf);

    return (rbuf->size == -1) ? NXT_UNIT_AGAIN : NXT_UNIT_OK;
}


static int
nxt_unit_app_queue_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port,
    nxt_unit_read_buf_t *rbuf)
{
    uint32_t              cookie;
    nxt_port_msg_t        *port_msg;
    nxt_app_queue_t       *queue;
    nxt_unit_impl_t       *lib;
    nxt_unit_port_impl_t  *port_impl;

    struct {
        nxt_port_msg_t    msg;
        uint8_t           quit_param;
    } nxt_packed m;

    port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port);
    queue = port_impl->queue;

retry:

    rbuf->size = nxt_app_queue_recv(queue, rbuf->buf, &cookie);

    nxt_unit_debug(NULL, "app_queue_recv: %d", (int) rbuf->size);

    if (rbuf->size >= (ssize_t) sizeof(nxt_port_msg_t)) {
        port_msg = (nxt_port_msg_t *) rbuf->buf;

        if (nxt_app_queue_cancel(queue, cookie, port_msg->stream)) {
            lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

            if (lib->request_limit != 0) {
                nxt_atomic_fetch_add(&lib->request_count, 1);

                if (nxt_slow_path(lib->request_count >= lib->request_limit)) {
                    nxt_unit_debug(ctx, "request limit reached");

                    memset(&m.msg, 0, sizeof(nxt_port_msg_t));

                    m.msg.pid = lib->pid;
                    m.msg.type = _NXT_PORT_MSG_QUIT;
                    m.quit_param = NXT_QUIT_GRACEFUL;

                    (void) nxt_unit_port_send(ctx, lib->main_ctx.read_port,
                                              &m, sizeof(m), NULL);
                }
            }

            return NXT_UNIT_OK;
        }

        nxt_unit_debug(NULL, "app_queue_recv: message cancelled");

        goto retry;
    }

    return (rbuf->size == -1) ? NXT_UNIT_AGAIN : NXT_UNIT_OK;
}


nxt_inline int
nxt_unit_close(int fd)
{
    int  res;

    res = close(fd);

    if (nxt_slow_path(res == -1)) {
        nxt_unit_alert(NULL, "close(%d) failed: %s (%d)",
                       fd, strerror(errno), errno);

    } else {
        nxt_unit_debug(NULL, "close(%d): %d", fd, res);
    }

    return res;
}


static int
nxt_unit_fd_blocking(int fd)
{
    int  nb;

    nb = 0;

    if (nxt_slow_path(ioctl(fd, FIONBIO, &nb) == -1)) {
        nxt_unit_alert(NULL, "ioctl(%d, FIONBIO, 0) failed: %s (%d)",
                       fd, strerror(errno), errno);

        return NXT_UNIT_ERROR;
    }

    return NXT_UNIT_OK;
}


static nxt_int_t
nxt_unit_port_hash_test(nxt_lvlhsh_query_t *lhq, void *data)
{
    nxt_unit_port_t          *port;
    nxt_unit_port_hash_id_t  *port_id;

    port = data;
    port_id = (nxt_unit_port_hash_id_t *) lhq->key.start;

    if (lhq->key.length == sizeof(nxt_unit_port_hash_id_t)
        && port_id->pid == port->id.pid
        && port_id->id == port->id.id)
    {
        return NXT_OK;
    }

    return NXT_DECLINED;
}


static const nxt_lvlhsh_proto_t  lvlhsh_ports_proto  nxt_aligned(64) = {
    NXT_LVLHSH_DEFAULT,
    nxt_unit_port_hash_test,
    nxt_unit_lvlhsh_alloc,
    nxt_unit_lvlhsh_free,
};


static inline void
nxt_unit_port_hash_lhq(nxt_lvlhsh_query_t *lhq,
    nxt_unit_port_hash_id_t *port_hash_id,
    nxt_unit_port_id_t *port_id)
{
    port_hash_id->pid = port_id->pid;
    port_hash_id->id = port_id->id;

    if (nxt_fast_path(port_id->hash != 0)) {
        lhq->key_hash = port_id->hash;

    } else {
        lhq->key_hash = nxt_murmur_hash2(port_hash_id, sizeof(*port_hash_id));

        port_id->hash = lhq->key_hash;

        nxt_unit_debug(NULL, "calculate hash for port_id (%d, %d): %04X",
                       (int) port_id->pid, (int) port_id->id,
                       (int) port_id->hash);
    }

    lhq->key.length = sizeof(nxt_unit_port_hash_id_t);
    lhq->key.start = (u_char *) port_hash_id;
    lhq->proto = &lvlhsh_ports_proto;
    lhq->pool = NULL;
}


static int
nxt_unit_port_hash_add(nxt_lvlhsh_t *port_hash, nxt_unit_port_t *port)
{
    nxt_int_t                res;
    nxt_lvlhsh_query_t       lhq;
    nxt_unit_port_hash_id_t  port_hash_id;

    nxt_unit_port_hash_lhq(&lhq, &port_hash_id, &port->id);
    lhq.replace = 0;
    lhq.value = port;

    res = nxt_lvlhsh_insert(port_hash, &lhq);

    switch (res) {

    case NXT_OK:
        return NXT_UNIT_OK;

    default:
        return NXT_UNIT_ERROR;
    }
}


static nxt_unit_port_t *
nxt_unit_port_hash_find(nxt_lvlhsh_t *port_hash, nxt_unit_port_id_t *port_id,
    int remove)
{
    nxt_int_t                res;
    nxt_lvlhsh_query_t       lhq;
    nxt_unit_port_hash_id_t  port_hash_id;

    nxt_unit_port_hash_lhq(&lhq, &port_hash_id, port_id);

    if (remove) {
        res = nxt_lvlhsh_delete(port_hash, &lhq);

    } else {
        res = nxt_lvlhsh_find(port_hash, &lhq);
    }

    switch (res) {

    case NXT_OK:
        if (!remove) {
            nxt_unit_port_use(lhq.value);
        }

        return lhq.value;

    default:
        return NULL;
    }
}


static nxt_int_t
nxt_unit_request_hash_test(nxt_lvlhsh_query_t *lhq, void *data)
{
    return NXT_OK;
}


static const nxt_lvlhsh_proto_t  lvlhsh_requests_proto  nxt_aligned(64) = {
    NXT_LVLHSH_DEFAULT,
    nxt_unit_request_hash_test,
    nxt_unit_lvlhsh_alloc,
    nxt_unit_lvlhsh_free,
};


static int
nxt_unit_request_hash_add(nxt_unit_ctx_t *ctx,
    nxt_unit_request_info_t *req)
{
    uint32_t                      *stream;
    nxt_int_t                     res;
    nxt_lvlhsh_query_t            lhq;
    nxt_unit_ctx_impl_t           *ctx_impl;
    nxt_unit_request_info_impl_t  *req_impl;

    req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);
    if (req_impl->in_hash) {
        return NXT_UNIT_OK;
    }

    stream = &req_impl->stream;

    lhq.key_hash = nxt_murmur_hash2(stream, sizeof(*stream));
    lhq.key.length = sizeof(*stream);
    lhq.key.start = (u_char *) stream;
    lhq.proto = &lvlhsh_requests_proto;
    lhq.pool = NULL;
    lhq.replace = 0;
    lhq.value = req_impl;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    pthread_mutex_lock(&ctx_impl->mutex);

    res = nxt_lvlhsh_insert(&ctx_impl->requests, &lhq);

    pthread_mutex_unlock(&ctx_impl->mutex);

    switch (res) {

    case NXT_OK:
        req_impl->in_hash = 1;
        return NXT_UNIT_OK;

    default:
        return NXT_UNIT_ERROR;
    }
}


static nxt_unit_request_info_t *
nxt_unit_request_hash_find(nxt_unit_ctx_t *ctx, uint32_t stream, int remove)
{
    nxt_int_t                     res;
    nxt_lvlhsh_query_t            lhq;
    nxt_unit_ctx_impl_t           *ctx_impl;
    nxt_unit_request_info_impl_t  *req_impl;

    lhq.key_hash = nxt_murmur_hash2(&stream, sizeof(stream));
    lhq.key.length = sizeof(stream);
    lhq.key.start = (u_char *) &stream;
    lhq.proto = &lvlhsh_requests_proto;
    lhq.pool = NULL;

    ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx);

    pthread_mutex_lock(&ctx_impl->mutex);

    if (remove) {
        res = nxt_lvlhsh_delete(&ctx_impl->requests, &lhq);

    } else {
        res = nxt_lvlhsh_find(&ctx_impl->requests, &lhq);
    }

    pthread_mutex_unlock(&ctx_impl->mutex);

    switch (res) {

    case NXT_OK:
        req_impl = nxt_container_of(lhq.value, nxt_unit_request_info_impl_t,
                                    req);
        if (remove) {
            req_impl->in_hash = 0;
        }

        return lhq.value;

    default:
        return NULL;
    }
}


void
nxt_unit_log(nxt_unit_ctx_t *ctx, int level, const char *fmt, ...)
{
    int              log_fd, n;
    char             msg[NXT_MAX_ERROR_STR], *p, *end;
    pid_t            pid;
    va_list          ap;
    nxt_unit_impl_t  *lib;

    if (nxt_fast_path(ctx != NULL)) {
        lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit);

        pid = lib->pid;
        log_fd = lib->log_fd;

    } else {
        pid = nxt_unit_pid;
        log_fd = STDERR_FILENO;
    }

    p = msg;
    end = p + sizeof(msg) - 1;

    p = nxt_unit_snprint_prefix(p, end, pid, level);

    va_start(ap, fmt);
    p += vsnprintf(p, end - p, fmt, ap);
    va_end(ap);

    if (nxt_slow_path(p > end)) {
        memcpy(end - 5, "[...]", 5);
        p = end;
    }

    *p++ = '\n';

    n = write(log_fd, msg, p - msg);
    if (nxt_slow_path(n < 0)) {
        fprintf(stderr, "Failed to write log: %.*s", (int) (p - msg), msg);
    }
}


void
nxt_unit_req_log(nxt_unit_request_info_t *req, int level, const char *fmt, ...)
{
    int                           log_fd, n;
    char                          msg[NXT_MAX_ERROR_STR], *p, *end;
    pid_t                         pid;
    va_list                       ap;
    nxt_unit_impl_t               *lib;
    nxt_unit_request_info_impl_t  *req_impl;

    if (nxt_fast_path(req != NULL)) {
        lib = nxt_container_of(req->ctx->unit, nxt_unit_impl_t, unit);

        pid = lib->pid;
        log_fd = lib->log_fd;

    } else {
        pid = nxt_unit_pid;
        log_fd = STDERR_FILENO;
    }

    p = msg;
    end = p + sizeof(msg) - 1;

    p = nxt_unit_snprint_prefix(p, end, pid, level);

    if (nxt_fast_path(req != NULL)) {
        req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req);

        p += snprintf(p, end - p, "#%"PRIu32": ", req_impl->stream);
    }

    va_start(ap, fmt);
    p += vsnprintf(p, end - p, fmt, ap);
    va_end(ap);

    if (nxt_slow_path(p > end)) {
        memcpy(end - 5, "[...]", 5);
        p = end;
    }

    *p++ = '\n';

    n = write(log_fd, msg, p - msg);
    if (nxt_slow_path(n < 0)) {
        fprintf(stderr, "Failed to write log: %.*s", (int) (p - msg), msg);
    }
}


static const char * nxt_unit_log_levels[] = {
    "alert",
    "error",
    "warn",
    "notice",
    "info",
    "debug",
};


static char *
nxt_unit_snprint_prefix(char *p, char *end, pid_t pid, int level)
{
    struct tm        tm;
    struct timespec  ts;

    (void) clock_gettime(CLOCK_REALTIME, &ts);

#if (NXT_HAVE_LOCALTIME_R)
    (void) localtime_r(&ts.tv_sec, &tm);
#else
    tm = *localtime(&ts.tv_sec);
#endif

#if (NXT_DEBUG)
    p += snprintf(p, end - p,
                  "%4d/%02d/%02d %02d:%02d:%02d.%03d ",
                  tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
                  tm.tm_hour, tm.tm_min, tm.tm_sec,
                  (int) ts.tv_nsec / 1000000);
#else
    p += snprintf(p, end - p,
                  "%4d/%02d/%02d %02d:%02d:%02d ",
                  tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
                  tm.tm_hour, tm.tm_min, tm.tm_sec);
#endif

    p += snprintf(p, end - p,
                  "[%s] %d#%"PRIu64" [unit] ", nxt_unit_log_levels[level],
                  (int) pid,
                  (uint64_t) (uintptr_t) nxt_thread_get_tid());

    return p;
}


static void *
nxt_unit_lvlhsh_alloc(void *data, size_t size)
{
    int   err;
    void  *p;

    err = posix_memalign(&p, size, size);

    if (nxt_fast_path(err == 0)) {
        nxt_unit_debug(NULL, "posix_memalign(%d, %d): %p",
                       (int) size, (int) size, p);
        return p;
    }

    nxt_unit_alert(NULL, "posix_memalign(%d, %d) failed: %s (%d)",
                   (int) size, (int) size, strerror(err), err);
    return NULL;
}


static void
nxt_unit_lvlhsh_free(void *data, void *p)
{
    nxt_unit_free(NULL, p);
}


void *
nxt_unit_malloc(nxt_unit_ctx_t *ctx, size_t size)
{
    void  *p;

    p = malloc(size);

    if (nxt_fast_path(p != NULL)) {
#if (NXT_DEBUG_ALLOC)
        nxt_unit_debug(ctx, "malloc(%d): %p", (int) size, p);
#endif

    } else {
        nxt_unit_alert(ctx, "malloc(%d) failed: %s (%d)",
                       (int) size, strerror(errno), errno);
    }

    return p;
}


void
nxt_unit_free(nxt_unit_ctx_t *ctx, void *p)
{
#if (NXT_DEBUG_ALLOC)
    nxt_unit_debug(ctx, "free(%p)", p);
#endif

    free(p);
}


static int
nxt_unit_memcasecmp(const void *p1, const void *p2, size_t length)
{
    u_char        c1, c2;
    nxt_int_t     n;
    const u_char  *s1, *s2;

    s1 = p1;
    s2 = p2;

    while (length-- != 0) {
        c1 = *s1++;
        c2 = *s2++;

        c1 = nxt_lowcase(c1);
        c2 = nxt_lowcase(c2);

        n = c1 - c2;

        if (n != 0) {
            return n;
        }
    }

    return 0;
}