180Smax.romanov@nginx.com 280Smax.romanov@nginx.com /* 380Smax.romanov@nginx.com * Copyright (C) Max Romanov 480Smax.romanov@nginx.com * Copyright (C) NGINX, Inc. 580Smax.romanov@nginx.com */ 680Smax.romanov@nginx.com 780Smax.romanov@nginx.com #ifndef _NXT_PORT_MEMORY_INT_H_INCLUDED_ 880Smax.romanov@nginx.com #define _NXT_PORT_MEMORY_INT_H_INCLUDED_ 980Smax.romanov@nginx.com 1080Smax.romanov@nginx.com 1180Smax.romanov@nginx.com #include <stdint.h> 1280Smax.romanov@nginx.com #include <nxt_atomic.h> 1380Smax.romanov@nginx.com 1480Smax.romanov@nginx.com 1580Smax.romanov@nginx.com #ifdef NXT_MMAP_TINY_CHUNK 1680Smax.romanov@nginx.com 1780Smax.romanov@nginx.com #define PORT_MMAP_CHUNK_SIZE 16 1880Smax.romanov@nginx.com #define PORT_MMAP_HEADER_SIZE 1024 1980Smax.romanov@nginx.com #define PORT_MMAP_DATA_SIZE 1024 2080Smax.romanov@nginx.com 2180Smax.romanov@nginx.com #else 2280Smax.romanov@nginx.com 2380Smax.romanov@nginx.com #define PORT_MMAP_CHUNK_SIZE (1024 * 16) 2480Smax.romanov@nginx.com #define PORT_MMAP_HEADER_SIZE (1024 * 4) 2580Smax.romanov@nginx.com #define PORT_MMAP_DATA_SIZE (1024 * 1024 * 10) 2680Smax.romanov@nginx.com 2780Smax.romanov@nginx.com #endif 2880Smax.romanov@nginx.com 2980Smax.romanov@nginx.com 3080Smax.romanov@nginx.com #define PORT_MMAP_SIZE (PORT_MMAP_HEADER_SIZE + PORT_MMAP_DATA_SIZE) 3180Smax.romanov@nginx.com #define PORT_MMAP_CHUNK_COUNT (PORT_MMAP_DATA_SIZE / PORT_MMAP_CHUNK_SIZE) 3280Smax.romanov@nginx.com 3380Smax.romanov@nginx.com 3480Smax.romanov@nginx.com typedef uint32_t nxt_chunk_id_t; 3580Smax.romanov@nginx.com 3680Smax.romanov@nginx.com typedef nxt_atomic_uint_t nxt_free_map_t; 3780Smax.romanov@nginx.com 3880Smax.romanov@nginx.com #define FREE_BITS (sizeof(nxt_free_map_t) * 8) 3980Smax.romanov@nginx.com 4080Smax.romanov@nginx.com #define FREE_IDX(nchunk) ((nchunk) / FREE_BITS) 4180Smax.romanov@nginx.com 4280Smax.romanov@nginx.com #define FREE_MASK(nchunk) \ 4380Smax.romanov@nginx.com ( 1ULL << ( (nchunk) % FREE_BITS ) ) 4480Smax.romanov@nginx.com 4580Smax.romanov@nginx.com #define MAX_FREE_IDX FREE_IDX(PORT_MMAP_CHUNK_COUNT) 4680Smax.romanov@nginx.com 4780Smax.romanov@nginx.com 4880Smax.romanov@nginx.com /* Mapped at the start of shared memory segment. */ 4980Smax.romanov@nginx.com struct nxt_port_mmap_header_s { 5080Smax.romanov@nginx.com uint32_t id; 51*363Smax.romanov@nginx.com nxt_pid_t src_pid; /* For sanity check. */ 52*363Smax.romanov@nginx.com nxt_pid_t dst_pid; /* For sanity check. */ 53323Smax.romanov@nginx.com nxt_port_id_t sent_over; 5480Smax.romanov@nginx.com nxt_free_map_t free_map[MAX_FREE_IDX]; 5580Smax.romanov@nginx.com }; 5680Smax.romanov@nginx.com 5780Smax.romanov@nginx.com 5880Smax.romanov@nginx.com /* 5980Smax.romanov@nginx.com * Element of nxt_process_t.incoming/outgoing, shared memory segment 6080Smax.romanov@nginx.com * descriptor. 6180Smax.romanov@nginx.com */ 6280Smax.romanov@nginx.com struct nxt_port_mmap_s { 6380Smax.romanov@nginx.com nxt_port_mmap_header_t *hdr; 6480Smax.romanov@nginx.com }; 6580Smax.romanov@nginx.com 6680Smax.romanov@nginx.com typedef struct nxt_port_mmap_msg_s nxt_port_mmap_msg_t; 6780Smax.romanov@nginx.com 6880Smax.romanov@nginx.com /* Passed as a second iov chunk when 'mmap' bit in nxt_port_msg_t is 1. */ 6980Smax.romanov@nginx.com struct nxt_port_mmap_msg_s { 7080Smax.romanov@nginx.com uint32_t mmap_id; /* Mmap index in nxt_process_t.outgoing. */ 7180Smax.romanov@nginx.com nxt_chunk_id_t chunk_id; /* Mmap chunk index. */ 7280Smax.romanov@nginx.com uint32_t size; /* Payload data size. */ 7380Smax.romanov@nginx.com }; 7480Smax.romanov@nginx.com 7580Smax.romanov@nginx.com 7680Smax.romanov@nginx.com static nxt_bool_t 7780Smax.romanov@nginx.com nxt_port_mmap_get_free_chunk(nxt_port_mmap_header_t *hdr, nxt_chunk_id_t *c); 7880Smax.romanov@nginx.com 7980Smax.romanov@nginx.com #define nxt_port_mmap_get_chunk_busy(hdr, c) \ 8080Smax.romanov@nginx.com ((hdr->free_map[FREE_IDX(c)] & FREE_MASK(c)) == 0) 8180Smax.romanov@nginx.com 8280Smax.romanov@nginx.com nxt_inline void 8380Smax.romanov@nginx.com nxt_port_mmap_set_chunk_busy(nxt_port_mmap_header_t *hdr, nxt_chunk_id_t c); 8480Smax.romanov@nginx.com 8580Smax.romanov@nginx.com nxt_inline nxt_bool_t 8680Smax.romanov@nginx.com nxt_port_mmap_chk_set_chunk_busy(nxt_port_mmap_header_t *hdr, nxt_chunk_id_t c); 8780Smax.romanov@nginx.com 8880Smax.romanov@nginx.com nxt_inline void 8980Smax.romanov@nginx.com nxt_port_mmap_set_chunk_free(nxt_port_mmap_header_t *hdr, nxt_chunk_id_t c); 9080Smax.romanov@nginx.com 9180Smax.romanov@nginx.com nxt_inline nxt_chunk_id_t 9280Smax.romanov@nginx.com nxt_port_mmap_chunk_id(nxt_port_mmap_header_t *hdr, u_char *p) 9380Smax.romanov@nginx.com { 9480Smax.romanov@nginx.com u_char *mm_start; 9580Smax.romanov@nginx.com 9680Smax.romanov@nginx.com mm_start = (u_char *) hdr; 9780Smax.romanov@nginx.com 9880Smax.romanov@nginx.com return ((p - mm_start) - PORT_MMAP_HEADER_SIZE) / PORT_MMAP_CHUNK_SIZE; 9980Smax.romanov@nginx.com } 10080Smax.romanov@nginx.com 10180Smax.romanov@nginx.com 10280Smax.romanov@nginx.com nxt_inline u_char * 10380Smax.romanov@nginx.com nxt_port_mmap_chunk_start(nxt_port_mmap_header_t *hdr, nxt_chunk_id_t c) 10480Smax.romanov@nginx.com { 10580Smax.romanov@nginx.com u_char *mm_start; 10680Smax.romanov@nginx.com 10780Smax.romanov@nginx.com mm_start = (u_char *) hdr; 10880Smax.romanov@nginx.com 10980Smax.romanov@nginx.com return mm_start + PORT_MMAP_HEADER_SIZE + c * PORT_MMAP_CHUNK_SIZE; 11080Smax.romanov@nginx.com } 11180Smax.romanov@nginx.com 11280Smax.romanov@nginx.com 11380Smax.romanov@nginx.com static nxt_bool_t 11480Smax.romanov@nginx.com nxt_port_mmap_get_free_chunk(nxt_port_mmap_header_t *hdr, nxt_chunk_id_t *c) 11580Smax.romanov@nginx.com { 11680Smax.romanov@nginx.com int ffs; 11780Smax.romanov@nginx.com size_t i; 11880Smax.romanov@nginx.com nxt_chunk_id_t chunk; 11980Smax.romanov@nginx.com nxt_free_map_t bits; 12080Smax.romanov@nginx.com nxt_free_map_t *free_map; 12180Smax.romanov@nginx.com 12280Smax.romanov@nginx.com free_map = hdr->free_map; 12380Smax.romanov@nginx.com 12480Smax.romanov@nginx.com for (i = 0; i < MAX_FREE_IDX; i++) { 12580Smax.romanov@nginx.com bits = free_map[i]; 12680Smax.romanov@nginx.com if (bits == 0) { 12780Smax.romanov@nginx.com continue; 12880Smax.romanov@nginx.com } 12980Smax.romanov@nginx.com 13080Smax.romanov@nginx.com ffs = __builtin_ffsll(bits); 13180Smax.romanov@nginx.com if (ffs != 0) { 13280Smax.romanov@nginx.com chunk = i * FREE_BITS + ffs - 1; 13380Smax.romanov@nginx.com 13480Smax.romanov@nginx.com if (nxt_port_mmap_chk_set_chunk_busy(hdr, chunk)) { 13580Smax.romanov@nginx.com *c = chunk; 13680Smax.romanov@nginx.com return 1; 13780Smax.romanov@nginx.com } 13880Smax.romanov@nginx.com } 13980Smax.romanov@nginx.com } 14080Smax.romanov@nginx.com 14180Smax.romanov@nginx.com return 0; 14280Smax.romanov@nginx.com } 14380Smax.romanov@nginx.com 14480Smax.romanov@nginx.com 14580Smax.romanov@nginx.com nxt_inline void 14680Smax.romanov@nginx.com nxt_port_mmap_set_chunk_busy(nxt_port_mmap_header_t *hdr, nxt_chunk_id_t c) 14780Smax.romanov@nginx.com { 14880Smax.romanov@nginx.com nxt_atomic_and_fetch(hdr->free_map + FREE_IDX(c), ~FREE_MASK(c)); 14980Smax.romanov@nginx.com } 15080Smax.romanov@nginx.com 15180Smax.romanov@nginx.com 15280Smax.romanov@nginx.com nxt_inline nxt_bool_t 15380Smax.romanov@nginx.com nxt_port_mmap_chk_set_chunk_busy(nxt_port_mmap_header_t *hdr, nxt_chunk_id_t c) 15480Smax.romanov@nginx.com { 15580Smax.romanov@nginx.com nxt_free_map_t *f; 15680Smax.romanov@nginx.com nxt_free_map_t free_val, busy_val; 15780Smax.romanov@nginx.com 15880Smax.romanov@nginx.com f = hdr->free_map + FREE_IDX(c); 15980Smax.romanov@nginx.com 16080Smax.romanov@nginx.com while ( (*f & FREE_MASK(c)) != 0 ) { 16180Smax.romanov@nginx.com 16280Smax.romanov@nginx.com free_val = *f | FREE_MASK(c); 16380Smax.romanov@nginx.com busy_val = free_val & ~FREE_MASK(c); 16480Smax.romanov@nginx.com 16580Smax.romanov@nginx.com if (nxt_atomic_cmp_set(f, free_val, busy_val) != 0) { 16680Smax.romanov@nginx.com return 1; 16780Smax.romanov@nginx.com } 16880Smax.romanov@nginx.com } 16980Smax.romanov@nginx.com 17080Smax.romanov@nginx.com return 0; 17180Smax.romanov@nginx.com } 17280Smax.romanov@nginx.com 17380Smax.romanov@nginx.com 17480Smax.romanov@nginx.com nxt_inline void 17580Smax.romanov@nginx.com nxt_port_mmap_set_chunk_free(nxt_port_mmap_header_t *hdr, nxt_chunk_id_t c) 17680Smax.romanov@nginx.com { 17780Smax.romanov@nginx.com nxt_atomic_or_fetch(hdr->free_map + FREE_IDX(c), FREE_MASK(c)); 17880Smax.romanov@nginx.com } 17980Smax.romanov@nginx.com 18080Smax.romanov@nginx.com 18180Smax.romanov@nginx.com #endif /* _NXT_PORT_MEMORY_INT_H_INCLUDED_ */ 182