180Smax.romanov@nginx.com 280Smax.romanov@nginx.com /* 380Smax.romanov@nginx.com * Copyright (C) Max Romanov 480Smax.romanov@nginx.com * Copyright (C) NGINX, Inc. 580Smax.romanov@nginx.com */ 680Smax.romanov@nginx.com 780Smax.romanov@nginx.com #ifndef _NXT_PORT_MEMORY_INT_H_INCLUDED_ 880Smax.romanov@nginx.com #define _NXT_PORT_MEMORY_INT_H_INCLUDED_ 980Smax.romanov@nginx.com 1080Smax.romanov@nginx.com 1180Smax.romanov@nginx.com #include <stdint.h> 1280Smax.romanov@nginx.com #include <nxt_atomic.h> 1380Smax.romanov@nginx.com 1480Smax.romanov@nginx.com 1580Smax.romanov@nginx.com #ifdef NXT_MMAP_TINY_CHUNK 1680Smax.romanov@nginx.com 1780Smax.romanov@nginx.com #define PORT_MMAP_CHUNK_SIZE 16 1880Smax.romanov@nginx.com #define PORT_MMAP_HEADER_SIZE 1024 1980Smax.romanov@nginx.com #define PORT_MMAP_DATA_SIZE 1024 2080Smax.romanov@nginx.com 2180Smax.romanov@nginx.com #else 2280Smax.romanov@nginx.com 2380Smax.romanov@nginx.com #define PORT_MMAP_CHUNK_SIZE (1024 * 16) 2480Smax.romanov@nginx.com #define PORT_MMAP_HEADER_SIZE (1024 * 4) 2580Smax.romanov@nginx.com #define PORT_MMAP_DATA_SIZE (1024 * 1024 * 10) 2680Smax.romanov@nginx.com 2780Smax.romanov@nginx.com #endif 2880Smax.romanov@nginx.com 2980Smax.romanov@nginx.com 3080Smax.romanov@nginx.com #define PORT_MMAP_SIZE (PORT_MMAP_HEADER_SIZE + PORT_MMAP_DATA_SIZE) 3180Smax.romanov@nginx.com #define PORT_MMAP_CHUNK_COUNT (PORT_MMAP_DATA_SIZE / PORT_MMAP_CHUNK_SIZE) 3280Smax.romanov@nginx.com 3380Smax.romanov@nginx.com 3480Smax.romanov@nginx.com typedef uint32_t nxt_chunk_id_t; 3580Smax.romanov@nginx.com 3680Smax.romanov@nginx.com typedef nxt_atomic_uint_t nxt_free_map_t; 3780Smax.romanov@nginx.com 3880Smax.romanov@nginx.com #define FREE_BITS (sizeof(nxt_free_map_t) * 8) 3980Smax.romanov@nginx.com 4080Smax.romanov@nginx.com #define FREE_IDX(nchunk) ((nchunk) / FREE_BITS) 4180Smax.romanov@nginx.com 4280Smax.romanov@nginx.com #define FREE_MASK(nchunk) \ 4380Smax.romanov@nginx.com ( 1ULL << ( (nchunk) % FREE_BITS ) ) 4480Smax.romanov@nginx.com 4580Smax.romanov@nginx.com #define MAX_FREE_IDX FREE_IDX(PORT_MMAP_CHUNK_COUNT) 4680Smax.romanov@nginx.com 4780Smax.romanov@nginx.com 4880Smax.romanov@nginx.com /* Mapped at the start of shared memory segment. */ 4980Smax.romanov@nginx.com struct nxt_port_mmap_header_s { 5080Smax.romanov@nginx.com uint32_t id; 51363Smax.romanov@nginx.com nxt_pid_t src_pid; /* For sanity check. */ 52363Smax.romanov@nginx.com nxt_pid_t dst_pid; /* For sanity check. */ 53323Smax.romanov@nginx.com nxt_port_id_t sent_over; 5480Smax.romanov@nginx.com nxt_free_map_t free_map[MAX_FREE_IDX]; 55423Smax.romanov@nginx.com nxt_free_map_t free_map_padding; 56423Smax.romanov@nginx.com nxt_free_map_t free_tracking_map[MAX_FREE_IDX]; 57423Smax.romanov@nginx.com nxt_free_map_t free_tracking_map_padding; 58423Smax.romanov@nginx.com nxt_atomic_t tracking[PORT_MMAP_CHUNK_COUNT]; 5980Smax.romanov@nginx.com }; 6080Smax.romanov@nginx.com 6180Smax.romanov@nginx.com 62365Smax.romanov@nginx.com struct nxt_port_mmap_handler_s { 63365Smax.romanov@nginx.com nxt_port_mmap_header_t *hdr; 64365Smax.romanov@nginx.com nxt_atomic_t use_count; 65365Smax.romanov@nginx.com }; 66365Smax.romanov@nginx.com 6780Smax.romanov@nginx.com /* 6880Smax.romanov@nginx.com * Element of nxt_process_t.incoming/outgoing, shared memory segment 6980Smax.romanov@nginx.com * descriptor. 7080Smax.romanov@nginx.com */ 7180Smax.romanov@nginx.com struct nxt_port_mmap_s { 72365Smax.romanov@nginx.com nxt_port_mmap_handler_t *mmap_handler; 7380Smax.romanov@nginx.com }; 7480Smax.romanov@nginx.com 7580Smax.romanov@nginx.com typedef struct nxt_port_mmap_msg_s nxt_port_mmap_msg_t; 7680Smax.romanov@nginx.com 7780Smax.romanov@nginx.com /* Passed as a second iov chunk when 'mmap' bit in nxt_port_msg_t is 1. */ 7880Smax.romanov@nginx.com struct nxt_port_mmap_msg_s { 7980Smax.romanov@nginx.com uint32_t mmap_id; /* Mmap index in nxt_process_t.outgoing. */ 8080Smax.romanov@nginx.com nxt_chunk_id_t chunk_id; /* Mmap chunk index. */ 8180Smax.romanov@nginx.com uint32_t size; /* Payload data size. */ 8280Smax.romanov@nginx.com }; 8380Smax.romanov@nginx.com 8480Smax.romanov@nginx.com 85423Smax.romanov@nginx.com typedef struct nxt_port_mmap_tracking_msg_s nxt_port_mmap_tracking_msg_t; 8680Smax.romanov@nginx.com 87423Smax.romanov@nginx.com struct nxt_port_mmap_tracking_msg_s { 88423Smax.romanov@nginx.com uint32_t mmap_id; /* Mmap index in nxt_process_t.outgoing. */ 89423Smax.romanov@nginx.com nxt_chunk_id_t tracking_id; /* Tracking index. */ 90423Smax.romanov@nginx.com }; 91423Smax.romanov@nginx.com 92*723Smax.romanov@nginx.com nxt_inline nxt_bool_t 93423Smax.romanov@nginx.com nxt_port_mmap_get_free_chunk(nxt_free_map_t *m, nxt_chunk_id_t *c); 94423Smax.romanov@nginx.com 95423Smax.romanov@nginx.com #define nxt_port_mmap_get_chunk_busy(m, c) \ 96423Smax.romanov@nginx.com ((m[FREE_IDX(c)] & FREE_MASK(c)) == 0) 9780Smax.romanov@nginx.com 9880Smax.romanov@nginx.com nxt_inline void 99423Smax.romanov@nginx.com nxt_port_mmap_set_chunk_busy(nxt_free_map_t *m, nxt_chunk_id_t c); 10080Smax.romanov@nginx.com 10180Smax.romanov@nginx.com nxt_inline nxt_bool_t 102423Smax.romanov@nginx.com nxt_port_mmap_chk_set_chunk_busy(nxt_free_map_t *m, nxt_chunk_id_t c); 10380Smax.romanov@nginx.com 10480Smax.romanov@nginx.com nxt_inline void 105423Smax.romanov@nginx.com nxt_port_mmap_set_chunk_free(nxt_free_map_t *m, nxt_chunk_id_t c); 10680Smax.romanov@nginx.com 10780Smax.romanov@nginx.com nxt_inline nxt_chunk_id_t 10880Smax.romanov@nginx.com nxt_port_mmap_chunk_id(nxt_port_mmap_header_t *hdr, u_char *p) 10980Smax.romanov@nginx.com { 11080Smax.romanov@nginx.com u_char *mm_start; 11180Smax.romanov@nginx.com 11280Smax.romanov@nginx.com mm_start = (u_char *) hdr; 11380Smax.romanov@nginx.com 11480Smax.romanov@nginx.com return ((p - mm_start) - PORT_MMAP_HEADER_SIZE) / PORT_MMAP_CHUNK_SIZE; 11580Smax.romanov@nginx.com } 11680Smax.romanov@nginx.com 11780Smax.romanov@nginx.com 11880Smax.romanov@nginx.com nxt_inline u_char * 11980Smax.romanov@nginx.com nxt_port_mmap_chunk_start(nxt_port_mmap_header_t *hdr, nxt_chunk_id_t c) 12080Smax.romanov@nginx.com { 12180Smax.romanov@nginx.com u_char *mm_start; 12280Smax.romanov@nginx.com 12380Smax.romanov@nginx.com mm_start = (u_char *) hdr; 12480Smax.romanov@nginx.com 12580Smax.romanov@nginx.com return mm_start + PORT_MMAP_HEADER_SIZE + c * PORT_MMAP_CHUNK_SIZE; 12680Smax.romanov@nginx.com } 12780Smax.romanov@nginx.com 12880Smax.romanov@nginx.com 129*723Smax.romanov@nginx.com nxt_inline nxt_bool_t 130423Smax.romanov@nginx.com nxt_port_mmap_get_free_chunk(nxt_free_map_t *m, nxt_chunk_id_t *c) 13180Smax.romanov@nginx.com { 132699Smax.romanov@nginx.com const nxt_free_map_t default_mask = (nxt_free_map_t) -1; 133699Smax.romanov@nginx.com 13480Smax.romanov@nginx.com int ffs; 135699Smax.romanov@nginx.com size_t i, start; 13680Smax.romanov@nginx.com nxt_chunk_id_t chunk; 137699Smax.romanov@nginx.com nxt_free_map_t bits, mask; 138699Smax.romanov@nginx.com 139699Smax.romanov@nginx.com start = FREE_IDX(*c); 140699Smax.romanov@nginx.com mask = default_mask << ((*c) % FREE_BITS); 14180Smax.romanov@nginx.com 142699Smax.romanov@nginx.com for (i = start; i < MAX_FREE_IDX; i++) { 143699Smax.romanov@nginx.com bits = m[i] & mask; 144699Smax.romanov@nginx.com mask = default_mask; 145699Smax.romanov@nginx.com 14680Smax.romanov@nginx.com if (bits == 0) { 14780Smax.romanov@nginx.com continue; 14880Smax.romanov@nginx.com } 14980Smax.romanov@nginx.com 15080Smax.romanov@nginx.com ffs = __builtin_ffsll(bits); 15180Smax.romanov@nginx.com if (ffs != 0) { 15280Smax.romanov@nginx.com chunk = i * FREE_BITS + ffs - 1; 15380Smax.romanov@nginx.com 154423Smax.romanov@nginx.com if (nxt_port_mmap_chk_set_chunk_busy(m, chunk)) { 15580Smax.romanov@nginx.com *c = chunk; 15680Smax.romanov@nginx.com return 1; 15780Smax.romanov@nginx.com } 15880Smax.romanov@nginx.com } 15980Smax.romanov@nginx.com } 16080Smax.romanov@nginx.com 16180Smax.romanov@nginx.com return 0; 16280Smax.romanov@nginx.com } 16380Smax.romanov@nginx.com 16480Smax.romanov@nginx.com 16580Smax.romanov@nginx.com nxt_inline void 166423Smax.romanov@nginx.com nxt_port_mmap_set_chunk_busy(nxt_free_map_t *m, nxt_chunk_id_t c) 16780Smax.romanov@nginx.com { 168423Smax.romanov@nginx.com nxt_atomic_and_fetch(m + FREE_IDX(c), ~FREE_MASK(c)); 16980Smax.romanov@nginx.com } 17080Smax.romanov@nginx.com 17180Smax.romanov@nginx.com 17280Smax.romanov@nginx.com nxt_inline nxt_bool_t 173423Smax.romanov@nginx.com nxt_port_mmap_chk_set_chunk_busy(nxt_free_map_t *m, nxt_chunk_id_t c) 17480Smax.romanov@nginx.com { 17580Smax.romanov@nginx.com nxt_free_map_t *f; 17680Smax.romanov@nginx.com nxt_free_map_t free_val, busy_val; 17780Smax.romanov@nginx.com 178423Smax.romanov@nginx.com f = m + FREE_IDX(c); 17980Smax.romanov@nginx.com 18080Smax.romanov@nginx.com while ( (*f & FREE_MASK(c)) != 0 ) { 18180Smax.romanov@nginx.com 18280Smax.romanov@nginx.com free_val = *f | FREE_MASK(c); 18380Smax.romanov@nginx.com busy_val = free_val & ~FREE_MASK(c); 18480Smax.romanov@nginx.com 18580Smax.romanov@nginx.com if (nxt_atomic_cmp_set(f, free_val, busy_val) != 0) { 18680Smax.romanov@nginx.com return 1; 18780Smax.romanov@nginx.com } 18880Smax.romanov@nginx.com } 18980Smax.romanov@nginx.com 19080Smax.romanov@nginx.com return 0; 19180Smax.romanov@nginx.com } 19280Smax.romanov@nginx.com 19380Smax.romanov@nginx.com 19480Smax.romanov@nginx.com nxt_inline void 195423Smax.romanov@nginx.com nxt_port_mmap_set_chunk_free(nxt_free_map_t *m, nxt_chunk_id_t c) 19680Smax.romanov@nginx.com { 197423Smax.romanov@nginx.com nxt_atomic_or_fetch(m + FREE_IDX(c), FREE_MASK(c)); 19880Smax.romanov@nginx.com } 19980Smax.romanov@nginx.com 20080Smax.romanov@nginx.com 20180Smax.romanov@nginx.com #endif /* _NXT_PORT_MEMORY_INT_H_INCLUDED_ */ 202