Back to home page

Nginx displayed by LXR

Source navigation ]
Diff markup ]
Identifier search ]
general search ]
 
 
Version: nginx-1.13.12 ]​[ nginx-1.12.2 ]​

0001 
0002 /*
0003  * Copyright (C) Igor Sysoev
0004  * Copyright (C) Nginx, Inc.
0005  */
0006 
0007 #include <ngx_config.h>
0008 #include <ngx_core.h>
0009 
0010 
0011 #define NGX_SLAB_PAGE_MASK   3
0012 #define NGX_SLAB_PAGE        0
0013 #define NGX_SLAB_BIG         1
0014 #define NGX_SLAB_EXACT       2
0015 #define NGX_SLAB_SMALL       3
0016 
0017 #if (NGX_PTR_SIZE == 4)
0018 
0019 #define NGX_SLAB_PAGE_FREE   0
0020 #define NGX_SLAB_PAGE_BUSY   0xffffffff
0021 #define NGX_SLAB_PAGE_START  0x80000000
0022 
0023 #define NGX_SLAB_SHIFT_MASK  0x0000000f
0024 #define NGX_SLAB_MAP_MASK    0xffff0000
0025 #define NGX_SLAB_MAP_SHIFT   16
0026 
0027 #define NGX_SLAB_BUSY        0xffffffff
0028 
0029 #else /* (NGX_PTR_SIZE == 8) */
0030 
0031 #define NGX_SLAB_PAGE_FREE   0
0032 #define NGX_SLAB_PAGE_BUSY   0xffffffffffffffff
0033 #define NGX_SLAB_PAGE_START  0x8000000000000000
0034 
0035 #define NGX_SLAB_SHIFT_MASK  0x000000000000000f
0036 #define NGX_SLAB_MAP_MASK    0xffffffff00000000
0037 #define NGX_SLAB_MAP_SHIFT   32
0038 
0039 #define NGX_SLAB_BUSY        0xffffffffffffffff
0040 
0041 #endif
0042 
0043 
0044 #define ngx_slab_slots(pool)                                                  \
0045     (ngx_slab_page_t *) ((u_char *) (pool) + sizeof(ngx_slab_pool_t))
0046 
0047 #define ngx_slab_page_type(page)   ((page)->prev & NGX_SLAB_PAGE_MASK)
0048 
0049 #define ngx_slab_page_prev(page)                                              \
0050     (ngx_slab_page_t *) ((page)->prev & ~NGX_SLAB_PAGE_MASK)
0051 
0052 #define ngx_slab_page_addr(pool, page)                                        \
0053     ((((page) - (pool)->pages) << ngx_pagesize_shift)                         \
0054      + (uintptr_t) (pool)->start)
0055 
0056 
0057 #if (NGX_DEBUG_MALLOC)
0058 
0059 #define ngx_slab_junk(p, size)     ngx_memset(p, 0xA5, size)
0060 
0061 #elif (NGX_HAVE_DEBUG_MALLOC)
0062 
0063 #define ngx_slab_junk(p, size)                                                \
0064     if (ngx_debug_malloc)          ngx_memset(p, 0xA5, size)
0065 
0066 #else
0067 
0068 #define ngx_slab_junk(p, size)
0069 
0070 #endif
0071 
0072 static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool,
0073     ngx_uint_t pages);
0074 static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
0075     ngx_uint_t pages);
0076 static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level,
0077     char *text);
0078 
0079 
0080 static ngx_uint_t  ngx_slab_max_size;
0081 static ngx_uint_t  ngx_slab_exact_size;
0082 static ngx_uint_t  ngx_slab_exact_shift;
0083 
0084 
0085 void
0086 ngx_slab_sizes_init(void)
0087 {
0088     ngx_uint_t  n;
0089 
0090     ngx_slab_max_size = ngx_pagesize / 2;
0091     ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
0092     for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
0093         /* void */
0094     }
0095 }
0096 
0097 
0098 void
0099 ngx_slab_init(ngx_slab_pool_t *pool)
0100 {
0101     u_char           *p;
0102     size_t            size;
0103     ngx_int_t         m;
0104     ngx_uint_t        i, n, pages;
0105     ngx_slab_page_t  *slots, *page;
0106 
0107     pool->min_size = (size_t) 1 << pool->min_shift;
0108 
0109     slots = ngx_slab_slots(pool);
0110 
0111     p = (u_char *) slots;
0112     size = pool->end - p;
0113 
0114     ngx_slab_junk(p, size);
0115 
0116     n = ngx_pagesize_shift - pool->min_shift;
0117 
0118     for (i = 0; i < n; i++) {
0119         /* only "next" is used in list head */
0120         slots[i].slab = 0;
0121         slots[i].next = &slots[i];
0122         slots[i].prev = 0;
0123     }
0124 
0125     p += n * sizeof(ngx_slab_page_t);
0126 
0127     pool->stats = (ngx_slab_stat_t *) p;
0128     ngx_memzero(pool->stats, n * sizeof(ngx_slab_stat_t));
0129 
0130     p += n * sizeof(ngx_slab_stat_t);
0131 
0132     size -= n * (sizeof(ngx_slab_page_t) + sizeof(ngx_slab_stat_t));
0133 
0134     pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t)));
0135 
0136     pool->pages = (ngx_slab_page_t *) p;
0137     ngx_memzero(pool->pages, pages * sizeof(ngx_slab_page_t));
0138 
0139     page = pool->pages;
0140 
0141     /* only "next" is used in list head */
0142     pool->free.slab = 0;
0143     pool->free.next = page;
0144     pool->free.prev = 0;
0145 
0146     page->slab = pages;
0147     page->next = &pool->free;
0148     page->prev = (uintptr_t) &pool->free;
0149 
0150     pool->start = ngx_align_ptr(p + pages * sizeof(ngx_slab_page_t),
0151                                 ngx_pagesize);
0152 
0153     m = pages - (pool->end - pool->start) / ngx_pagesize;
0154     if (m > 0) {
0155         pages -= m;
0156         page->slab = pages;
0157     }
0158 
0159     pool->last = pool->pages + pages;
0160     pool->pfree = pages;
0161 
0162     pool->log_nomem = 1;
0163     pool->log_ctx = &pool->zero;
0164     pool->zero = '\0';
0165 }
0166 
0167 
0168 void *
0169 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
0170 {
0171     void  *p;
0172 
0173     ngx_shmtx_lock(&pool->mutex);
0174 
0175     p = ngx_slab_alloc_locked(pool, size);
0176 
0177     ngx_shmtx_unlock(&pool->mutex);
0178 
0179     return p;
0180 }
0181 
0182 
0183 void *
0184 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size)
0185 {
0186     size_t            s;
0187     uintptr_t         p, m, mask, *bitmap;
0188     ngx_uint_t        i, n, slot, shift, map;
0189     ngx_slab_page_t  *page, *prev, *slots;
0190 
0191     if (size > ngx_slab_max_size) {
0192 
0193         ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
0194                        "slab alloc: %uz", size);
0195 
0196         page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift)
0197                                           + ((size % ngx_pagesize) ? 1 : 0));
0198         if (page) {
0199             p = ngx_slab_page_addr(pool, page);
0200 
0201         } else {
0202             p = 0;
0203         }
0204 
0205         goto done;
0206     }
0207 
0208     if (size > pool->min_size) {
0209         shift = 1;
0210         for (s = size - 1; s >>= 1; shift++) { /* void */ }
0211         slot = shift - pool->min_shift;
0212 
0213     } else {
0214         shift = pool->min_shift;
0215         slot = 0;
0216     }
0217 
0218     pool->stats[slot].reqs++;
0219 
0220     ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
0221                    "slab alloc: %uz slot: %ui", size, slot);
0222 
0223     slots = ngx_slab_slots(pool);
0224     page = slots[slot].next;
0225 
0226     if (page->next != page) {
0227 
0228         if (shift < ngx_slab_exact_shift) {
0229 
0230             bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
0231 
0232             map = (ngx_pagesize >> shift) / (8 * sizeof(uintptr_t));
0233 
0234             for (n = 0; n < map; n++) {
0235 
0236                 if (bitmap[n] != NGX_SLAB_BUSY) {
0237 
0238                     for (m = 1, i = 0; m; m <<= 1, i++) {
0239                         if (bitmap[n] & m) {
0240                             continue;
0241                         }
0242 
0243                         bitmap[n] |= m;
0244 
0245                         i = (n * 8 * sizeof(uintptr_t) + i) << shift;
0246 
0247                         p = (uintptr_t) bitmap + i;
0248 
0249                         pool->stats[slot].used++;
0250 
0251                         if (bitmap[n] == NGX_SLAB_BUSY) {
0252                             for (n = n + 1; n < map; n++) {
0253                                 if (bitmap[n] != NGX_SLAB_BUSY) {
0254                                     goto done;
0255                                 }
0256                             }
0257 
0258                             prev = ngx_slab_page_prev(page);
0259                             prev->next = page->next;
0260                             page->next->prev = page->prev;
0261 
0262                             page->next = NULL;
0263                             page->prev = NGX_SLAB_SMALL;
0264                         }
0265 
0266                         goto done;
0267                     }
0268                 }
0269             }
0270 
0271         } else if (shift == ngx_slab_exact_shift) {
0272 
0273             for (m = 1, i = 0; m; m <<= 1, i++) {
0274                 if (page->slab & m) {
0275                     continue;
0276                 }
0277 
0278                 page->slab |= m;
0279 
0280                 if (page->slab == NGX_SLAB_BUSY) {
0281                     prev = ngx_slab_page_prev(page);
0282                     prev->next = page->next;
0283                     page->next->prev = page->prev;
0284 
0285                     page->next = NULL;
0286                     page->prev = NGX_SLAB_EXACT;
0287                 }
0288 
0289                 p = ngx_slab_page_addr(pool, page) + (i << shift);
0290 
0291                 pool->stats[slot].used++;
0292 
0293                 goto done;
0294             }
0295 
0296         } else { /* shift > ngx_slab_exact_shift */
0297 
0298             mask = ((uintptr_t) 1 << (ngx_pagesize >> shift)) - 1;
0299             mask <<= NGX_SLAB_MAP_SHIFT;
0300 
0301             for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0;
0302                  m & mask;
0303                  m <<= 1, i++)
0304             {
0305                 if (page->slab & m) {
0306                     continue;
0307                 }
0308 
0309                 page->slab |= m;
0310 
0311                 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
0312                     prev = ngx_slab_page_prev(page);
0313                     prev->next = page->next;
0314                     page->next->prev = page->prev;
0315 
0316                     page->next = NULL;
0317                     page->prev = NGX_SLAB_BIG;
0318                 }
0319 
0320                 p = ngx_slab_page_addr(pool, page) + (i << shift);
0321 
0322                 pool->stats[slot].used++;
0323 
0324                 goto done;
0325             }
0326         }
0327 
0328         ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_alloc(): page is busy");
0329         ngx_debug_point();
0330     }
0331 
0332     page = ngx_slab_alloc_pages(pool, 1);
0333 
0334     if (page) {
0335         if (shift < ngx_slab_exact_shift) {
0336             bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
0337 
0338             n = (ngx_pagesize >> shift) / ((1 << shift) * 8);
0339 
0340             if (n == 0) {
0341                 n = 1;
0342             }
0343 
0344             /* "n" elements for bitmap, plus one requested */
0345 
0346             for (i = 0; i < (n + 1) / (8 * sizeof(uintptr_t)); i++) {
0347                 bitmap[i] = NGX_SLAB_BUSY;
0348             }
0349 
0350             m = ((uintptr_t) 1 << ((n + 1) % (8 * sizeof(uintptr_t)))) - 1;
0351             bitmap[i] = m;
0352 
0353             map = (ngx_pagesize >> shift) / (8 * sizeof(uintptr_t));
0354 
0355             for (i = i + 1; i < map; i++) {
0356                 bitmap[i] = 0;
0357             }
0358 
0359             page->slab = shift;
0360             page->next = &slots[slot];
0361             page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
0362 
0363             slots[slot].next = page;
0364 
0365             pool->stats[slot].total += (ngx_pagesize >> shift) - n;
0366 
0367             p = ngx_slab_page_addr(pool, page) + (n << shift);
0368 
0369             pool->stats[slot].used++;
0370 
0371             goto done;
0372 
0373         } else if (shift == ngx_slab_exact_shift) {
0374 
0375             page->slab = 1;
0376             page->next = &slots[slot];
0377             page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
0378 
0379             slots[slot].next = page;
0380 
0381             pool->stats[slot].total += 8 * sizeof(uintptr_t);
0382 
0383             p = ngx_slab_page_addr(pool, page);
0384 
0385             pool->stats[slot].used++;
0386 
0387             goto done;
0388 
0389         } else { /* shift > ngx_slab_exact_shift */
0390 
0391             page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift;
0392             page->next = &slots[slot];
0393             page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
0394 
0395             slots[slot].next = page;
0396 
0397             pool->stats[slot].total += ngx_pagesize >> shift;
0398 
0399             p = ngx_slab_page_addr(pool, page);
0400 
0401             pool->stats[slot].used++;
0402 
0403             goto done;
0404         }
0405     }
0406 
0407     p = 0;
0408 
0409     pool->stats[slot].fails++;
0410 
0411 done:
0412 
0413     ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
0414                    "slab alloc: %p", (void *) p);
0415 
0416     return (void *) p;
0417 }
0418 
0419 
0420 void *
0421 ngx_slab_calloc(ngx_slab_pool_t *pool, size_t size)
0422 {
0423     void  *p;
0424 
0425     ngx_shmtx_lock(&pool->mutex);
0426 
0427     p = ngx_slab_calloc_locked(pool, size);
0428 
0429     ngx_shmtx_unlock(&pool->mutex);
0430 
0431     return p;
0432 }
0433 
0434 
0435 void *
0436 ngx_slab_calloc_locked(ngx_slab_pool_t *pool, size_t size)
0437 {
0438     void  *p;
0439 
0440     p = ngx_slab_alloc_locked(pool, size);
0441     if (p) {
0442         ngx_memzero(p, size);
0443     }
0444 
0445     return p;
0446 }
0447 
0448 
0449 void
0450 ngx_slab_free(ngx_slab_pool_t *pool, void *p)
0451 {
0452     ngx_shmtx_lock(&pool->mutex);
0453 
0454     ngx_slab_free_locked(pool, p);
0455 
0456     ngx_shmtx_unlock(&pool->mutex);
0457 }
0458 
0459 
0460 void
0461 ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p)
0462 {
0463     size_t            size;
0464     uintptr_t         slab, m, *bitmap;
0465     ngx_uint_t        i, n, type, slot, shift, map;
0466     ngx_slab_page_t  *slots, *page;
0467 
0468     ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p);
0469 
0470     if ((u_char *) p < pool->start || (u_char *) p > pool->end) {
0471         ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool");
0472         goto fail;
0473     }
0474 
0475     n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
0476     page = &pool->pages[n];
0477     slab = page->slab;
0478     type = ngx_slab_page_type(page);
0479 
0480     switch (type) {
0481 
0482     case NGX_SLAB_SMALL:
0483 
0484         shift = slab & NGX_SLAB_SHIFT_MASK;
0485         size = (size_t) 1 << shift;
0486 
0487         if ((uintptr_t) p & (size - 1)) {
0488             goto wrong_chunk;
0489         }
0490 
0491         n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift;
0492         m = (uintptr_t) 1 << (n % (8 * sizeof(uintptr_t)));
0493         n /= 8 * sizeof(uintptr_t);
0494         bitmap = (uintptr_t *)
0495                              ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1));
0496 
0497         if (bitmap[n] & m) {
0498             slot = shift - pool->min_shift;
0499 
0500             if (page->next == NULL) {
0501                 slots = ngx_slab_slots(pool);
0502 
0503                 page->next = slots[slot].next;
0504                 slots[slot].next = page;
0505 
0506                 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
0507                 page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL;
0508             }
0509 
0510             bitmap[n] &= ~m;
0511 
0512             n = (ngx_pagesize >> shift) / ((1 << shift) * 8);
0513 
0514             if (n == 0) {
0515                 n = 1;
0516             }
0517 
0518             i = n / (8 * sizeof(uintptr_t));
0519             m = ((uintptr_t) 1 << (n % (8 * sizeof(uintptr_t)))) - 1;
0520 
0521             if (bitmap[i] & ~m) {
0522                 goto done;
0523             }
0524 
0525             map = (ngx_pagesize >> shift) / (8 * sizeof(uintptr_t));
0526 
0527             for (i = i + 1; i < map; i++) {
0528                 if (bitmap[i]) {
0529                     goto done;
0530                 }
0531             }
0532 
0533             ngx_slab_free_pages(pool, page, 1);
0534 
0535             pool->stats[slot].total -= (ngx_pagesize >> shift) - n;
0536 
0537             goto done;
0538         }
0539 
0540         goto chunk_already_free;
0541 
0542     case NGX_SLAB_EXACT:
0543 
0544         m = (uintptr_t) 1 <<
0545                 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift);
0546         size = ngx_slab_exact_size;
0547 
0548         if ((uintptr_t) p & (size - 1)) {
0549             goto wrong_chunk;
0550         }
0551 
0552         if (slab & m) {
0553             slot = ngx_slab_exact_shift - pool->min_shift;
0554 
0555             if (slab == NGX_SLAB_BUSY) {
0556                 slots = ngx_slab_slots(pool);
0557 
0558                 page->next = slots[slot].next;
0559                 slots[slot].next = page;
0560 
0561                 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
0562                 page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT;
0563             }
0564 
0565             page->slab &= ~m;
0566 
0567             if (page->slab) {
0568                 goto done;
0569             }
0570 
0571             ngx_slab_free_pages(pool, page, 1);
0572 
0573             pool->stats[slot].total -= 8 * sizeof(uintptr_t);
0574 
0575             goto done;
0576         }
0577 
0578         goto chunk_already_free;
0579 
0580     case NGX_SLAB_BIG:
0581 
0582         shift = slab & NGX_SLAB_SHIFT_MASK;
0583         size = (size_t) 1 << shift;
0584 
0585         if ((uintptr_t) p & (size - 1)) {
0586             goto wrong_chunk;
0587         }
0588 
0589         m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift)
0590                               + NGX_SLAB_MAP_SHIFT);
0591 
0592         if (slab & m) {
0593             slot = shift - pool->min_shift;
0594 
0595             if (page->next == NULL) {
0596                 slots = ngx_slab_slots(pool);
0597 
0598                 page->next = slots[slot].next;
0599                 slots[slot].next = page;
0600 
0601                 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
0602                 page->next->prev = (uintptr_t) page | NGX_SLAB_BIG;
0603             }
0604 
0605             page->slab &= ~m;
0606 
0607             if (page->slab & NGX_SLAB_MAP_MASK) {
0608                 goto done;
0609             }
0610 
0611             ngx_slab_free_pages(pool, page, 1);
0612 
0613             pool->stats[slot].total -= ngx_pagesize >> shift;
0614 
0615             goto done;
0616         }
0617 
0618         goto chunk_already_free;
0619 
0620     case NGX_SLAB_PAGE:
0621 
0622         if ((uintptr_t) p & (ngx_pagesize - 1)) {
0623             goto wrong_chunk;
0624         }
0625 
0626         if (!(slab & NGX_SLAB_PAGE_START)) {
0627             ngx_slab_error(pool, NGX_LOG_ALERT,
0628                            "ngx_slab_free(): page is already free");
0629             goto fail;
0630         }
0631 
0632         if (slab == NGX_SLAB_PAGE_BUSY) {
0633             ngx_slab_error(pool, NGX_LOG_ALERT,
0634                            "ngx_slab_free(): pointer to wrong page");
0635             goto fail;
0636         }
0637 
0638         n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
0639         size = slab & ~NGX_SLAB_PAGE_START;
0640 
0641         ngx_slab_free_pages(pool, &pool->pages[n], size);
0642 
0643         ngx_slab_junk(p, size << ngx_pagesize_shift);
0644 
0645         return;
0646     }
0647 
0648     /* not reached */
0649 
0650     return;
0651 
0652 done:
0653 
0654     pool->stats[slot].used--;
0655 
0656     ngx_slab_junk(p, size);
0657 
0658     return;
0659 
0660 wrong_chunk:
0661 
0662     ngx_slab_error(pool, NGX_LOG_ALERT,
0663                    "ngx_slab_free(): pointer to wrong chunk");
0664 
0665     goto fail;
0666 
0667 chunk_already_free:
0668 
0669     ngx_slab_error(pool, NGX_LOG_ALERT,
0670                    "ngx_slab_free(): chunk is already free");
0671 
0672 fail:
0673 
0674     return;
0675 }
0676 
0677 
0678 static ngx_slab_page_t *
0679 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages)
0680 {
0681     ngx_slab_page_t  *page, *p;
0682 
0683     for (page = pool->free.next; page != &pool->free; page = page->next) {
0684 
0685         if (page->slab >= pages) {
0686 
0687             if (page->slab > pages) {
0688                 page[page->slab - 1].prev = (uintptr_t) &page[pages];
0689 
0690                 page[pages].slab = page->slab - pages;
0691                 page[pages].next = page->next;
0692                 page[pages].prev = page->prev;
0693 
0694                 p = (ngx_slab_page_t *) page->prev;
0695                 p->next = &page[pages];
0696                 page->next->prev = (uintptr_t) &page[pages];
0697 
0698             } else {
0699                 p = (ngx_slab_page_t *) page->prev;
0700                 p->next = page->next;
0701                 page->next->prev = page->prev;
0702             }
0703 
0704             page->slab = pages | NGX_SLAB_PAGE_START;
0705             page->next = NULL;
0706             page->prev = NGX_SLAB_PAGE;
0707 
0708             pool->pfree -= pages;
0709 
0710             if (--pages == 0) {
0711                 return page;
0712             }
0713 
0714             for (p = page + 1; pages; pages--) {
0715                 p->slab = NGX_SLAB_PAGE_BUSY;
0716                 p->next = NULL;
0717                 p->prev = NGX_SLAB_PAGE;
0718                 p++;
0719             }
0720 
0721             return page;
0722         }
0723     }
0724 
0725     if (pool->log_nomem) {
0726         ngx_slab_error(pool, NGX_LOG_CRIT,
0727                        "ngx_slab_alloc() failed: no memory");
0728     }
0729 
0730     return NULL;
0731 }
0732 
0733 
0734 static void
0735 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
0736     ngx_uint_t pages)
0737 {
0738     ngx_slab_page_t  *prev, *join;
0739 
0740     pool->pfree += pages;
0741 
0742     page->slab = pages--;
0743 
0744     if (pages) {
0745         ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
0746     }
0747 
0748     if (page->next) {
0749         prev = ngx_slab_page_prev(page);
0750         prev->next = page->next;
0751         page->next->prev = page->prev;
0752     }
0753 
0754     join = page + page->slab;
0755 
0756     if (join < pool->last) {
0757 
0758         if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
0759 
0760             if (join->next != NULL) {
0761                 pages += join->slab;
0762                 page->slab += join->slab;
0763 
0764                 prev = ngx_slab_page_prev(join);
0765                 prev->next = join->next;
0766                 join->next->prev = join->prev;
0767 
0768                 join->slab = NGX_SLAB_PAGE_FREE;
0769                 join->next = NULL;
0770                 join->prev = NGX_SLAB_PAGE;
0771             }
0772         }
0773     }
0774 
0775     if (page > pool->pages) {
0776         join = page - 1;
0777 
0778         if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
0779 
0780             if (join->slab == NGX_SLAB_PAGE_FREE) {
0781                 join = ngx_slab_page_prev(join);
0782             }
0783 
0784             if (join->next != NULL) {
0785                 pages += join->slab;
0786                 join->slab += page->slab;
0787 
0788                 prev = ngx_slab_page_prev(join);
0789                 prev->next = join->next;
0790                 join->next->prev = join->prev;
0791 
0792                 page->slab = NGX_SLAB_PAGE_FREE;
0793                 page->next = NULL;
0794                 page->prev = NGX_SLAB_PAGE;
0795 
0796                 page = join;
0797             }
0798         }
0799     }
0800 
0801     if (pages) {
0802         page[pages].prev = (uintptr_t) page;
0803     }
0804 
0805     page->prev = (uintptr_t) &pool->free;
0806     page->next = pool->free.next;
0807 
0808     page->next->prev = (uintptr_t) page;
0809 
0810     pool->free.next = page;
0811 }
0812 
0813 
0814 static void
0815 ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, char *text)
0816 {
0817     ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx);
0818 }