Back to home page

Nginx displayed by LXR

Source navigation ]
Diff markup ]
Identifier search ]
general search ]
 
 
Version: nginx-1.13.12 ]​[ nginx-1.12.2 ]​

0001 
0002 /*
0003  * Copyright (C) Igor Sysoev
0004  * Copyright (C) Nginx, Inc.
0005  */
0006 
0007 #include <ngx_config.h>
0008 #include <ngx_core.h>
0009 
0010 
0011 #define NGX_SLAB_PAGE_MASK   3
0012 #define NGX_SLAB_PAGE        0
0013 #define NGX_SLAB_BIG         1
0014 #define NGX_SLAB_EXACT       2
0015 #define NGX_SLAB_SMALL       3
0016 
0017 #if (NGX_PTR_SIZE == 4)
0018 
0019 #define NGX_SLAB_PAGE_FREE   0
0020 #define NGX_SLAB_PAGE_BUSY   0xffffffff
0021 #define NGX_SLAB_PAGE_START  0x80000000
0022 
0023 #define NGX_SLAB_SHIFT_MASK  0x0000000f
0024 #define NGX_SLAB_MAP_MASK    0xffff0000
0025 #define NGX_SLAB_MAP_SHIFT   16
0026 
0027 #define NGX_SLAB_BUSY        0xffffffff
0028 
0029 #else /* (NGX_PTR_SIZE == 8) */
0030 
0031 #define NGX_SLAB_PAGE_FREE   0
0032 #define NGX_SLAB_PAGE_BUSY   0xffffffffffffffff
0033 #define NGX_SLAB_PAGE_START  0x8000000000000000
0034 
0035 #define NGX_SLAB_SHIFT_MASK  0x000000000000000f
0036 #define NGX_SLAB_MAP_MASK    0xffffffff00000000
0037 #define NGX_SLAB_MAP_SHIFT   32
0038 
0039 #define NGX_SLAB_BUSY        0xffffffffffffffff
0040 
0041 #endif
0042 
0043 
0044 #define ngx_slab_slots(pool)                                                  \
0045     (ngx_slab_page_t *) ((u_char *) (pool) + sizeof(ngx_slab_pool_t))
0046 
0047 #define ngx_slab_page_type(page)   ((page)->prev & NGX_SLAB_PAGE_MASK)
0048 
0049 #define ngx_slab_page_prev(page)                                              \
0050     (ngx_slab_page_t *) ((page)->prev & ~NGX_SLAB_PAGE_MASK)
0051 
0052 #define ngx_slab_page_addr(pool, page)                                        \
0053     ((((page) - (pool)->pages) << ngx_pagesize_shift)                         \
0054      + (uintptr_t) (pool)->start)
0055 
0056 
0057 #if (NGX_DEBUG_MALLOC)
0058 
0059 #define ngx_slab_junk(p, size)     ngx_memset(p, 0xA5, size)
0060 
0061 #elif (NGX_HAVE_DEBUG_MALLOC)
0062 
0063 #define ngx_slab_junk(p, size)                                                \
0064     if (ngx_debug_malloc)          ngx_memset(p, 0xA5, size)
0065 
0066 #else
0067 
0068 #define ngx_slab_junk(p, size)
0069 
0070 #endif
0071 
0072 static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool,
0073     ngx_uint_t pages);
0074 static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
0075     ngx_uint_t pages);
0076 static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level,
0077     char *text);
0078 
0079 
0080 static ngx_uint_t  ngx_slab_max_size;
0081 static ngx_uint_t  ngx_slab_exact_size;
0082 static ngx_uint_t  ngx_slab_exact_shift;
0083 
0084 
0085 void
0086 ngx_slab_init(ngx_slab_pool_t *pool)
0087 {
0088     u_char           *p;
0089     size_t            size;
0090     ngx_int_t         m;
0091     ngx_uint_t        i, n, pages;
0092     ngx_slab_page_t  *slots, *page;
0093 
0094     /* STUB */
0095     if (ngx_slab_max_size == 0) {
0096         ngx_slab_max_size = ngx_pagesize / 2;
0097         ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
0098         for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
0099             /* void */
0100         }
0101     }
0102     /**/
0103 
0104     pool->min_size = (size_t) 1 << pool->min_shift;
0105 
0106     slots = ngx_slab_slots(pool);
0107 
0108     p = (u_char *) slots;
0109     size = pool->end - p;
0110 
0111     ngx_slab_junk(p, size);
0112 
0113     n = ngx_pagesize_shift - pool->min_shift;
0114 
0115     for (i = 0; i < n; i++) {
0116         /* only "next" is used in list head */
0117         slots[i].slab = 0;
0118         slots[i].next = &slots[i];
0119         slots[i].prev = 0;
0120     }
0121 
0122     p += n * sizeof(ngx_slab_page_t);
0123 
0124     pool->stats = (ngx_slab_stat_t *) p;
0125     ngx_memzero(pool->stats, n * sizeof(ngx_slab_stat_t));
0126 
0127     p += n * sizeof(ngx_slab_stat_t);
0128 
0129     size -= n * (sizeof(ngx_slab_page_t) + sizeof(ngx_slab_stat_t));
0130 
0131     pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t)));
0132 
0133     pool->pages = (ngx_slab_page_t *) p;
0134     ngx_memzero(pool->pages, pages * sizeof(ngx_slab_page_t));
0135 
0136     page = pool->pages;
0137 
0138     /* only "next" is used in list head */
0139     pool->free.slab = 0;
0140     pool->free.next = page;
0141     pool->free.prev = 0;
0142 
0143     page->slab = pages;
0144     page->next = &pool->free;
0145     page->prev = (uintptr_t) &pool->free;
0146 
0147     pool->start = ngx_align_ptr(p + pages * sizeof(ngx_slab_page_t),
0148                                 ngx_pagesize);
0149 
0150     m = pages - (pool->end - pool->start) / ngx_pagesize;
0151     if (m > 0) {
0152         pages -= m;
0153         page->slab = pages;
0154     }
0155 
0156     pool->last = pool->pages + pages;
0157     pool->pfree = pages;
0158 
0159     pool->log_nomem = 1;
0160     pool->log_ctx = &pool->zero;
0161     pool->zero = '\0';
0162 }
0163 
0164 
0165 void *
0166 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
0167 {
0168     void  *p;
0169 
0170     ngx_shmtx_lock(&pool->mutex);
0171 
0172     p = ngx_slab_alloc_locked(pool, size);
0173 
0174     ngx_shmtx_unlock(&pool->mutex);
0175 
0176     return p;
0177 }
0178 
0179 
0180 void *
0181 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size)
0182 {
0183     size_t            s;
0184     uintptr_t         p, n, m, mask, *bitmap;
0185     ngx_uint_t        i, slot, shift, map;
0186     ngx_slab_page_t  *page, *prev, *slots;
0187 
0188     if (size > ngx_slab_max_size) {
0189 
0190         ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
0191                        "slab alloc: %uz", size);
0192 
0193         page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift)
0194                                           + ((size % ngx_pagesize) ? 1 : 0));
0195         if (page) {
0196             p = ngx_slab_page_addr(pool, page);
0197 
0198         } else {
0199             p = 0;
0200         }
0201 
0202         goto done;
0203     }
0204 
0205     if (size > pool->min_size) {
0206         shift = 1;
0207         for (s = size - 1; s >>= 1; shift++) { /* void */ }
0208         slot = shift - pool->min_shift;
0209 
0210     } else {
0211         shift = pool->min_shift;
0212         slot = 0;
0213     }
0214 
0215     pool->stats[slot].reqs++;
0216 
0217     ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
0218                    "slab alloc: %uz slot: %ui", size, slot);
0219 
0220     slots = ngx_slab_slots(pool);
0221     page = slots[slot].next;
0222 
0223     if (page->next != page) {
0224 
0225         if (shift < ngx_slab_exact_shift) {
0226 
0227             bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
0228 
0229             map = (ngx_pagesize >> shift) / (sizeof(uintptr_t) * 8);
0230 
0231             for (n = 0; n < map; n++) {
0232 
0233                 if (bitmap[n] != NGX_SLAB_BUSY) {
0234 
0235                     for (m = 1, i = 0; m; m <<= 1, i++) {
0236                         if (bitmap[n] & m) {
0237                             continue;
0238                         }
0239 
0240                         bitmap[n] |= m;
0241 
0242                         i = (n * sizeof(uintptr_t) * 8 + i) << shift;
0243 
0244                         p = (uintptr_t) bitmap + i;
0245 
0246                         pool->stats[slot].used++;
0247 
0248                         if (bitmap[n] == NGX_SLAB_BUSY) {
0249                             for (n = n + 1; n < map; n++) {
0250                                 if (bitmap[n] != NGX_SLAB_BUSY) {
0251                                     goto done;
0252                                 }
0253                             }
0254 
0255                             prev = ngx_slab_page_prev(page);
0256                             prev->next = page->next;
0257                             page->next->prev = page->prev;
0258 
0259                             page->next = NULL;
0260                             page->prev = NGX_SLAB_SMALL;
0261                         }
0262 
0263                         goto done;
0264                     }
0265                 }
0266             }
0267 
0268         } else if (shift == ngx_slab_exact_shift) {
0269 
0270             for (m = 1, i = 0; m; m <<= 1, i++) {
0271                 if (page->slab & m) {
0272                     continue;
0273                 }
0274 
0275                 page->slab |= m;
0276 
0277                 if (page->slab == NGX_SLAB_BUSY) {
0278                     prev = ngx_slab_page_prev(page);
0279                     prev->next = page->next;
0280                     page->next->prev = page->prev;
0281 
0282                     page->next = NULL;
0283                     page->prev = NGX_SLAB_EXACT;
0284                 }
0285 
0286                 p = ngx_slab_page_addr(pool, page) + (i << shift);
0287 
0288                 pool->stats[slot].used++;
0289 
0290                 goto done;
0291             }
0292 
0293         } else { /* shift > ngx_slab_exact_shift */
0294 
0295             mask = ((uintptr_t) 1 << (ngx_pagesize >> shift)) - 1;
0296             mask <<= NGX_SLAB_MAP_SHIFT;
0297 
0298             for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0;
0299                  m & mask;
0300                  m <<= 1, i++)
0301             {
0302                 if (page->slab & m) {
0303                     continue;
0304                 }
0305 
0306                 page->slab |= m;
0307 
0308                 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
0309                     prev = ngx_slab_page_prev(page);
0310                     prev->next = page->next;
0311                     page->next->prev = page->prev;
0312 
0313                     page->next = NULL;
0314                     page->prev = NGX_SLAB_BIG;
0315                 }
0316 
0317                 p = ngx_slab_page_addr(pool, page) + (i << shift);
0318 
0319                 pool->stats[slot].used++;
0320 
0321                 goto done;
0322             }
0323         }
0324 
0325         ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_alloc(): page is busy");
0326         ngx_debug_point();
0327     }
0328 
0329     page = ngx_slab_alloc_pages(pool, 1);
0330 
0331     if (page) {
0332         if (shift < ngx_slab_exact_shift) {
0333             bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
0334 
0335             n = (ngx_pagesize >> shift) / ((1 << shift) * 8);
0336 
0337             if (n == 0) {
0338                 n = 1;
0339             }
0340 
0341             /* "n" elements for bitmap, plus one requested */
0342             bitmap[0] = ((uintptr_t) 2 << n) - 1;
0343 
0344             map = (ngx_pagesize >> shift) / (sizeof(uintptr_t) * 8);
0345 
0346             for (i = 1; i < map; i++) {
0347                 bitmap[i] = 0;
0348             }
0349 
0350             page->slab = shift;
0351             page->next = &slots[slot];
0352             page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
0353 
0354             slots[slot].next = page;
0355 
0356             pool->stats[slot].total += (ngx_pagesize >> shift) - n;
0357 
0358             p = ngx_slab_page_addr(pool, page) + (n << shift);
0359 
0360             pool->stats[slot].used++;
0361 
0362             goto done;
0363 
0364         } else if (shift == ngx_slab_exact_shift) {
0365 
0366             page->slab = 1;
0367             page->next = &slots[slot];
0368             page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
0369 
0370             slots[slot].next = page;
0371 
0372             pool->stats[slot].total += sizeof(uintptr_t) * 8;
0373 
0374             p = ngx_slab_page_addr(pool, page);
0375 
0376             pool->stats[slot].used++;
0377 
0378             goto done;
0379 
0380         } else { /* shift > ngx_slab_exact_shift */
0381 
0382             page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift;
0383             page->next = &slots[slot];
0384             page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
0385 
0386             slots[slot].next = page;
0387 
0388             pool->stats[slot].total += ngx_pagesize >> shift;
0389 
0390             p = ngx_slab_page_addr(pool, page);
0391 
0392             pool->stats[slot].used++;
0393 
0394             goto done;
0395         }
0396     }
0397 
0398     p = 0;
0399 
0400     pool->stats[slot].fails++;
0401 
0402 done:
0403 
0404     ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
0405                    "slab alloc: %p", (void *) p);
0406 
0407     return (void *) p;
0408 }
0409 
0410 
0411 void *
0412 ngx_slab_calloc(ngx_slab_pool_t *pool, size_t size)
0413 {
0414     void  *p;
0415 
0416     ngx_shmtx_lock(&pool->mutex);
0417 
0418     p = ngx_slab_calloc_locked(pool, size);
0419 
0420     ngx_shmtx_unlock(&pool->mutex);
0421 
0422     return p;
0423 }
0424 
0425 
0426 void *
0427 ngx_slab_calloc_locked(ngx_slab_pool_t *pool, size_t size)
0428 {
0429     void  *p;
0430 
0431     p = ngx_slab_alloc_locked(pool, size);
0432     if (p) {
0433         ngx_memzero(p, size);
0434     }
0435 
0436     return p;
0437 }
0438 
0439 
0440 void
0441 ngx_slab_free(ngx_slab_pool_t *pool, void *p)
0442 {
0443     ngx_shmtx_lock(&pool->mutex);
0444 
0445     ngx_slab_free_locked(pool, p);
0446 
0447     ngx_shmtx_unlock(&pool->mutex);
0448 }
0449 
0450 
0451 void
0452 ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p)
0453 {
0454     size_t            size;
0455     uintptr_t         slab, m, *bitmap;
0456     ngx_uint_t        i, n, type, slot, shift, map;
0457     ngx_slab_page_t  *slots, *page;
0458 
0459     ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p);
0460 
0461     if ((u_char *) p < pool->start || (u_char *) p > pool->end) {
0462         ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool");
0463         goto fail;
0464     }
0465 
0466     n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
0467     page = &pool->pages[n];
0468     slab = page->slab;
0469     type = ngx_slab_page_type(page);
0470 
0471     switch (type) {
0472 
0473     case NGX_SLAB_SMALL:
0474 
0475         shift = slab & NGX_SLAB_SHIFT_MASK;
0476         size = (size_t) 1 << shift;
0477 
0478         if ((uintptr_t) p & (size - 1)) {
0479             goto wrong_chunk;
0480         }
0481 
0482         n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift;
0483         m = (uintptr_t) 1 << (n % (sizeof(uintptr_t) * 8));
0484         n /= sizeof(uintptr_t) * 8;
0485         bitmap = (uintptr_t *)
0486                              ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1));
0487 
0488         if (bitmap[n] & m) {
0489             slot = shift - pool->min_shift;
0490 
0491             if (page->next == NULL) {
0492                 slots = ngx_slab_slots(pool);
0493 
0494                 page->next = slots[slot].next;
0495                 slots[slot].next = page;
0496 
0497                 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
0498                 page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL;
0499             }
0500 
0501             bitmap[n] &= ~m;
0502 
0503             n = (ngx_pagesize >> shift) / ((1 << shift) * 8);
0504 
0505             if (n == 0) {
0506                 n = 1;
0507             }
0508 
0509             if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) {
0510                 goto done;
0511             }
0512 
0513             map = (ngx_pagesize >> shift) / (sizeof(uintptr_t) * 8);
0514 
0515             for (i = 1; i < map; i++) {
0516                 if (bitmap[i]) {
0517                     goto done;
0518                 }
0519             }
0520 
0521             ngx_slab_free_pages(pool, page, 1);
0522 
0523             pool->stats[slot].total -= (ngx_pagesize >> shift) - n;
0524 
0525             goto done;
0526         }
0527 
0528         goto chunk_already_free;
0529 
0530     case NGX_SLAB_EXACT:
0531 
0532         m = (uintptr_t) 1 <<
0533                 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift);
0534         size = ngx_slab_exact_size;
0535 
0536         if ((uintptr_t) p & (size - 1)) {
0537             goto wrong_chunk;
0538         }
0539 
0540         if (slab & m) {
0541             slot = ngx_slab_exact_shift - pool->min_shift;
0542 
0543             if (slab == NGX_SLAB_BUSY) {
0544                 slots = ngx_slab_slots(pool);
0545 
0546                 page->next = slots[slot].next;
0547                 slots[slot].next = page;
0548 
0549                 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
0550                 page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT;
0551             }
0552 
0553             page->slab &= ~m;
0554 
0555             if (page->slab) {
0556                 goto done;
0557             }
0558 
0559             ngx_slab_free_pages(pool, page, 1);
0560 
0561             pool->stats[slot].total -= sizeof(uintptr_t) * 8;
0562 
0563             goto done;
0564         }
0565 
0566         goto chunk_already_free;
0567 
0568     case NGX_SLAB_BIG:
0569 
0570         shift = slab & NGX_SLAB_SHIFT_MASK;
0571         size = (size_t) 1 << shift;
0572 
0573         if ((uintptr_t) p & (size - 1)) {
0574             goto wrong_chunk;
0575         }
0576 
0577         m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift)
0578                               + NGX_SLAB_MAP_SHIFT);
0579 
0580         if (slab & m) {
0581             slot = shift - pool->min_shift;
0582 
0583             if (page->next == NULL) {
0584                 slots = ngx_slab_slots(pool);
0585 
0586                 page->next = slots[slot].next;
0587                 slots[slot].next = page;
0588 
0589                 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
0590                 page->next->prev = (uintptr_t) page | NGX_SLAB_BIG;
0591             }
0592 
0593             page->slab &= ~m;
0594 
0595             if (page->slab & NGX_SLAB_MAP_MASK) {
0596                 goto done;
0597             }
0598 
0599             ngx_slab_free_pages(pool, page, 1);
0600 
0601             pool->stats[slot].total -= ngx_pagesize >> shift;
0602 
0603             goto done;
0604         }
0605 
0606         goto chunk_already_free;
0607 
0608     case NGX_SLAB_PAGE:
0609 
0610         if ((uintptr_t) p & (ngx_pagesize - 1)) {
0611             goto wrong_chunk;
0612         }
0613 
0614         if (!(slab & NGX_SLAB_PAGE_START)) {
0615             ngx_slab_error(pool, NGX_LOG_ALERT,
0616                            "ngx_slab_free(): page is already free");
0617             goto fail;
0618         }
0619 
0620         if (slab == NGX_SLAB_PAGE_BUSY) {
0621             ngx_slab_error(pool, NGX_LOG_ALERT,
0622                            "ngx_slab_free(): pointer to wrong page");
0623             goto fail;
0624         }
0625 
0626         n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
0627         size = slab & ~NGX_SLAB_PAGE_START;
0628 
0629         ngx_slab_free_pages(pool, &pool->pages[n], size);
0630 
0631         ngx_slab_junk(p, size << ngx_pagesize_shift);
0632 
0633         return;
0634     }
0635 
0636     /* not reached */
0637 
0638     return;
0639 
0640 done:
0641 
0642     pool->stats[slot].used--;
0643 
0644     ngx_slab_junk(p, size);
0645 
0646     return;
0647 
0648 wrong_chunk:
0649 
0650     ngx_slab_error(pool, NGX_LOG_ALERT,
0651                    "ngx_slab_free(): pointer to wrong chunk");
0652 
0653     goto fail;
0654 
0655 chunk_already_free:
0656 
0657     ngx_slab_error(pool, NGX_LOG_ALERT,
0658                    "ngx_slab_free(): chunk is already free");
0659 
0660 fail:
0661 
0662     return;
0663 }
0664 
0665 
0666 static ngx_slab_page_t *
0667 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages)
0668 {
0669     ngx_slab_page_t  *page, *p;
0670 
0671     for (page = pool->free.next; page != &pool->free; page = page->next) {
0672 
0673         if (page->slab >= pages) {
0674 
0675             if (page->slab > pages) {
0676                 page[page->slab - 1].prev = (uintptr_t) &page[pages];
0677 
0678                 page[pages].slab = page->slab - pages;
0679                 page[pages].next = page->next;
0680                 page[pages].prev = page->prev;
0681 
0682                 p = (ngx_slab_page_t *) page->prev;
0683                 p->next = &page[pages];
0684                 page->next->prev = (uintptr_t) &page[pages];
0685 
0686             } else {
0687                 p = (ngx_slab_page_t *) page->prev;
0688                 p->next = page->next;
0689                 page->next->prev = page->prev;
0690             }
0691 
0692             page->slab = pages | NGX_SLAB_PAGE_START;
0693             page->next = NULL;
0694             page->prev = NGX_SLAB_PAGE;
0695 
0696             pool->pfree -= pages;
0697 
0698             if (--pages == 0) {
0699                 return page;
0700             }
0701 
0702             for (p = page + 1; pages; pages--) {
0703                 p->slab = NGX_SLAB_PAGE_BUSY;
0704                 p->next = NULL;
0705                 p->prev = NGX_SLAB_PAGE;
0706                 p++;
0707             }
0708 
0709             return page;
0710         }
0711     }
0712 
0713     if (pool->log_nomem) {
0714         ngx_slab_error(pool, NGX_LOG_CRIT,
0715                        "ngx_slab_alloc() failed: no memory");
0716     }
0717 
0718     return NULL;
0719 }
0720 
0721 
0722 static void
0723 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
0724     ngx_uint_t pages)
0725 {
0726     ngx_slab_page_t  *prev, *join;
0727 
0728     pool->pfree += pages;
0729 
0730     page->slab = pages--;
0731 
0732     if (pages) {
0733         ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
0734     }
0735 
0736     if (page->next) {
0737         prev = ngx_slab_page_prev(page);
0738         prev->next = page->next;
0739         page->next->prev = page->prev;
0740     }
0741 
0742     join = page + page->slab;
0743 
0744     if (join < pool->last) {
0745 
0746         if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
0747 
0748             if (join->next != NULL) {
0749                 pages += join->slab;
0750                 page->slab += join->slab;
0751 
0752                 prev = ngx_slab_page_prev(join);
0753                 prev->next = join->next;
0754                 join->next->prev = join->prev;
0755 
0756                 join->slab = NGX_SLAB_PAGE_FREE;
0757                 join->next = NULL;
0758                 join->prev = NGX_SLAB_PAGE;
0759             }
0760         }
0761     }
0762 
0763     if (page > pool->pages) {
0764         join = page - 1;
0765 
0766         if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
0767 
0768             if (join->slab == NGX_SLAB_PAGE_FREE) {
0769                 join = ngx_slab_page_prev(join);
0770             }
0771 
0772             if (join->next != NULL) {
0773                 pages += join->slab;
0774                 join->slab += page->slab;
0775 
0776                 prev = ngx_slab_page_prev(join);
0777                 prev->next = join->next;
0778                 join->next->prev = join->prev;
0779 
0780                 page->slab = NGX_SLAB_PAGE_FREE;
0781                 page->next = NULL;
0782                 page->prev = NGX_SLAB_PAGE;
0783 
0784                 page = join;
0785             }
0786         }
0787     }
0788 
0789     if (pages) {
0790         page[pages].prev = (uintptr_t) page;
0791     }
0792 
0793     page->prev = (uintptr_t) &pool->free;
0794     page->next = pool->free.next;
0795 
0796     page->next->prev = (uintptr_t) page;
0797 
0798     pool->free.next = page;
0799 }
0800 
0801 
0802 static void
0803 ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, char *text)
0804 {
0805     ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx);
0806 }