Back to home page

Nginx displayed by LXR

Source navigation ]
Diff markup ]
Identifier search ]
general search ]
 
 
Version: nginx-1.13.12 ]​[ nginx-1.12.2 ]​

0001 
0002 /*
0003  * Copyright (C) Igor Sysoev
0004  * Copyright (C) Nginx, Inc.
0005  */
0006 
0007 
0008 #include <ngx_config.h>
0009 #include <ngx_core.h>
0010 #include <ngx_event.h>
0011 
0012 
0013 #if 0
0014 #define NGX_SENDFILE_LIMIT  4096
0015 #endif
0016 
0017 /*
0018  * When DIRECTIO is enabled FreeBSD, Solaris, and MacOSX read directly
0019  * to an application memory from a device if parameters are aligned
0020  * to device sector boundary (512 bytes).  They fallback to usual read
0021  * operation if the parameters are not aligned.
0022  * Linux allows DIRECTIO only if the parameters are aligned to a filesystem
0023  * sector boundary, otherwise it returns EINVAL.  The sector size is
0024  * usually 512 bytes, however, on XFS it may be 4096 bytes.
0025  */
0026 
0027 #define NGX_NONE            1
0028 
0029 
0030 static ngx_inline ngx_int_t
0031     ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
0032 #if (NGX_HAVE_AIO_SENDFILE)
0033 static ngx_int_t ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx,
0034     ngx_file_t *file);
0035 #endif
0036 static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
0037     ngx_chain_t **chain, ngx_chain_t *in);
0038 static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx,
0039     off_t bsize);
0040 static ngx_int_t ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx,
0041     off_t bsize);
0042 static ngx_int_t ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx);
0043 
0044 
0045 ngx_int_t
0046 ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in)
0047 {
0048     off_t         bsize;
0049     ngx_int_t     rc, last;
0050     ngx_chain_t  *cl, *out, **last_out;
0051 
0052     if (ctx->in == NULL && ctx->busy == NULL
0053 #if (NGX_HAVE_FILE_AIO || NGX_THREADS)
0054         && !ctx->aio
0055 #endif
0056        )
0057     {
0058         /*
0059          * the short path for the case when the ctx->in and ctx->busy chains
0060          * are empty, the incoming chain is empty too or has the single buf
0061          * that does not require the copy
0062          */
0063 
0064         if (in == NULL) {
0065             return ctx->output_filter(ctx->filter_ctx, in);
0066         }
0067 
0068         if (in->next == NULL
0069 #if (NGX_SENDFILE_LIMIT)
0070             && !(in->buf->in_file && in->buf->file_last > NGX_SENDFILE_LIMIT)
0071 #endif
0072             && ngx_output_chain_as_is(ctx, in->buf))
0073         {
0074             return ctx->output_filter(ctx->filter_ctx, in);
0075         }
0076     }
0077 
0078     /* add the incoming buf to the chain ctx->in */
0079 
0080     if (in) {
0081         if (ngx_output_chain_add_copy(ctx->pool, &ctx->in, in) == NGX_ERROR) {
0082             return NGX_ERROR;
0083         }
0084     }
0085 
0086     out = NULL;
0087     last_out = &out;
0088     last = NGX_NONE;
0089 
0090     for ( ;; ) {
0091 
0092 #if (NGX_HAVE_FILE_AIO || NGX_THREADS)
0093         if (ctx->aio) {
0094             return NGX_AGAIN;
0095         }
0096 #endif
0097 
0098         while (ctx->in) {
0099 
0100             /*
0101              * cycle while there are the ctx->in bufs
0102              * and there are the free output bufs to copy in
0103              */
0104 
0105             bsize = ngx_buf_size(ctx->in->buf);
0106 
0107             if (bsize == 0 && !ngx_buf_special(ctx->in->buf)) {
0108 
0109                 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
0110                               "zero size buf in output "
0111                               "t:%d r:%d f:%d %p %p-%p %p %O-%O",
0112                               ctx->in->buf->temporary,
0113                               ctx->in->buf->recycled,
0114                               ctx->in->buf->in_file,
0115                               ctx->in->buf->start,
0116                               ctx->in->buf->pos,
0117                               ctx->in->buf->last,
0118                               ctx->in->buf->file,
0119                               ctx->in->buf->file_pos,
0120                               ctx->in->buf->file_last);
0121 
0122                 ngx_debug_point();
0123 
0124                 ctx->in = ctx->in->next;
0125 
0126                 continue;
0127             }
0128 
0129             if (ngx_output_chain_as_is(ctx, ctx->in->buf)) {
0130 
0131                 /* move the chain link to the output chain */
0132 
0133                 cl = ctx->in;
0134                 ctx->in = cl->next;
0135 
0136                 *last_out = cl;
0137                 last_out = &cl->next;
0138                 cl->next = NULL;
0139 
0140                 continue;
0141             }
0142 
0143             if (ctx->buf == NULL) {
0144 
0145                 rc = ngx_output_chain_align_file_buf(ctx, bsize);
0146 
0147                 if (rc == NGX_ERROR) {
0148                     return NGX_ERROR;
0149                 }
0150 
0151                 if (rc != NGX_OK) {
0152 
0153                     if (ctx->free) {
0154 
0155                         /* get the free buf */
0156 
0157                         cl = ctx->free;
0158                         ctx->buf = cl->buf;
0159                         ctx->free = cl->next;
0160 
0161                         ngx_free_chain(ctx->pool, cl);
0162 
0163                     } else if (out || ctx->allocated == ctx->bufs.num) {
0164 
0165                         break;
0166 
0167                     } else if (ngx_output_chain_get_buf(ctx, bsize) != NGX_OK) {
0168                         return NGX_ERROR;
0169                     }
0170                 }
0171             }
0172 
0173             rc = ngx_output_chain_copy_buf(ctx);
0174 
0175             if (rc == NGX_ERROR) {
0176                 return rc;
0177             }
0178 
0179             if (rc == NGX_AGAIN) {
0180                 if (out) {
0181                     break;
0182                 }
0183 
0184                 return rc;
0185             }
0186 
0187             /* delete the completed buf from the ctx->in chain */
0188 
0189             if (ngx_buf_size(ctx->in->buf) == 0) {
0190                 ctx->in = ctx->in->next;
0191             }
0192 
0193             cl = ngx_alloc_chain_link(ctx->pool);
0194             if (cl == NULL) {
0195                 return NGX_ERROR;
0196             }
0197 
0198             cl->buf = ctx->buf;
0199             cl->next = NULL;
0200             *last_out = cl;
0201             last_out = &cl->next;
0202             ctx->buf = NULL;
0203         }
0204 
0205         if (out == NULL && last != NGX_NONE) {
0206 
0207             if (ctx->in) {
0208                 return NGX_AGAIN;
0209             }
0210 
0211             return last;
0212         }
0213 
0214         last = ctx->output_filter(ctx->filter_ctx, out);
0215 
0216         if (last == NGX_ERROR || last == NGX_DONE) {
0217             return last;
0218         }
0219 
0220         ngx_chain_update_chains(ctx->pool, &ctx->free, &ctx->busy, &out,
0221                                 ctx->tag);
0222         last_out = &out;
0223     }
0224 }
0225 
0226 
0227 static ngx_inline ngx_int_t
0228 ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
0229 {
0230     ngx_uint_t  sendfile;
0231 
0232     if (ngx_buf_special(buf)) {
0233         return 1;
0234     }
0235 
0236 #if (NGX_THREADS)
0237     if (buf->in_file) {
0238         buf->file->thread_handler = ctx->thread_handler;
0239         buf->file->thread_ctx = ctx->filter_ctx;
0240     }
0241 #endif
0242 
0243     if (buf->in_file && buf->file->directio) {
0244         return 0;
0245     }
0246 
0247     sendfile = ctx->sendfile;
0248 
0249 #if (NGX_SENDFILE_LIMIT)
0250 
0251     if (buf->in_file && buf->file_pos >= NGX_SENDFILE_LIMIT) {
0252         sendfile = 0;
0253     }
0254 
0255 #endif
0256 
0257     if (!sendfile) {
0258 
0259         if (!ngx_buf_in_memory(buf)) {
0260             return 0;
0261         }
0262 
0263         buf->in_file = 0;
0264     }
0265 
0266 #if (NGX_HAVE_AIO_SENDFILE)
0267     if (ctx->aio_preload && buf->in_file) {
0268         (void) ngx_output_chain_aio_setup(ctx, buf->file);
0269     }
0270 #endif
0271 
0272     if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
0273         return 0;
0274     }
0275 
0276     if (ctx->need_in_temp && (buf->memory || buf->mmap)) {
0277         return 0;
0278     }
0279 
0280     return 1;
0281 }
0282 
0283 
0284 #if (NGX_HAVE_AIO_SENDFILE)
0285 
0286 static ngx_int_t
0287 ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx, ngx_file_t *file)
0288 {
0289     ngx_event_aio_t  *aio;
0290 
0291     if (file->aio == NULL && ngx_file_aio_init(file, ctx->pool) != NGX_OK) {
0292         return NGX_ERROR;
0293     }
0294 
0295     aio = file->aio;
0296 
0297     aio->data = ctx->filter_ctx;
0298     aio->preload_handler = ctx->aio_preload;
0299 
0300     return NGX_OK;
0301 }
0302 
0303 #endif
0304 
0305 
0306 static ngx_int_t
0307 ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain,
0308     ngx_chain_t *in)
0309 {
0310     ngx_chain_t  *cl, **ll;
0311 #if (NGX_SENDFILE_LIMIT)
0312     ngx_buf_t    *b, *buf;
0313 #endif
0314 
0315     ll = chain;
0316 
0317     for (cl = *chain; cl; cl = cl->next) {
0318         ll = &cl->next;
0319     }
0320 
0321     while (in) {
0322 
0323         cl = ngx_alloc_chain_link(pool);
0324         if (cl == NULL) {
0325             return NGX_ERROR;
0326         }
0327 
0328 #if (NGX_SENDFILE_LIMIT)
0329 
0330         buf = in->buf;
0331 
0332         if (buf->in_file
0333             && buf->file_pos < NGX_SENDFILE_LIMIT
0334             && buf->file_last > NGX_SENDFILE_LIMIT)
0335         {
0336             /* split a file buf on two bufs by the sendfile limit */
0337 
0338             b = ngx_calloc_buf(pool);
0339             if (b == NULL) {
0340                 return NGX_ERROR;
0341             }
0342 
0343             ngx_memcpy(b, buf, sizeof(ngx_buf_t));
0344 
0345             if (ngx_buf_in_memory(buf)) {
0346                 buf->pos += (ssize_t) (NGX_SENDFILE_LIMIT - buf->file_pos);
0347                 b->last = buf->pos;
0348             }
0349 
0350             buf->file_pos = NGX_SENDFILE_LIMIT;
0351             b->file_last = NGX_SENDFILE_LIMIT;
0352 
0353             cl->buf = b;
0354 
0355         } else {
0356             cl->buf = buf;
0357             in = in->next;
0358         }
0359 
0360 #else
0361         cl->buf = in->buf;
0362         in = in->next;
0363 
0364 #endif
0365 
0366         cl->next = NULL;
0367         *ll = cl;
0368         ll = &cl->next;
0369     }
0370 
0371     return NGX_OK;
0372 }
0373 
0374 
0375 static ngx_int_t
0376 ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
0377 {
0378     size_t      size;
0379     ngx_buf_t  *in;
0380 
0381     in = ctx->in->buf;
0382 
0383     if (in->file == NULL || !in->file->directio) {
0384         return NGX_DECLINED;
0385     }
0386 
0387     ctx->directio = 1;
0388 
0389     size = (size_t) (in->file_pos - (in->file_pos & ~(ctx->alignment - 1)));
0390 
0391     if (size == 0) {
0392 
0393         if (bsize >= (off_t) ctx->bufs.size) {
0394             return NGX_DECLINED;
0395         }
0396 
0397         size = (size_t) bsize;
0398 
0399     } else {
0400         size = (size_t) ctx->alignment - size;
0401 
0402         if ((off_t) size > bsize) {
0403             size = (size_t) bsize;
0404         }
0405     }
0406 
0407     ctx->buf = ngx_create_temp_buf(ctx->pool, size);
0408     if (ctx->buf == NULL) {
0409         return NGX_ERROR;
0410     }
0411 
0412     /*
0413      * we do not set ctx->buf->tag, because we do not want
0414      * to reuse the buf via ctx->free list
0415      */
0416 
0417 #if (NGX_HAVE_ALIGNED_DIRECTIO)
0418     ctx->unaligned = 1;
0419 #endif
0420 
0421     return NGX_OK;
0422 }
0423 
0424 
0425 static ngx_int_t
0426 ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
0427 {
0428     size_t       size;
0429     ngx_buf_t   *b, *in;
0430     ngx_uint_t   recycled;
0431 
0432     in = ctx->in->buf;
0433     size = ctx->bufs.size;
0434     recycled = 1;
0435 
0436     if (in->last_in_chain) {
0437 
0438         if (bsize < (off_t) size) {
0439 
0440             /*
0441              * allocate a small temp buf for a small last buf
0442              * or its small last part
0443              */
0444 
0445             size = (size_t) bsize;
0446             recycled = 0;
0447 
0448         } else if (!ctx->directio
0449                    && ctx->bufs.num == 1
0450                    && (bsize < (off_t) (size + size / 4)))
0451         {
0452             /*
0453              * allocate a temp buf that equals to a last buf,
0454              * if there is no directio, the last buf size is lesser
0455              * than 1.25 of bufs.size and the temp buf is single
0456              */
0457 
0458             size = (size_t) bsize;
0459             recycled = 0;
0460         }
0461     }
0462 
0463     b = ngx_calloc_buf(ctx->pool);
0464     if (b == NULL) {
0465         return NGX_ERROR;
0466     }
0467 
0468     if (ctx->directio) {
0469 
0470         /*
0471          * allocate block aligned to a disk sector size to enable
0472          * userland buffer direct usage conjunctly with directio
0473          */
0474 
0475         b->start = ngx_pmemalign(ctx->pool, size, (size_t) ctx->alignment);
0476         if (b->start == NULL) {
0477             return NGX_ERROR;
0478         }
0479 
0480     } else {
0481         b->start = ngx_palloc(ctx->pool, size);
0482         if (b->start == NULL) {
0483             return NGX_ERROR;
0484         }
0485     }
0486 
0487     b->pos = b->start;
0488     b->last = b->start;
0489     b->end = b->last + size;
0490     b->temporary = 1;
0491     b->tag = ctx->tag;
0492     b->recycled = recycled;
0493 
0494     ctx->buf = b;
0495     ctx->allocated++;
0496 
0497     return NGX_OK;
0498 }
0499 
0500 
0501 static ngx_int_t
0502 ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx)
0503 {
0504     off_t        size;
0505     ssize_t      n;
0506     ngx_buf_t   *src, *dst;
0507     ngx_uint_t   sendfile;
0508 
0509     src = ctx->in->buf;
0510     dst = ctx->buf;
0511 
0512     size = ngx_buf_size(src);
0513     size = ngx_min(size, dst->end - dst->pos);
0514 
0515     sendfile = ctx->sendfile && !ctx->directio;
0516 
0517 #if (NGX_SENDFILE_LIMIT)
0518 
0519     if (src->in_file && src->file_pos >= NGX_SENDFILE_LIMIT) {
0520         sendfile = 0;
0521     }
0522 
0523 #endif
0524 
0525     if (ngx_buf_in_memory(src)) {
0526         ngx_memcpy(dst->pos, src->pos, (size_t) size);
0527         src->pos += (size_t) size;
0528         dst->last += (size_t) size;
0529 
0530         if (src->in_file) {
0531 
0532             if (sendfile) {
0533                 dst->in_file = 1;
0534                 dst->file = src->file;
0535                 dst->file_pos = src->file_pos;
0536                 dst->file_last = src->file_pos + size;
0537 
0538             } else {
0539                 dst->in_file = 0;
0540             }
0541 
0542             src->file_pos += size;
0543 
0544         } else {
0545             dst->in_file = 0;
0546         }
0547 
0548         if (src->pos == src->last) {
0549             dst->flush = src->flush;
0550             dst->last_buf = src->last_buf;
0551             dst->last_in_chain = src->last_in_chain;
0552         }
0553 
0554     } else {
0555 
0556 #if (NGX_HAVE_ALIGNED_DIRECTIO)
0557 
0558         if (ctx->unaligned) {
0559             if (ngx_directio_off(src->file->fd) == NGX_FILE_ERROR) {
0560                 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
0561                               ngx_directio_off_n " \"%s\" failed",
0562                               src->file->name.data);
0563             }
0564         }
0565 
0566 #endif
0567 
0568 #if (NGX_HAVE_FILE_AIO)
0569         if (ctx->aio_handler) {
0570             n = ngx_file_aio_read(src->file, dst->pos, (size_t) size,
0571                                   src->file_pos, ctx->pool);
0572             if (n == NGX_AGAIN) {
0573                 ctx->aio_handler(ctx, src->file);
0574                 return NGX_AGAIN;
0575             }
0576 
0577         } else
0578 #endif
0579 #if (NGX_THREADS)
0580         if (ctx->thread_handler) {
0581             src->file->thread_task = ctx->thread_task;
0582             src->file->thread_handler = ctx->thread_handler;
0583             src->file->thread_ctx = ctx->filter_ctx;
0584 
0585             n = ngx_thread_read(src->file, dst->pos, (size_t) size,
0586                                 src->file_pos, ctx->pool);
0587             if (n == NGX_AGAIN) {
0588                 ctx->thread_task = src->file->thread_task;
0589                 return NGX_AGAIN;
0590             }
0591 
0592         } else
0593 #endif
0594         {
0595             n = ngx_read_file(src->file, dst->pos, (size_t) size,
0596                               src->file_pos);
0597         }
0598 
0599 #if (NGX_HAVE_ALIGNED_DIRECTIO)
0600 
0601         if (ctx->unaligned) {
0602             ngx_err_t  err;
0603 
0604             err = ngx_errno;
0605 
0606             if (ngx_directio_on(src->file->fd) == NGX_FILE_ERROR) {
0607                 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
0608                               ngx_directio_on_n " \"%s\" failed",
0609                               src->file->name.data);
0610             }
0611 
0612             ngx_set_errno(err);
0613 
0614             ctx->unaligned = 0;
0615         }
0616 
0617 #endif
0618 
0619         if (n == NGX_ERROR) {
0620             return (ngx_int_t) n;
0621         }
0622 
0623         if (n != size) {
0624             ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
0625                           ngx_read_file_n " read only %z of %O from \"%s\"",
0626                           n, size, src->file->name.data);
0627             return NGX_ERROR;
0628         }
0629 
0630         dst->last += n;
0631 
0632         if (sendfile) {
0633             dst->in_file = 1;
0634             dst->file = src->file;
0635             dst->file_pos = src->file_pos;
0636             dst->file_last = src->file_pos + n;
0637 
0638         } else {
0639             dst->in_file = 0;
0640         }
0641 
0642         src->file_pos += n;
0643 
0644         if (src->file_pos == src->file_last) {
0645             dst->flush = src->flush;
0646             dst->last_buf = src->last_buf;
0647             dst->last_in_chain = src->last_in_chain;
0648         }
0649     }
0650 
0651     return NGX_OK;
0652 }
0653 
0654 
0655 ngx_int_t
0656 ngx_chain_writer(void *data, ngx_chain_t *in)
0657 {
0658     ngx_chain_writer_ctx_t *ctx = data;
0659 
0660     off_t              size;
0661     ngx_chain_t       *cl, *ln, *chain;
0662     ngx_connection_t  *c;
0663 
0664     c = ctx->connection;
0665 
0666     for (size = 0; in; in = in->next) {
0667 
0668 #if 1
0669         if (ngx_buf_size(in->buf) == 0 && !ngx_buf_special(in->buf)) {
0670 
0671             ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
0672                           "zero size buf in chain writer "
0673                           "t:%d r:%d f:%d %p %p-%p %p %O-%O",
0674                           in->buf->temporary,
0675                           in->buf->recycled,
0676                           in->buf->in_file,
0677                           in->buf->start,
0678                           in->buf->pos,
0679                           in->buf->last,
0680                           in->buf->file,
0681                           in->buf->file_pos,
0682                           in->buf->file_last);
0683 
0684             ngx_debug_point();
0685 
0686             continue;
0687         }
0688 #endif
0689 
0690         size += ngx_buf_size(in->buf);
0691 
0692         ngx_log_debug2(NGX_LOG_DEBUG_CORE, c->log, 0,
0693                        "chain writer buf fl:%d s:%uO",
0694                        in->buf->flush, ngx_buf_size(in->buf));
0695 
0696         cl = ngx_alloc_chain_link(ctx->pool);
0697         if (cl == NULL) {
0698             return NGX_ERROR;
0699         }
0700 
0701         cl->buf = in->buf;
0702         cl->next = NULL;
0703         *ctx->last = cl;
0704         ctx->last = &cl->next;
0705     }
0706 
0707     ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
0708                    "chain writer in: %p", ctx->out);
0709 
0710     for (cl = ctx->out; cl; cl = cl->next) {
0711 
0712 #if 1
0713         if (ngx_buf_size(cl->buf) == 0 && !ngx_buf_special(cl->buf)) {
0714 
0715             ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
0716                           "zero size buf in chain writer "
0717                           "t:%d r:%d f:%d %p %p-%p %p %O-%O",
0718                           cl->buf->temporary,
0719                           cl->buf->recycled,
0720                           cl->buf->in_file,
0721                           cl->buf->start,
0722                           cl->buf->pos,
0723                           cl->buf->last,
0724                           cl->buf->file,
0725                           cl->buf->file_pos,
0726                           cl->buf->file_last);
0727 
0728             ngx_debug_point();
0729 
0730             continue;
0731         }
0732 #endif
0733 
0734         size += ngx_buf_size(cl->buf);
0735     }
0736 
0737     if (size == 0 && !c->buffered) {
0738         return NGX_OK;
0739     }
0740 
0741     chain = c->send_chain(c, ctx->out, ctx->limit);
0742 
0743     ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
0744                    "chain writer out: %p", chain);
0745 
0746     if (chain == NGX_CHAIN_ERROR) {
0747         return NGX_ERROR;
0748     }
0749 
0750     for (cl = ctx->out; cl && cl != chain; /* void */) {
0751         ln = cl;
0752         cl = cl->next;
0753         ngx_free_chain(ctx->pool, ln);
0754     }
0755 
0756     ctx->out = chain;
0757 
0758     if (ctx->out == NULL) {
0759         ctx->last = &ctx->out;
0760 
0761         if (!c->buffered) {
0762             return NGX_OK;
0763         }
0764     }
0765 
0766     return NGX_AGAIN;
0767 }