xref: /nginx/src/os/unix/ngx_linux_sendfile_chain.c (revision 7974:555533169506)
1 
2 /*
3  * Copyright (C) Igor Sysoev
4  * Copyright (C) Nginx, Inc.
5  */
6 
7 
8 #include <ngx_config.h>
9 #include <ngx_core.h>
10 #include <ngx_event.h>
11 
12 
13 static ssize_t ngx_linux_sendfile(ngx_connection_t *c, ngx_buf_t *file,
14     size_t size);
15 
16 #if (NGX_THREADS)
17 #include <ngx_thread_pool.h>
18 
19 #if !(NGX_HAVE_SENDFILE64)
20 #error sendfile64() is required!
21 #endif
22 
23 static ssize_t ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file,
24     size_t size);
25 static void ngx_linux_sendfile_thread_handler(void *data, ngx_log_t *log);
26 #endif
27 
28 
29 /*
30  * On Linux up to 2.4.21 sendfile() (syscall #187) works with 32-bit
31  * offsets only, and the including <sys/sendfile.h> breaks the compiling,
32  * if off_t is 64 bit wide.  So we use own sendfile() definition, where offset
33  * parameter is int32_t, and use sendfile() for the file parts below 2G only,
34  * see src/os/unix/ngx_linux_config.h
35  *
36  * Linux 2.4.21 has the new sendfile64() syscall #239.
37  *
38  * On Linux up to 2.6.16 sendfile() does not allow to pass the count parameter
39  * more than 2G-1 bytes even on 64-bit platforms: it returns EINVAL,
40  * so we limit it to 2G-1 bytes.
41  *
42  * On Linux 2.6.16 and later, sendfile() silently limits the count parameter
43  * to 2G minus the page size, even on 64-bit platforms.
44  */
45 
46 #define NGX_SENDFILE_MAXSIZE  2147483647L
47 
48 
49 ngx_chain_t *
50 ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
51 {
52     int            tcp_nodelay;
53     off_t          send, prev_send;
54     size_t         file_size, sent;
55     ssize_t        n;
56     ngx_err_t      err;
57     ngx_buf_t     *file;
58     ngx_event_t   *wev;
59     ngx_chain_t   *cl;
60     ngx_iovec_t    header;
61     struct iovec   headers[NGX_IOVS_PREALLOCATE];
62 
63     wev = c->write;
64 
65     if (!wev->ready) {
66         return in;
67     }
68 
69 
70     /* the maximum limit size is 2G-1 - the page size */
71 
72     if (limit == 0 || limit > (off_t) (NGX_SENDFILE_MAXSIZE - ngx_pagesize)) {
73         limit = NGX_SENDFILE_MAXSIZE - ngx_pagesize;
74     }
75 
76 
77     send = 0;
78 
79     header.iovs = headers;
80     header.nalloc = NGX_IOVS_PREALLOCATE;
81 
82     for ( ;; ) {
83         prev_send = send;
84 
85         /* create the iovec and coalesce the neighbouring bufs */
86 
87         cl = ngx_output_chain_to_iovec(&header, in, limit - send, c->log);
88 
89         if (cl == NGX_CHAIN_ERROR) {
90             return NGX_CHAIN_ERROR;
91         }
92 
93         send += header.size;
94 
95         /* set TCP_CORK if there is a header before a file */
96 
97         if (c->tcp_nopush == NGX_TCP_NOPUSH_UNSET
98             && header.count != 0
99             && cl
100             && cl->buf->in_file)
101         {
102             /* the TCP_CORK and TCP_NODELAY are mutually exclusive */
103 
104             if (c->tcp_nodelay == NGX_TCP_NODELAY_SET) {
105 
106                 tcp_nodelay = 0;
107 
108                 if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY,
109                                (const void *) &tcp_nodelay, sizeof(int)) == -1)
110                 {
111                     err = ngx_socket_errno;
112 
113                     /*
114                      * there is a tiny chance to be interrupted, however,
115                      * we continue a processing with the TCP_NODELAY
116                      * and without the TCP_CORK
117                      */
118 
119                     if (err != NGX_EINTR) {
120                         wev->error = 1;
121                         ngx_connection_error(c, err,
122                                              "setsockopt(TCP_NODELAY) failed");
123                         return NGX_CHAIN_ERROR;
124                     }
125 
126                 } else {
127                     c->tcp_nodelay = NGX_TCP_NODELAY_UNSET;
128 
129                     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
130                                    "no tcp_nodelay");
131                 }
132             }
133 
134             if (c->tcp_nodelay == NGX_TCP_NODELAY_UNSET) {
135 
136                 if (ngx_tcp_nopush(c->fd) == -1) {
137                     err = ngx_socket_errno;
138 
139                     /*
140                      * there is a tiny chance to be interrupted, however,
141                      * we continue a processing without the TCP_CORK
142                      */
143 
144                     if (err != NGX_EINTR) {
145                         wev->error = 1;
146                         ngx_connection_error(c, err,
147                                              ngx_tcp_nopush_n " failed");
148                         return NGX_CHAIN_ERROR;
149                     }
150 
151                 } else {
152                     c->tcp_nopush = NGX_TCP_NOPUSH_SET;
153 
154                     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
155                                    "tcp_nopush");
156                 }
157             }
158         }
159 
160         /* get the file buf */
161 
162         if (header.count == 0 && cl && cl->buf->in_file && send < limit) {
163             file = cl->buf;
164 
165             /* coalesce the neighbouring file bufs */
166 
167             file_size = (size_t) ngx_chain_coalesce_file(&cl, limit - send);
168 
169             send += file_size;
170 #if 1
171             if (file_size == 0) {
172                 ngx_debug_point();
173                 return NGX_CHAIN_ERROR;
174             }
175 #endif
176 
177             n = ngx_linux_sendfile(c, file, file_size);
178 
179             if (n == NGX_ERROR) {
180                 return NGX_CHAIN_ERROR;
181             }
182 
183             if (n == NGX_DONE) {
184                 /* thread task posted */
185                 return in;
186             }
187 
188             sent = (n == NGX_AGAIN) ? 0 : n;
189 
190         } else {
191             n = ngx_writev(c, &header);
192 
193             if (n == NGX_ERROR) {
194                 return NGX_CHAIN_ERROR;
195             }
196 
197             sent = (n == NGX_AGAIN) ? 0 : n;
198         }
199 
200         c->sent += sent;
201 
202         in = ngx_chain_update_sent(in, sent);
203 
204         if (n == NGX_AGAIN) {
205             wev->ready = 0;
206             return in;
207         }
208 
209         if ((size_t) (send - prev_send) != sent) {
210 
211             /*
212              * sendfile() on Linux 4.3+ might be interrupted at any time,
213              * and provides no indication if it was interrupted or not,
214              * so we have to retry till an explicit EAGAIN
215              *
216              * sendfile() in threads can also report less bytes written
217              * than we are prepared to send now, since it was started in
218              * some point in the past, so we again have to retry
219              */
220 
221             send = prev_send + sent;
222         }
223 
224         if (send >= limit || in == NULL) {
225             return in;
226         }
227     }
228 }
229 
230 
231 static ssize_t
232 ngx_linux_sendfile(ngx_connection_t *c, ngx_buf_t *file, size_t size)
233 {
234 #if (NGX_HAVE_SENDFILE64)
235     off_t      offset;
236 #else
237     int32_t    offset;
238 #endif
239     ssize_t    n;
240     ngx_err_t  err;
241 
242 #if (NGX_THREADS)
243 
244     if (file->file->thread_handler) {
245         return ngx_linux_sendfile_thread(c, file, size);
246     }
247 
248 #endif
249 
250 #if (NGX_HAVE_SENDFILE64)
251     offset = file->file_pos;
252 #else
253     offset = (int32_t) file->file_pos;
254 #endif
255 
256 eintr:
257 
258     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
259                    "sendfile: @%O %uz", file->file_pos, size);
260 
261     n = sendfile(c->fd, file->file->fd, &offset, size);
262 
263     if (n == -1) {
264         err = ngx_errno;
265 
266         switch (err) {
267         case NGX_EAGAIN:
268             ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
269                            "sendfile() is not ready");
270             return NGX_AGAIN;
271 
272         case NGX_EINTR:
273             ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
274                            "sendfile() was interrupted");
275             goto eintr;
276 
277         default:
278             c->write->error = 1;
279             ngx_connection_error(c, err, "sendfile() failed");
280             return NGX_ERROR;
281         }
282     }
283 
284     if (n == 0) {
285         /*
286          * if sendfile returns zero, then someone has truncated the file,
287          * so the offset became beyond the end of the file
288          */
289 
290         ngx_log_error(NGX_LOG_ALERT, c->log, 0,
291                       "sendfile() reported that \"%s\" was truncated at %O",
292                       file->file->name.data, file->file_pos);
293 
294         return NGX_ERROR;
295     }
296 
297     ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, "sendfile: %z of %uz @%O",
298                    n, size, file->file_pos);
299 
300     return n;
301 }
302 
303 
304 #if (NGX_THREADS)
305 
306 typedef struct {
307     ngx_buf_t     *file;
308     ngx_socket_t   socket;
309     size_t         size;
310 
311     size_t         sent;
312     ngx_err_t      err;
313 } ngx_linux_sendfile_ctx_t;
314 
315 
316 static ssize_t
317 ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file, size_t size)
318 {
319     ngx_event_t               *wev;
320     ngx_thread_task_t         *task;
321     ngx_linux_sendfile_ctx_t  *ctx;
322 
323     ngx_log_debug3(NGX_LOG_DEBUG_CORE, c->log, 0,
324                    "linux sendfile thread: %d, %uz, %O",
325                    file->file->fd, size, file->file_pos);
326 
327     task = c->sendfile_task;
328 
329     if (task == NULL) {
330         task = ngx_thread_task_alloc(c->pool, sizeof(ngx_linux_sendfile_ctx_t));
331         if (task == NULL) {
332             return NGX_ERROR;
333         }
334 
335         task->handler = ngx_linux_sendfile_thread_handler;
336 
337         c->sendfile_task = task;
338     }
339 
340     ctx = task->ctx;
341     wev = c->write;
342 
343     if (task->event.complete) {
344         task->event.complete = 0;
345 
346         if (ctx->err == NGX_EAGAIN) {
347             /*
348              * if wev->complete is set, this means that a write event
349              * happened while we were waiting for the thread task, so
350              * we have to retry sending even on EAGAIN
351              */
352 
353             if (wev->complete) {
354                 return 0;
355             }
356 
357             return NGX_AGAIN;
358         }
359 
360         if (ctx->err) {
361             wev->error = 1;
362             ngx_connection_error(c, ctx->err, "sendfile() failed");
363             return NGX_ERROR;
364         }
365 
366         if (ctx->sent == 0) {
367             /*
368              * if sendfile returns zero, then someone has truncated the file,
369              * so the offset became beyond the end of the file
370              */
371 
372             ngx_log_error(NGX_LOG_ALERT, c->log, 0,
373                           "sendfile() reported that \"%s\" was truncated at %O",
374                           file->file->name.data, file->file_pos);
375 
376             return NGX_ERROR;
377         }
378 
379         return ctx->sent;
380     }
381 
382     ctx->file = file;
383     ctx->socket = c->fd;
384     ctx->size = size;
385 
386     wev->complete = 0;
387 
388     if (file->file->thread_handler(task, file->file) != NGX_OK) {
389         return NGX_ERROR;
390     }
391 
392     return NGX_DONE;
393 }
394 
395 
396 static void
397 ngx_linux_sendfile_thread_handler(void *data, ngx_log_t *log)
398 {
399     ngx_linux_sendfile_ctx_t *ctx = data;
400 
401     off_t       offset;
402     ssize_t     n;
403     ngx_buf_t  *file;
404 
405     ngx_log_debug0(NGX_LOG_DEBUG_CORE, log, 0, "linux sendfile thread handler");
406 
407     file = ctx->file;
408     offset = file->file_pos;
409 
410 again:
411 
412     n = sendfile(ctx->socket, file->file->fd, &offset, ctx->size);
413 
414     if (n == -1) {
415         ctx->err = ngx_errno;
416 
417     } else {
418         ctx->sent = n;
419         ctx->err = 0;
420     }
421 
422 #if 0
423     ngx_time_update();
424 #endif
425 
426     ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0,
427                    "sendfile: %z (err: %d) of %uz @%O",
428                    n, ctx->err, ctx->size, file->file_pos);
429 
430     if (ctx->err == NGX_EINTR) {
431         goto again;
432     }
433 }
434 
435 #endif /* NGX_THREADS */
436