ngx_output_chain.c revision e18a033b
1
2/*
3 * Copyright (C) Igor Sysoev
4 * Copyright (C) Nginx, Inc.
5 */
6
7
8#include <ngx_config.h>
9#include <ngx_core.h>
10#include <ngx_event.h>
11
12
13#if 0
14#define NGX_SENDFILE_LIMIT  4096
15#endif
16
17/*
18 * When DIRECTIO is enabled FreeBSD, Solaris, and MacOSX read directly
19 * to an application memory from a device if parameters are aligned
20 * to device sector boundary (512 bytes).  They fallback to usual read
21 * operation if the parameters are not aligned.
22 * Linux allows DIRECTIO only if the parameters are aligned to a filesystem
23 * sector boundary, otherwise it returns EINVAL.  The sector size is
24 * usually 512 bytes, however, on XFS it may be 4096 bytes.
25 */
26
27#define NGX_NONE            1
28
29
30static ngx_inline ngx_int_t
31    ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
32#if (NGX_HAVE_AIO_SENDFILE)
33static ngx_int_t ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx,
34    ngx_file_t *file);
35#endif
36static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
37    ngx_chain_t **chain, ngx_chain_t *in);
38static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx,
39    off_t bsize);
40static ngx_int_t ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx,
41    off_t bsize);
42static ngx_int_t ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx);
43
44
45ngx_int_t
46ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in)
47{
48    off_t         bsize;
49    ngx_int_t     rc, last;
50    ngx_chain_t  *cl, *out, **last_out;
51
52    if (ctx->in == NULL && ctx->busy == NULL
53#if (NGX_HAVE_FILE_AIO || NGX_THREADS)
54        && !ctx->aio
55#endif
56       )
57    {
58        /*
59         * the short path for the case when the ctx->in and ctx->busy chains
60         * are empty, the incoming chain is empty too or has the single buf
61         * that does not require the copy
62         */
63
64        if (in == NULL) {
65            return ctx->output_filter(ctx->filter_ctx, in);
66        }
67
68        if (in->next == NULL
69#if (NGX_SENDFILE_LIMIT)
70            && !(in->buf->in_file && in->buf->file_last > NGX_SENDFILE_LIMIT)
71#endif
72            && ngx_output_chain_as_is(ctx, in->buf))
73        {
74            return ctx->output_filter(ctx->filter_ctx, in);
75        }
76    }
77
78    /* add the incoming buf to the chain ctx->in */
79
80    if (in) {
81        if (ngx_output_chain_add_copy(ctx->pool, &ctx->in, in) == NGX_ERROR) {
82            return NGX_ERROR;
83        }
84    }
85
86    out = NULL;
87    last_out = &out;
88    last = NGX_NONE;
89
90    for ( ;; ) {
91
92#if (NGX_HAVE_FILE_AIO || NGX_THREADS)
93        if (ctx->aio) {
94            return NGX_AGAIN;
95        }
96#endif
97
98        while (ctx->in) {
99
100            /*
101             * cycle while there are the ctx->in bufs
102             * and there are the free output bufs to copy in
103             */
104
105            bsize = ngx_buf_size(ctx->in->buf);
106
107            if (bsize == 0 && !ngx_buf_special(ctx->in->buf)) {
108
109                ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
110                              "zero size buf in output "
111                              "t:%d r:%d f:%d %p %p-%p %p %O-%O",
112                              ctx->in->buf->temporary,
113                              ctx->in->buf->recycled,
114                              ctx->in->buf->in_file,
115                              ctx->in->buf->start,
116                              ctx->in->buf->pos,
117                              ctx->in->buf->last,
118                              ctx->in->buf->file,
119                              ctx->in->buf->file_pos,
120                              ctx->in->buf->file_last);
121
122                ngx_debug_point();
123
124                ctx->in = ctx->in->next;
125
126                continue;
127            }
128
129            if (ngx_output_chain_as_is(ctx, ctx->in->buf)) {
130
131                /* move the chain link to the output chain */
132
133                cl = ctx->in;
134                ctx->in = cl->next;
135
136                *last_out = cl;
137                last_out = &cl->next;
138                cl->next = NULL;
139
140                continue;
141            }
142
143            if (ctx->buf == NULL) {
144
145                rc = ngx_output_chain_align_file_buf(ctx, bsize);
146
147                if (rc == NGX_ERROR) {
148                    return NGX_ERROR;
149                }
150
151                if (rc != NGX_OK) {
152
153                    if (ctx->free) {
154
155                        /* get the free buf */
156
157                        cl = ctx->free;
158                        ctx->buf = cl->buf;
159                        ctx->free = cl->next;
160
161                        ngx_free_chain(ctx->pool, cl);
162
163                    } else if (out || ctx->allocated == ctx->bufs.num) {
164
165                        break;
166
167                    } else if (ngx_output_chain_get_buf(ctx, bsize) != NGX_OK) {
168                        return NGX_ERROR;
169                    }
170                }
171            }
172
173            rc = ngx_output_chain_copy_buf(ctx);
174
175            if (rc == NGX_ERROR) {
176                return rc;
177            }
178
179            if (rc == NGX_AGAIN) {
180                if (out) {
181                    break;
182                }
183
184                return rc;
185            }
186
187            /* delete the completed buf from the ctx->in chain */
188
189            if (ngx_buf_size(ctx->in->buf) == 0) {
190                ctx->in = ctx->in->next;
191            }
192
193            cl = ngx_alloc_chain_link(ctx->pool);
194            if (cl == NULL) {
195                return NGX_ERROR;
196            }
197
198            cl->buf = ctx->buf;
199            cl->next = NULL;
200            *last_out = cl;
201            last_out = &cl->next;
202            ctx->buf = NULL;
203        }
204
205        if (out == NULL && last != NGX_NONE) {
206
207            if (ctx->in) {
208                return NGX_AGAIN;
209            }
210
211            return last;
212        }
213
214        last = ctx->output_filter(ctx->filter_ctx, out);
215
216        if (last == NGX_ERROR || last == NGX_DONE) {
217            return last;
218        }
219
220        ngx_chain_update_chains(ctx->pool, &ctx->free, &ctx->busy, &out,
221                                ctx->tag);
222        last_out = &out;
223    }
224}
225
226
227static ngx_inline ngx_int_t
228ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
229{
230    ngx_uint_t  sendfile;
231
232    if (ngx_buf_special(buf)) {
233        return 1;
234    }
235
236#if (NGX_THREADS)
237    if (buf->in_file) {
238        buf->file->thread_handler = ctx->thread_handler;
239        buf->file->thread_ctx = ctx->filter_ctx;
240    }
241#endif
242
243    if (buf->in_file && buf->file->directio) {
244        return 0;
245    }
246
247    sendfile = ctx->sendfile;
248
249#if (NGX_SENDFILE_LIMIT)
250
251    if (buf->in_file && buf->file_pos >= NGX_SENDFILE_LIMIT) {
252        sendfile = 0;
253    }
254
255#endif
256
257    if (!sendfile) {
258
259        if (!ngx_buf_in_memory(buf)) {
260            return 0;
261        }
262
263        buf->in_file = 0;
264    }
265
266#if (NGX_HAVE_AIO_SENDFILE)
267    if (ctx->aio_preload && buf->in_file) {
268        (void) ngx_output_chain_aio_setup(ctx, buf->file);
269    }
270#endif
271
272    if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
273        return 0;
274    }
275
276    if (ctx->need_in_temp && (buf->memory || buf->mmap)) {
277        return 0;
278    }
279
280    return 1;
281}
282
283
284#if (NGX_HAVE_AIO_SENDFILE)
285
286static ngx_int_t
287ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx, ngx_file_t *file)
288{
289    ngx_event_aio_t  *aio;
290
291    if (file->aio == NULL && ngx_file_aio_init(file, ctx->pool) != NGX_OK) {
292        return NGX_ERROR;
293    }
294
295    aio = file->aio;
296
297    aio->data = ctx->filter_ctx;
298    aio->preload_handler = ctx->aio_preload;
299
300    return NGX_OK;
301}
302
303#endif
304
305
306static ngx_int_t
307ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain,
308    ngx_chain_t *in)
309{
310    ngx_chain_t  *cl, **ll;
311#if (NGX_SENDFILE_LIMIT)
312    ngx_buf_t    *b, *buf;
313#endif
314
315    ll = chain;
316
317    for (cl = *chain; cl; cl = cl->next) {
318        ll = &cl->next;
319    }
320
321    while (in) {
322
323        cl = ngx_alloc_chain_link(pool);
324        if (cl == NULL) {
325            return NGX_ERROR;
326        }
327
328#if (NGX_SENDFILE_LIMIT)
329
330        buf = in->buf;
331
332        if (buf->in_file
333            && buf->file_pos < NGX_SENDFILE_LIMIT
334            && buf->file_last > NGX_SENDFILE_LIMIT)
335        {
336            /* split a file buf on two bufs by the sendfile limit */
337
338            b = ngx_calloc_buf(pool);
339            if (b == NULL) {
340                return NGX_ERROR;
341            }
342
343            ngx_memcpy(b, buf, sizeof(ngx_buf_t));
344
345            if (ngx_buf_in_memory(buf)) {
346                buf->pos += (ssize_t) (NGX_SENDFILE_LIMIT - buf->file_pos);
347                b->last = buf->pos;
348            }
349
350            buf->file_pos = NGX_SENDFILE_LIMIT;
351            b->file_last = NGX_SENDFILE_LIMIT;
352
353            cl->buf = b;
354
355        } else {
356            cl->buf = buf;
357            in = in->next;
358        }
359
360#else
361        cl->buf = in->buf;
362        in = in->next;
363
364#endif
365
366        cl->next = NULL;
367        *ll = cl;
368        ll = &cl->next;
369    }
370
371    return NGX_OK;
372}
373
374
375static ngx_int_t
376ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
377{
378    size_t      size;
379    ngx_buf_t  *in;
380
381    in = ctx->in->buf;
382
383    if (in->file == NULL || !in->file->directio) {
384        return NGX_DECLINED;
385    }
386
387    ctx->directio = 1;
388
389    size = (size_t) (in->file_pos - (in->file_pos & ~(ctx->alignment - 1)));
390
391    if (size == 0) {
392
393        if (bsize >= (off_t) ctx->bufs.size) {
394            return NGX_DECLINED;
395        }
396
397        size = (size_t) bsize;
398
399    } else {
400        size = (size_t) ctx->alignment - size;
401
402        if ((off_t) size > bsize) {
403            size = (size_t) bsize;
404        }
405    }
406
407    ctx->buf = ngx_create_temp_buf(ctx->pool, size);
408    if (ctx->buf == NULL) {
409        return NGX_ERROR;
410    }
411
412    /*
413     * we do not set ctx->buf->tag, because we do not want
414     * to reuse the buf via ctx->free list
415     */
416
417#if (NGX_HAVE_ALIGNED_DIRECTIO)
418    ctx->unaligned = 1;
419#endif
420
421    return NGX_OK;
422}
423
424
425static ngx_int_t
426ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
427{
428    size_t       size;
429    ngx_buf_t   *b, *in;
430    ngx_uint_t   recycled;
431
432    in = ctx->in->buf;
433    size = ctx->bufs.size;
434    recycled = 1;
435
436    if (in->last_in_chain) {
437
438        if (bsize < (off_t) size) {
439
440            /*
441             * allocate a small temp buf for a small last buf
442             * or its small last part
443             */
444
445            size = (size_t) bsize;
446            recycled = 0;
447
448        } else if (!ctx->directio
449                   && ctx->bufs.num == 1
450                   && (bsize < (off_t) (size + size / 4)))
451        {
452            /*
453             * allocate a temp buf that equals to a last buf,
454             * if there is no directio, the last buf size is lesser
455             * than 1.25 of bufs.size and the temp buf is single
456             */
457
458            size = (size_t) bsize;
459            recycled = 0;
460        }
461    }
462
463    b = ngx_calloc_buf(ctx->pool);
464    if (b == NULL) {
465        return NGX_ERROR;
466    }
467
468    if (ctx->directio) {
469
470        /*
471         * allocate block aligned to a disk sector size to enable
472         * userland buffer direct usage conjunctly with directio
473         */
474
475        b->start = ngx_pmemalign(ctx->pool, size, (size_t) ctx->alignment);
476        if (b->start == NULL) {
477            return NGX_ERROR;
478        }
479
480    } else {
481        b->start = ngx_palloc(ctx->pool, size);
482        if (b->start == NULL) {
483            return NGX_ERROR;
484        }
485    }
486
487    b->pos = b->start;
488    b->last = b->start;
489    b->end = b->last + size;
490    b->temporary = 1;
491    b->tag = ctx->tag;
492    b->recycled = recycled;
493
494    ctx->buf = b;
495    ctx->allocated++;
496
497    return NGX_OK;
498}
499
500
501static ngx_int_t
502ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx)
503{
504    off_t        size;
505    ssize_t      n;
506    ngx_buf_t   *src, *dst;
507    ngx_uint_t   sendfile;
508
509    src = ctx->in->buf;
510    dst = ctx->buf;
511
512    size = ngx_buf_size(src);
513    size = ngx_min(size, dst->end - dst->pos);
514
515    sendfile = ctx->sendfile && !ctx->directio;
516
517#if (NGX_SENDFILE_LIMIT)
518
519    if (src->in_file && src->file_pos >= NGX_SENDFILE_LIMIT) {
520        sendfile = 0;
521    }
522
523#endif
524
525    if (ngx_buf_in_memory(src)) {
526        ngx_memcpy(dst->pos, src->pos, (size_t) size);
527        src->pos += (size_t) size;
528        dst->last += (size_t) size;
529
530        if (src->in_file) {
531
532            if (sendfile) {
533                dst->in_file = 1;
534                dst->file = src->file;
535                dst->file_pos = src->file_pos;
536                dst->file_last = src->file_pos + size;
537
538            } else {
539                dst->in_file = 0;
540            }
541
542            src->file_pos += size;
543
544        } else {
545            dst->in_file = 0;
546        }
547
548        if (src->pos == src->last) {
549            dst->flush = src->flush;
550            dst->last_buf = src->last_buf;
551            dst->last_in_chain = src->last_in_chain;
552        }
553
554    } else {
555
556#if (NGX_HAVE_ALIGNED_DIRECTIO)
557
558        if (ctx->unaligned) {
559            if (ngx_directio_off(src->file->fd) == NGX_FILE_ERROR) {
560                ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
561                              ngx_directio_off_n " \"%s\" failed",
562                              src->file->name.data);
563            }
564        }
565
566#endif
567
568#if (NGX_HAVE_FILE_AIO)
569        if (ctx->aio_handler) {
570            n = ngx_file_aio_read(src->file, dst->pos, (size_t) size,
571                                  src->file_pos, ctx->pool);
572            if (n == NGX_AGAIN) {
573                ctx->aio_handler(ctx, src->file);
574                return NGX_AGAIN;
575            }
576
577        } else
578#endif
579#if (NGX_THREADS)
580        if (ctx->thread_handler) {
581            src->file->thread_task = ctx->thread_task;
582            src->file->thread_handler = ctx->thread_handler;
583            src->file->thread_ctx = ctx->filter_ctx;
584
585            n = ngx_thread_read(src->file, dst->pos, (size_t) size,
586                                src->file_pos, ctx->pool);
587            if (n == NGX_AGAIN) {
588                ctx->thread_task = src->file->thread_task;
589                return NGX_AGAIN;
590            }
591
592        } else
593#endif
594        {
595            n = ngx_read_file(src->file, dst->pos, (size_t) size,
596                              src->file_pos);
597        }
598
599#if (NGX_HAVE_ALIGNED_DIRECTIO)
600
601        if (ctx->unaligned) {
602            ngx_err_t  err;
603
604            err = ngx_errno;
605
606            if (ngx_directio_on(src->file->fd) == NGX_FILE_ERROR) {
607                ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
608                              ngx_directio_on_n " \"%s\" failed",
609                              src->file->name.data);
610            }
611
612            ngx_set_errno(err);
613
614            ctx->unaligned = 0;
615        }
616
617#endif
618
619        if (n == NGX_ERROR) {
620            return (ngx_int_t) n;
621        }
622
623        if (n != size) {
624            ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
625                          ngx_read_file_n " read only %z of %O from \"%s\"",
626                          n, size, src->file->name.data);
627            return NGX_ERROR;
628        }
629
630        dst->last += n;
631
632        if (sendfile) {
633            dst->in_file = 1;
634            dst->file = src->file;
635            dst->file_pos = src->file_pos;
636            dst->file_last = src->file_pos + n;
637
638        } else {
639            dst->in_file = 0;
640        }
641
642        src->file_pos += n;
643
644        if (src->file_pos == src->file_last) {
645            dst->flush = src->flush;
646            dst->last_buf = src->last_buf;
647            dst->last_in_chain = src->last_in_chain;
648        }
649    }
650
651    return NGX_OK;
652}
653
654
655ngx_int_t
656ngx_chain_writer(void *data, ngx_chain_t *in)
657{
658    ngx_chain_writer_ctx_t *ctx = data;
659
660    off_t              size;
661    ngx_chain_t       *cl, *ln, *chain;
662    ngx_connection_t  *c;
663
664    c = ctx->connection;
665
666    for (size = 0; in; in = in->next) {
667
668#if 1
669        if (ngx_buf_size(in->buf) == 0 && !ngx_buf_special(in->buf)) {
670
671            ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
672                          "zero size buf in chain writer "
673                          "t:%d r:%d f:%d %p %p-%p %p %O-%O",
674                          in->buf->temporary,
675                          in->buf->recycled,
676                          in->buf->in_file,
677                          in->buf->start,
678                          in->buf->pos,
679                          in->buf->last,
680                          in->buf->file,
681                          in->buf->file_pos,
682                          in->buf->file_last);
683
684            ngx_debug_point();
685
686            continue;
687        }
688#endif
689
690        size += ngx_buf_size(in->buf);
691
692        ngx_log_debug2(NGX_LOG_DEBUG_CORE, c->log, 0,
693                       "chain writer buf fl:%d s:%uO",
694                       in->buf->flush, ngx_buf_size(in->buf));
695
696        cl = ngx_alloc_chain_link(ctx->pool);
697        if (cl == NULL) {
698            return NGX_ERROR;
699        }
700
701        cl->buf = in->buf;
702        cl->next = NULL;
703        *ctx->last = cl;
704        ctx->last = &cl->next;
705    }
706
707    ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
708                   "chain writer in: %p", ctx->out);
709
710    for (cl = ctx->out; cl; cl = cl->next) {
711
712#if 1
713        if (ngx_buf_size(cl->buf) == 0 && !ngx_buf_special(cl->buf)) {
714
715            ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
716                          "zero size buf in chain writer "
717                          "t:%d r:%d f:%d %p %p-%p %p %O-%O",
718                          cl->buf->temporary,
719                          cl->buf->recycled,
720                          cl->buf->in_file,
721                          cl->buf->start,
722                          cl->buf->pos,
723                          cl->buf->last,
724                          cl->buf->file,
725                          cl->buf->file_pos,
726                          cl->buf->file_last);
727
728            ngx_debug_point();
729
730            continue;
731        }
732#endif
733
734        size += ngx_buf_size(cl->buf);
735    }
736
737    if (size == 0 && !c->buffered) {
738        return NGX_OK;
739    }
740
741    chain = c->send_chain(c, ctx->out, ctx->limit);
742
743    ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
744                   "chain writer out: %p", chain);
745
746    if (chain == NGX_CHAIN_ERROR) {
747        return NGX_ERROR;
748    }
749
750    for (cl = ctx->out; cl && cl != chain; /* void */) {
751        ln = cl;
752        cl = cl->next;
753        ngx_free_chain(ctx->pool, ln);
754    }
755
756    ctx->out = chain;
757
758    if (ctx->out == NULL) {
759        ctx->last = &ctx->out;
760
761        if (!c->buffered) {
762            return NGX_OK;
763        }
764    }
765
766    return NGX_AGAIN;
767}
768