1 /** @file
2 
3   A brief file description
4 
5   @section license License
6 
7   Licensed to the Apache Software Foundation (ASF) under one
8   or more contributor license agreements.  See the NOTICE file
9   distributed with this work for additional information
10   regarding copyright ownership.  The ASF licenses this file
11   to you under the Apache License, Version 2.0 (the
12   "License"); you may not use this file except in compliance
13   with the License.  You may obtain a copy of the License at
14 
15       http://www.apache.org/licenses/LICENSE-2.0
16 
17   Unless required by applicable law or agreed to in writing, software
18   distributed under the License is distributed on an "AS IS" BASIS,
19   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20   See the License for the specific language governing permissions and
21   limitations under the License.
22  */
23 
24 #pragma once
25 
26 #include "tscore/ink_platform.h"
27 #include "tscore/ink_resource.h"
28 
29 // TODO: I think we're overly aggressive here on making MIOBuffer 64-bit
30 // but not sure it's worthwhile changing anything to 32-bit honestly.
31 
32 //////////////////////////////////////////////////////////////
33 //
34 // returns 0 for DEFAULT_BUFFER_BASE_SIZE,
35 // +1 for each power of 2
36 //
37 //////////////////////////////////////////////////////////////
38 TS_INLINE int64_t
39 buffer_size_to_index(int64_t size, int64_t max = max_iobuffer_size)
40 {
41   int64_t r = max;
42 
43   while (r && BUFFER_SIZE_FOR_INDEX(r - 1) >= size) {
44     r--;
45   }
46   return r;
47 }
48 
49 TS_INLINE int64_t
50 iobuffer_size_to_index(int64_t size, int64_t max)
51 {
52   if (size > BUFFER_SIZE_FOR_INDEX(max)) {
53     return BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(size);
54   }
55   return buffer_size_to_index(size, max);
56 }
57 
58 TS_INLINE int64_t
59 index_to_buffer_size(int64_t idx)
60 {
61   if (BUFFER_SIZE_INDEX_IS_FAST_ALLOCATED(idx)) {
62     return BUFFER_SIZE_FOR_INDEX(idx);
63   } else if (BUFFER_SIZE_INDEX_IS_XMALLOCED(idx)) {
64     return BUFFER_SIZE_FOR_XMALLOC(idx);
65     // coverity[dead_error_condition]
66   } else if (BUFFER_SIZE_INDEX_IS_CONSTANT(idx)) {
67     return BUFFER_SIZE_FOR_CONSTANT(idx);
68   }
69   // coverity[dead_error_line]
70   return 0;
71 }
72 
73 TS_INLINE IOBufferBlock *
74 iobufferblock_clone(IOBufferBlock *src, int64_t offset, int64_t len)
75 {
76   IOBufferBlock *start_buf   = nullptr;
77   IOBufferBlock *current_buf = nullptr;
78 
79   while (src && len >= 0) {
80     char *start       = src->_start;
81     char *end         = src->_end;
82     int64_t max_bytes = end - start;
83 
84     max_bytes -= offset;
85     if (max_bytes <= 0) {
86       offset = -max_bytes;
87       src    = src->next.get();
88       continue;
89     }
90 
91     int64_t bytes = len;
92     if (bytes >= max_bytes) {
93       bytes = max_bytes;
94     }
95 
96     IOBufferBlock *new_buf = src->clone();
97     new_buf->_start += offset;
98     new_buf->_buf_end = new_buf->_end = new_buf->_start + bytes;
99 
100     if (!start_buf) {
101       start_buf   = new_buf;
102       current_buf = start_buf;
103     } else {
104       current_buf->next = new_buf;
105       current_buf       = new_buf;
106     }
107 
108     len -= bytes;
109     src    = src->next.get();
110     offset = 0;
111   }
112 
113   return start_buf;
114 }
115 
116 TS_INLINE IOBufferBlock *
117 iobufferblock_skip(IOBufferBlock *b, int64_t *poffset, int64_t *plen, int64_t write)
118 {
119   int64_t offset = *poffset;
120   int64_t len    = write;
121 
122   while (b && len >= 0) {
123     int64_t max_bytes = b->read_avail();
124 
125     // If this block ends before the start offset, skip it
126     // and adjust the offset to consume its length.
127     max_bytes -= offset;
128     if (max_bytes <= 0) {
129       offset = -max_bytes;
130       b      = b->next.get();
131       continue;
132     }
133 
134     if (len >= max_bytes) {
135       b = b->next.get();
136       len -= max_bytes;
137       offset = 0;
138     } else {
139       offset = offset + len;
140       break;
141     }
142   }
143 
144   *poffset = offset;
145   *plen -= write;
146   return b;
147 }
148 
149 #ifdef TRACK_BUFFER_USER
150 TS_INLINE void
151 iobuffer_mem_inc(const char *_loc, int64_t _size_index)
152 {
153   if (!res_track_memory) {
154     return;
155   }
156 
157   if (!BUFFER_SIZE_INDEX_IS_FAST_ALLOCATED(_size_index)) {
158     return;
159   }
160 
161   if (!_loc) {
162     _loc = "memory/IOBuffer/UNKNOWN-LOCATION";
163   }
164   ResourceTracker::increment(_loc, index_to_buffer_size(_size_index));
165 }
166 
167 TS_INLINE void
168 iobuffer_mem_dec(const char *_loc, int64_t _size_index)
169 {
170   if (!res_track_memory) {
171     return;
172   }
173 
174   if (!BUFFER_SIZE_INDEX_IS_FAST_ALLOCATED(_size_index)) {
175     return;
176   }
177   if (!_loc) {
178     _loc = "memory/IOBuffer/UNKNOWN-LOCATION";
179   }
180   ResourceTracker::increment(_loc, -index_to_buffer_size(_size_index));
181 }
182 #endif
183 
184 //////////////////////////////////////////////////////////////////
185 //
186 // inline functions definitions
187 //
188 //////////////////////////////////////////////////////////////////
189 //////////////////////////////////////////////////////////////////
190 //
191 //  class IOBufferData --
192 //         inline functions definitions
193 //
194 //////////////////////////////////////////////////////////////////
195 TS_INLINE int64_t
196 IOBufferData::block_size()
197 {
198   return index_to_buffer_size(_size_index);
199 }
200 
201 TS_INLINE IOBufferData *
202 new_IOBufferData_internal(
203 #ifdef TRACK_BUFFER_USER
204   const char *location,
205 #endif
206   void *b, int64_t size, int64_t asize_index)
207 {
208   (void)size;
209   IOBufferData *d = THREAD_ALLOC(ioDataAllocator, this_thread());
210   d->_size_index  = asize_index;
211   ink_assert(BUFFER_SIZE_INDEX_IS_CONSTANT(asize_index) || size <= d->block_size());
212 #ifdef TRACK_BUFFER_USER
213   d->_location = location;
214 #endif
215   d->_data = (char *)b;
216   return d;
217 }
218 
219 TS_INLINE IOBufferData *
220 new_constant_IOBufferData_internal(
221 #ifdef TRACK_BUFFER_USER
222   const char *loc,
223 #endif
224   void *b, int64_t size)
225 {
226   return new_IOBufferData_internal(
227 #ifdef TRACK_BUFFER_USER
228     loc,
229 #endif
230     b, size, BUFFER_SIZE_INDEX_FOR_CONSTANT_SIZE(size));
231 }
232 
233 TS_INLINE IOBufferData *
234 new_xmalloc_IOBufferData_internal(
235 #ifdef TRACK_BUFFER_USER
236   const char *location,
237 #endif
238   void *b, int64_t size)
239 {
240   return new_IOBufferData_internal(
241 #ifdef TRACK_BUFFER_USER
242     location,
243 #endif
244     b, size, BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(size));
245 }
246 
247 TS_INLINE IOBufferData *
248 new_IOBufferData_internal(
249 #ifdef TRACK_BUFFER_USER
250   const char *location,
251 #endif
252   void *b, int64_t size)
253 {
254   return new_IOBufferData_internal(
255 #ifdef TRACK_BUFFER_USER
256     location,
257 #endif
258     b, size, iobuffer_size_to_index(size));
259 }
260 
261 TS_INLINE IOBufferData *
262 new_IOBufferData_internal(
263 #ifdef TRACK_BUFFER_USER
264   const char *loc,
265 #endif
266   int64_t size_index, AllocType type)
267 {
268   IOBufferData *d = THREAD_ALLOC(ioDataAllocator, this_thread());
269 #ifdef TRACK_BUFFER_USER
270   d->_location = loc;
271 #endif
272   d->alloc(size_index, type);
273   return d;
274 }
275 
276 // IRIX has a compiler bug which prevents this function
277 // from being compiled correctly at -O3
278 // so it is DUPLICATED in IOBuffer.cc
279 // ****** IF YOU CHANGE THIS FUNCTION change that one as well.
280 TS_INLINE void
281 IOBufferData::alloc(int64_t size_index, AllocType type)
282 {
283   if (_data) {
284     dealloc();
285   }
286   _size_index = size_index;
287   _mem_type   = type;
288 #ifdef TRACK_BUFFER_USER
289   iobuffer_mem_inc(_location, size_index);
290 #endif
291   switch (type) {
292   case MEMALIGNED:
293     if (BUFFER_SIZE_INDEX_IS_FAST_ALLOCATED(size_index)) {
294       _data = (char *)ioBufAllocator[size_index].alloc_void();
295       // coverity[dead_error_condition]
296     } else if (BUFFER_SIZE_INDEX_IS_XMALLOCED(size_index)) {
297       _data = (char *)ats_memalign(ats_pagesize(), index_to_buffer_size(size_index));
298     }
299     break;
300   default:
301   case DEFAULT_ALLOC:
302     if (BUFFER_SIZE_INDEX_IS_FAST_ALLOCATED(size_index)) {
303       _data = (char *)ioBufAllocator[size_index].alloc_void();
304     } else if (BUFFER_SIZE_INDEX_IS_XMALLOCED(size_index)) {
305       _data = (char *)ats_malloc(BUFFER_SIZE_FOR_XMALLOC(size_index));
306     }
307     break;
308   }
309 }
310 
311 // ****** IF YOU CHANGE THIS FUNCTION change that one as well.
312 
313 TS_INLINE void
314 IOBufferData::dealloc()
315 {
316 #ifdef TRACK_BUFFER_USER
317   iobuffer_mem_dec(_location, _size_index);
318 #endif
319   switch (_mem_type) {
320   case MEMALIGNED:
321     if (BUFFER_SIZE_INDEX_IS_FAST_ALLOCATED(_size_index)) {
322       ioBufAllocator[_size_index].free_void(_data);
323     } else if (BUFFER_SIZE_INDEX_IS_XMALLOCED(_size_index)) {
324       ::free((void *)_data);
325     }
326     break;
327   default:
328   case DEFAULT_ALLOC:
329     if (BUFFER_SIZE_INDEX_IS_FAST_ALLOCATED(_size_index)) {
330       ioBufAllocator[_size_index].free_void(_data);
331     } else if (BUFFER_SIZE_INDEX_IS_XMALLOCED(_size_index)) {
332       ats_free(_data);
333     }
334     break;
335   }
336   _data       = nullptr;
337   _size_index = BUFFER_SIZE_NOT_ALLOCATED;
338   _mem_type   = NO_ALLOC;
339 }
340 
341 TS_INLINE void
342 IOBufferData::free()
343 {
344   dealloc();
345   THREAD_FREE(this, ioDataAllocator, this_thread());
346 }
347 
348 //////////////////////////////////////////////////////////////////
349 //
350 //  class IOBufferBlock --
351 //         inline functions definitions
352 //
353 //////////////////////////////////////////////////////////////////
354 TS_INLINE IOBufferBlock *
355 new_IOBufferBlock_internal(
356 #ifdef TRACK_BUFFER_USER
357   const char *location
358 #endif
359 )
360 {
361   IOBufferBlock *b = THREAD_ALLOC(ioBlockAllocator, this_thread());
362 #ifdef TRACK_BUFFER_USER
363   b->_location = location;
364 #endif
365   return b;
366 }
367 
368 TS_INLINE IOBufferBlock *
369 new_IOBufferBlock_internal(
370 #ifdef TRACK_BUFFER_USER
371   const char *location,
372 #endif
373   IOBufferData *d, int64_t len, int64_t offset)
374 {
375   IOBufferBlock *b = THREAD_ALLOC(ioBlockAllocator, this_thread());
376 #ifdef TRACK_BUFFER_USER
377   b->_location = location;
378 #endif
379   b->set(d, len, offset);
380   return b;
381 }
382 
383 TS_INLINE
384 IOBufferBlock::IOBufferBlock()
385 {
386   return;
387 }
388 
389 TS_INLINE void
390 IOBufferBlock::consume(int64_t len)
391 {
392   _start += len;
393   ink_assert(_start <= _end);
394 }
395 
396 TS_INLINE void
397 IOBufferBlock::fill(int64_t len)
398 {
399   _end += len;
400   ink_assert(_end <= _buf_end);
401 }
402 
403 TS_INLINE void
404 IOBufferBlock::reset()
405 {
406   _end = _start = buf();
407   _buf_end      = buf() + data->block_size();
408 }
409 
410 TS_INLINE void
411 IOBufferBlock::alloc(int64_t i)
412 {
413   ink_assert(BUFFER_SIZE_ALLOCATED(i));
414 #ifdef TRACK_BUFFER_USER
415   data = new_IOBufferData_internal(_location, i);
416 #else
417   data             = new_IOBufferData_internal(i);
418 #endif
419   reset();
420 }
421 
422 TS_INLINE void
423 IOBufferBlock::clear()
424 {
425   data = nullptr;
426 
427   IOBufferBlock *p = next.get();
428   while (p) {
429     // If our block pointer refcount dropped to zero,
430     // recursively free the list.
431     if (p->refcount_dec() == 0) {
432       IOBufferBlock *n = p->next.detach();
433       p->free();
434       p = n;
435     } else {
436       // We don't hold the last refcount, so we are done.
437       break;
438     }
439   }
440 
441   // Nuke the next pointer without dropping the refcount
442   // because we already manually did that.
443   next.detach();
444 
445   _buf_end = _end = _start = nullptr;
446 }
447 
448 TS_INLINE IOBufferBlock *
449 IOBufferBlock::clone() const
450 {
451 #ifdef TRACK_BUFFER_USER
452   IOBufferBlock *b = new_IOBufferBlock_internal(_location);
453 #else
454   IOBufferBlock *b = new_IOBufferBlock_internal();
455 #endif
456   b->data     = data;
457   b->_start   = _start;
458   b->_end     = _end;
459   b->_buf_end = _end;
460 #ifdef TRACK_BUFFER_USER
461   b->_location = _location;
462 #endif
463   return b;
464 }
465 
466 TS_INLINE void
467 IOBufferBlock::dealloc()
468 {
469   clear();
470 }
471 
472 TS_INLINE void
473 IOBufferBlock::free()
474 {
475   dealloc();
476   THREAD_FREE(this, ioBlockAllocator, this_thread());
477 }
478 
479 TS_INLINE void
480 IOBufferBlock::set_internal(void *b, int64_t len, int64_t asize_index)
481 {
482 #ifdef TRACK_BUFFER_USER
483   data = new_IOBufferData_internal(_location, BUFFER_SIZE_NOT_ALLOCATED);
484 #else
485   data             = new_IOBufferData_internal(BUFFER_SIZE_NOT_ALLOCATED);
486 #endif
487   data->_data = (char *)b;
488 #ifdef TRACK_BUFFER_USER
489   iobuffer_mem_inc(_location, asize_index);
490 #endif
491   data->_size_index = asize_index;
492   reset();
493   _end = _start + len;
494 }
495 
496 TS_INLINE void
497 IOBufferBlock::set(IOBufferData *d, int64_t len, int64_t offset)
498 {
499   data     = d;
500   _start   = buf() + offset;
501   _end     = _start + len;
502   _buf_end = buf() + d->block_size();
503 }
504 
505 TS_INLINE void
506 IOBufferBlock::realloc_set_internal(void *b, int64_t buf_size, int64_t asize_index)
507 {
508   int64_t data_size = size();
509   memcpy(b, _start, size());
510   dealloc();
511   set_internal(b, buf_size, asize_index);
512   _end = _start + data_size;
513 }
514 
515 TS_INLINE void
516 IOBufferBlock::realloc(void *b, int64_t buf_size)
517 {
518   realloc_set_internal(b, buf_size, BUFFER_SIZE_NOT_ALLOCATED);
519 }
520 
521 TS_INLINE void
522 IOBufferBlock::realloc_xmalloc(void *b, int64_t buf_size)
523 {
524   realloc_set_internal(b, buf_size, -buf_size);
525 }
526 
527 TS_INLINE void
528 IOBufferBlock::realloc_xmalloc(int64_t buf_size)
529 {
530   realloc_set_internal(ats_malloc(buf_size), buf_size, -buf_size);
531 }
532 
533 TS_INLINE void
534 IOBufferBlock::realloc(int64_t i)
535 {
536   if ((i == data->_size_index) || (i >= (int64_t)countof(ioBufAllocator))) {
537     return;
538   }
539 
540   ink_release_assert(i > data->_size_index && i != BUFFER_SIZE_NOT_ALLOCATED);
541   void *b = ioBufAllocator[i].alloc_void();
542   realloc_set_internal(b, BUFFER_SIZE_FOR_INDEX(i), i);
543 }
544 
545 //////////////////////////////////////////////////////////////////
546 //
547 //  class IOBufferReader --
548 //         inline functions definitions
549 //
550 //////////////////////////////////////////////////////////////////
551 TS_INLINE void
552 IOBufferReader::skip_empty_blocks()
553 {
554   while (block->next && block->next->read_avail() && start_offset >= block->size()) {
555     start_offset -= block->size();
556     block = block->next;
557   }
558 }
559 
560 TS_INLINE bool
561 IOBufferReader::low_water()
562 {
563   return mbuf->low_water();
564 }
565 
566 TS_INLINE bool
567 IOBufferReader::high_water()
568 {
569   return read_avail() >= mbuf->water_mark;
570 }
571 
572 TS_INLINE bool
573 IOBufferReader::current_low_water()
574 {
575   return mbuf->current_low_water();
576 }
577 
578 TS_INLINE IOBufferBlock *
579 IOBufferReader::get_current_block()
580 {
581   return block.get();
582 }
583 
584 TS_INLINE char *
585 IOBufferReader::start()
586 {
587   if (!block) {
588     return nullptr;
589   }
590 
591   skip_empty_blocks();
592   return block->start() + start_offset;
593 }
594 
595 TS_INLINE char *
596 IOBufferReader::end()
597 {
598   if (!block) {
599     return nullptr;
600   }
601 
602   skip_empty_blocks();
603   return block->end();
604 }
605 
606 TS_INLINE int64_t
607 IOBufferReader::block_read_avail()
608 {
609   if (!block) {
610     return 0;
611   }
612 
613   skip_empty_blocks();
614   return (int64_t)(block->end() - (block->start() + start_offset));
615 }
616 
617 inline std::string_view
618 IOBufferReader::block_read_view()
619 {
620   const char *start = this->start(); // empty blocks are skipped in here.
621   return start ? std::string_view{start, static_cast<size_t>(block->end() - start)} : std::string_view{};
622 }
623 
624 TS_INLINE int
625 IOBufferReader::block_count()
626 {
627   int count        = 0;
628   IOBufferBlock *b = block.get();
629 
630   while (b) {
631     count++;
632     b = b->next.get();
633   }
634 
635   return count;
636 }
637 
638 TS_INLINE int64_t
639 IOBufferReader::read_avail()
640 {
641   int64_t t        = 0;
642   IOBufferBlock *b = block.get();
643 
644   while (b) {
645     t += b->read_avail();
646     b = b->next.get();
647   }
648 
649   t -= start_offset;
650   if (size_limit != INT64_MAX && t > size_limit) {
651     t = size_limit;
652   }
653 
654   return t;
655 }
656 
657 TS_INLINE bool
658 IOBufferReader::is_read_avail_more_than(int64_t size)
659 {
660   int64_t t        = -start_offset;
661   IOBufferBlock *b = block.get();
662 
663   while (b) {
664     t += b->read_avail();
665     if (t > size) {
666       return true;
667     }
668     b = b->next.get();
669   }
670   return false;
671 }
672 
673 TS_INLINE void
674 IOBufferReader::consume(int64_t n)
675 {
676   start_offset += n;
677   if (size_limit != INT64_MAX) {
678     size_limit -= n;
679   }
680 
681   ink_assert(size_limit >= 0);
682   if (!block) {
683     return;
684   }
685 
686   int64_t r = block->read_avail();
687   int64_t s = start_offset;
688   while (r <= s && block->next && block->next->read_avail()) {
689     s -= r;
690     start_offset = s;
691     block        = block->next;
692     r            = block->read_avail();
693   }
694 }
695 
696 TS_INLINE char &IOBufferReader::operator[](int64_t i)
697 {
698   static char default_ret = '\0'; // This is just to avoid compiler warnings...
699   IOBufferBlock *b        = block.get();
700 
701   i += start_offset;
702   while (b) {
703     int64_t bytes = b->read_avail();
704     if (bytes > i) {
705       return b->start()[i];
706     }
707     i -= bytes;
708     b = b->next.get();
709   }
710 
711   ink_release_assert(!"out of range");
712   // Never used, just to satisfy compilers not understanding the fatality of ink_release_assert().
713   return default_ret;
714 }
715 
716 TS_INLINE void
717 IOBufferReader::clear()
718 {
719   accessor     = nullptr;
720   block        = nullptr;
721   mbuf         = nullptr;
722   start_offset = 0;
723   size_limit   = INT64_MAX;
724 }
725 
726 TS_INLINE void
727 IOBufferReader::reset()
728 {
729   block        = mbuf->_writer;
730   start_offset = 0;
731   size_limit   = INT64_MAX;
732 }
733 
734 ////////////////////////////////////////////////////////////////
735 //
736 //  class MIOBuffer --
737 //      inline functions definitions
738 //
739 ////////////////////////////////////////////////////////////////
740 inkcoreapi extern ClassAllocator<MIOBuffer> ioAllocator;
741 ////////////////////////////////////////////////////////////////
742 //
743 //  MIOBuffer::MIOBuffer()
744 //
745 //  This constructor accepts a pre-allocated memory buffer,
746 //  wraps if in a IOBufferData and IOBufferBlock structures
747 //  and sets it as the current block.
748 //  NOTE that in this case the memory buffer will not be freed
749 //  by the MIOBuffer class. It is the user responsibility to
750 //  free the memory buffer. The wrappers (MIOBufferBlock and
751 //  MIOBufferData) will be freed by this class.
752 //
753 ////////////////////////////////////////////////////////////////
754 TS_INLINE
755 MIOBuffer::MIOBuffer(void *b, int64_t bufsize, int64_t aWater_mark)
756 {
757 #ifdef TRACK_BUFFER_USER
758   _location = nullptr;
759 #endif
760   set(b, bufsize);
761   water_mark = aWater_mark;
762   size_index = BUFFER_SIZE_NOT_ALLOCATED;
763   return;
764 }
765 
766 TS_INLINE
767 MIOBuffer::MIOBuffer(int64_t default_size_index)
768 {
769   clear();
770   size_index = default_size_index;
771 #ifdef TRACK_BUFFER_USER
772   _location = nullptr;
773 #endif
774   return;
775 }
776 
777 TS_INLINE
778 MIOBuffer::MIOBuffer()
779 {
780   clear();
781 #ifdef TRACK_BUFFER_USER
782   _location = nullptr;
783 #endif
784   return;
785 }
786 
787 TS_INLINE
788 MIOBuffer::~MIOBuffer()
789 {
790   _writer = nullptr;
791   dealloc_all_readers();
792 }
793 
794 TS_INLINE MIOBuffer *
795 new_MIOBuffer_internal(
796 #ifdef TRACK_BUFFER_USER
797   const char *location,
798 #endif
799   int64_t size_index)
800 {
801   MIOBuffer *b = THREAD_ALLOC(ioAllocator, this_thread());
802 #ifdef TRACK_BUFFER_USER
803   b->_location = location;
804 #endif
805   b->alloc(size_index);
806   b->water_mark = 0;
807   return b;
808 }
809 
810 TS_INLINE void
811 free_MIOBuffer(MIOBuffer *mio)
812 {
813   mio->_writer = nullptr;
814   mio->dealloc_all_readers();
815   THREAD_FREE(mio, ioAllocator, this_thread());
816 }
817 
818 TS_INLINE MIOBuffer *
819 new_empty_MIOBuffer_internal(
820 #ifdef TRACK_BUFFER_USER
821   const char *location,
822 #endif
823   int64_t size_index)
824 {
825   MIOBuffer *b  = THREAD_ALLOC(ioAllocator, this_thread());
826   b->size_index = size_index;
827   b->water_mark = 0;
828 #ifdef TRACK_BUFFER_USER
829   b->_location = location;
830 #endif
831   return b;
832 }
833 
834 TS_INLINE void
835 free_empty_MIOBuffer(MIOBuffer *mio)
836 {
837   THREAD_FREE(mio, ioAllocator, this_thread());
838 }
839 
840 TS_INLINE IOBufferReader *
841 MIOBuffer::alloc_accessor(MIOBufferAccessor *anAccessor)
842 {
843   int i;
844   for (i = 0; i < MAX_MIOBUFFER_READERS; i++) {
845     if (!readers[i].allocated()) {
846       break;
847     }
848   }
849 
850   // TODO refactor code to return nullptr at some point
851   ink_release_assert(i < MAX_MIOBUFFER_READERS);
852 
853   IOBufferReader *e = &readers[i];
854   e->mbuf           = this;
855   e->reset();
856   e->accessor = anAccessor;
857 
858   return e;
859 }
860 
861 TS_INLINE IOBufferReader *
862 MIOBuffer::alloc_reader()
863 {
864   int i;
865   for (i = 0; i < MAX_MIOBUFFER_READERS; i++) {
866     if (!readers[i].allocated()) {
867       break;
868     }
869   }
870 
871   // TODO refactor code to return nullptr at some point
872   ink_release_assert(i < MAX_MIOBUFFER_READERS);
873 
874   IOBufferReader *e = &readers[i];
875   e->mbuf           = this;
876   e->reset();
877   e->accessor = nullptr;
878 
879   return e;
880 }
881 
882 TS_INLINE int64_t
883 MIOBuffer::block_size()
884 {
885   return index_to_buffer_size(size_index);
886 }
887 TS_INLINE IOBufferReader *
888 MIOBuffer::clone_reader(IOBufferReader *r)
889 {
890   int i;
891   for (i = 0; i < MAX_MIOBUFFER_READERS; i++) {
892     if (!readers[i].allocated()) {
893       break;
894     }
895   }
896 
897   // TODO refactor code to return nullptr at some point
898   ink_release_assert(i < MAX_MIOBUFFER_READERS);
899 
900   IOBufferReader *e = &readers[i];
901   e->mbuf           = this;
902   e->accessor       = nullptr;
903   e->block          = r->block;
904   e->start_offset   = r->start_offset;
905   e->size_limit     = r->size_limit;
906   ink_assert(e->size_limit >= 0);
907 
908   return e;
909 }
910 
911 TS_INLINE int64_t
912 MIOBuffer::block_write_avail()
913 {
914   IOBufferBlock *b = first_write_block();
915   return b ? b->write_avail() : 0;
916 }
917 
918 ////////////////////////////////////////////////////////////////
919 //
920 //  MIOBuffer::append_block()
921 //
922 //  Appends a block to writer->next and make it the current
923 //  block.
924 //  Note that the block is not appended to the end of the list.
925 //  That means that if writer->next was not null before this
926 //  call then the block that writer->next was pointing to will
927 //  have its reference count decremented and writer->next
928 //  will have a new value which is the new block.
929 //  In any case the new appended block becomes the current
930 //  block.
931 //
932 ////////////////////////////////////////////////////////////////
933 TS_INLINE void
934 MIOBuffer::append_block_internal(IOBufferBlock *b)
935 {
936   // It would be nice to remove an empty buffer at the beginning,
937   // but this breaks HTTP.
938   // if (!_writer || !_writer->read_avail())
939   if (!_writer) {
940     _writer = b;
941     init_readers();
942   } else {
943     ink_assert(!_writer->next || !_writer->next->read_avail());
944     _writer->next = b;
945     while (b->read_avail()) {
946       _writer = b;
947       b       = b->next.get();
948       if (!b) {
949         break;
950       }
951     }
952   }
953   while (_writer->next && !_writer->write_avail() && _writer->next->read_avail()) {
954     _writer = _writer->next;
955   }
956 }
957 
958 TS_INLINE void
959 MIOBuffer::append_block(IOBufferBlock *b)
960 {
961   ink_assert(b->read_avail());
962   append_block_internal(b);
963 }
964 
965 ////////////////////////////////////////////////////////////////
966 //
967 //  MIOBuffer::append_block()
968 //
969 //  Allocate a block, appends it to current->next
970 //  and make the new block the current block (writer).
971 //
972 ////////////////////////////////////////////////////////////////
973 TS_INLINE void
974 MIOBuffer::append_block(int64_t asize_index)
975 {
976   ink_assert(BUFFER_SIZE_ALLOCATED(asize_index));
977 #ifdef TRACK_BUFFER_USER
978   IOBufferBlock *b = new_IOBufferBlock_internal(_location);
979 #else
980   IOBufferBlock *b = new_IOBufferBlock_internal();
981 #endif
982   b->alloc(asize_index);
983   append_block_internal(b);
984   return;
985 }
986 
987 TS_INLINE void
988 MIOBuffer::add_block()
989 {
990   append_block(size_index);
991 }
992 
993 TS_INLINE void
994 MIOBuffer::check_add_block()
995 {
996   if (!high_water() && current_low_water()) {
997     add_block();
998   }
999 }
1000 
1001 TS_INLINE IOBufferBlock *
1002 MIOBuffer::get_current_block()
1003 {
1004   return first_write_block();
1005 }
1006 
1007 //////////////////////////////////////////////////////////////////
1008 //
1009 //  MIOBuffer::current_write_avail()
1010 //
1011 //  returns the total space available in all blocks.
1012 //  This function is different than write_avail() because
1013 //  it will not append a new block if there is no space
1014 //  or below the watermark space available.
1015 //
1016 //////////////////////////////////////////////////////////////////
1017 TS_INLINE int64_t
1018 MIOBuffer::current_write_avail()
1019 {
1020   int64_t t        = 0;
1021   IOBufferBlock *b = _writer.get();
1022   while (b) {
1023     t += b->write_avail();
1024     b = b->next.get();
1025   }
1026   return t;
1027 }
1028 
1029 //////////////////////////////////////////////////////////////////
1030 //
1031 //  MIOBuffer::write_avail()
1032 //
1033 //  returns the number of bytes available in the current block.
1034 //  If there is no current block or not enough free space in
1035 //  the current block then a new block is appended.
1036 //
1037 //////////////////////////////////////////////////////////////////
1038 TS_INLINE int64_t
1039 MIOBuffer::write_avail()
1040 {
1041   check_add_block();
1042   return current_write_avail();
1043 }
1044 
1045 TS_INLINE void
1046 MIOBuffer::fill(int64_t len)
1047 {
1048   int64_t f = _writer->write_avail();
1049   while (f < len) {
1050     _writer->fill(f);
1051     len -= f;
1052     if (len > 0) {
1053       _writer = _writer->next;
1054     }
1055     f = _writer->write_avail();
1056   }
1057   _writer->fill(len);
1058 }
1059 
1060 TS_INLINE int
1061 MIOBuffer::max_block_count()
1062 {
1063   int maxb = 0;
1064   for (auto &reader : readers) {
1065     if (reader.allocated()) {
1066       int c = reader.block_count();
1067       if (c > maxb) {
1068         maxb = c;
1069       }
1070     }
1071   }
1072   return maxb;
1073 }
1074 
1075 TS_INLINE int64_t
1076 MIOBuffer::max_read_avail()
1077 {
1078   int64_t s = 0;
1079   int found = 0;
1080   for (auto &reader : readers) {
1081     if (reader.allocated()) {
1082       int64_t ss = reader.read_avail();
1083       if (ss > s) {
1084         s = ss;
1085       }
1086       found = 1;
1087     }
1088   }
1089   if (!found && _writer) {
1090     return _writer->read_avail();
1091   }
1092   return s;
1093 }
1094 
1095 TS_INLINE void
1096 MIOBuffer::set(void *b, int64_t len)
1097 {
1098 #ifdef TRACK_BUFFER_USER
1099   _writer = new_IOBufferBlock_internal(_location);
1100 #else
1101   _writer          = new_IOBufferBlock_internal();
1102 #endif
1103   _writer->set_internal(b, len, BUFFER_SIZE_INDEX_FOR_CONSTANT_SIZE(len));
1104   init_readers();
1105 }
1106 
1107 TS_INLINE void
1108 MIOBuffer::set_xmalloced(void *b, int64_t len)
1109 {
1110 #ifdef TRACK_BUFFER_USER
1111   _writer = new_IOBufferBlock_internal(_location);
1112 #else
1113   _writer          = new_IOBufferBlock_internal();
1114 #endif
1115   _writer->set_internal(b, len, BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(len));
1116   init_readers();
1117 }
1118 
1119 TS_INLINE void
1120 MIOBuffer::append_xmalloced(void *b, int64_t len)
1121 {
1122 #ifdef TRACK_BUFFER_USER
1123   IOBufferBlock *x = new_IOBufferBlock_internal(_location);
1124 #else
1125   IOBufferBlock *x = new_IOBufferBlock_internal();
1126 #endif
1127   x->set_internal(b, len, BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(len));
1128   append_block_internal(x);
1129 }
1130 
1131 TS_INLINE void
1132 MIOBuffer::append_fast_allocated(void *b, int64_t len, int64_t fast_size_index)
1133 {
1134 #ifdef TRACK_BUFFER_USER
1135   IOBufferBlock *x = new_IOBufferBlock_internal(_location);
1136 #else
1137   IOBufferBlock *x = new_IOBufferBlock_internal();
1138 #endif
1139   x->set_internal(b, len, fast_size_index);
1140   append_block_internal(x);
1141 }
1142 
1143 TS_INLINE void
1144 MIOBuffer::alloc(int64_t i)
1145 {
1146 #ifdef TRACK_BUFFER_USER
1147   _writer = new_IOBufferBlock_internal(_location);
1148 #else
1149   _writer          = new_IOBufferBlock_internal();
1150 #endif
1151   _writer->alloc(i);
1152   size_index = i;
1153   init_readers();
1154 }
1155 
1156 TS_INLINE void
1157 MIOBuffer::alloc_xmalloc(int64_t buf_size)
1158 {
1159   char *b = (char *)ats_malloc(buf_size);
1160   set_xmalloced(b, buf_size);
1161 }
1162 
1163 TS_INLINE void
1164 MIOBuffer::dealloc_reader(IOBufferReader *e)
1165 {
1166   if (e->accessor) {
1167     ink_assert(e->accessor->writer() == this);
1168     ink_assert(e->accessor->reader() == e);
1169     e->accessor->clear();
1170   }
1171   e->clear();
1172 }
1173 
1174 TS_INLINE IOBufferReader *
1175 IOBufferReader::clone()
1176 {
1177   return mbuf->clone_reader(this);
1178 }
1179 
1180 TS_INLINE void
1181 IOBufferReader::dealloc()
1182 {
1183   mbuf->dealloc_reader(this);
1184 }
1185 
1186 TS_INLINE void
1187 MIOBuffer::dealloc_all_readers()
1188 {
1189   for (auto &reader : readers) {
1190     if (reader.allocated()) {
1191       dealloc_reader(&reader);
1192     }
1193   }
1194 }
1195 
1196 TS_INLINE void
1197 MIOBuffer::set_size_index(int64_t size)
1198 {
1199   size_index = iobuffer_size_to_index(size);
1200 }
1201 
1202 TS_INLINE void
1203 MIOBufferAccessor::reader_for(MIOBuffer *abuf)
1204 {
1205   mbuf = abuf;
1206   if (abuf) {
1207     entry = mbuf->alloc_accessor(this);
1208   } else {
1209     entry = nullptr;
1210   }
1211 }
1212 
1213 TS_INLINE void
1214 MIOBufferAccessor::reader_for(IOBufferReader *areader)
1215 {
1216   if (entry == areader) {
1217     return;
1218   }
1219   mbuf  = areader->mbuf;
1220   entry = areader;
1221   ink_assert(mbuf);
1222 }
1223 
1224 TS_INLINE void
1225 MIOBufferAccessor::writer_for(MIOBuffer *abuf)
1226 {
1227   mbuf  = abuf;
1228   entry = nullptr;
1229 }
1230 
1231 TS_INLINE
1232 MIOBufferAccessor::~MIOBufferAccessor() {}
1233