Something went wrong. Try again.
Sliding byte-reader window
Something went wrong. Try again.
4.4 kB · 125 lines
OCaml
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126(*--------------------------------------------------------------------------- Copyright (c) 2026 Thomas Gazagnaire. All rights reserved. SPDX-License-Identifier: ISC ---------------------------------------------------------------------------*)
open Bytesrw
let default_size = 8192
type t = { reader : Bytes.Reader.t; mutable buf : bytes; mutable pos : int; mutable len : int; mutable eof : bool; mutable discarded : int; max_size : int; on_bytes : bytes -> int -> int -> unit;}
let no_observer _ _ _ = ()
let v ?(size = default_size) ?(max_size = Sys.max_string_length) ?(on_bytes = no_observer) reader = let size = max 1 size in { reader; buf = Stdlib.Bytes.create size; pos = 0; len = 0; eof = false; discarded = 0; max_size = max size max_size; on_bytes; }
let[@inline] pos c = c.poslet[@inline] length c = c.lenlet[@inline] available c = c.len - c.poslet[@inline] discarded c = c.discardedlet[@inline] eof c = c.eoflet[@inline] get c i = Stdlib.Bytes.get c.buf ilet sub_string c ~first ~length = Stdlib.Bytes.sub_string c.buf first lengthlet add_to_buffer c b ~first ~length = Buffer.add_subbytes b c.buf first lengthlet[@inline] skip c n = c.pos <- c.pos + n
(* Widen the buffer to hold [need] bytes, doubling so that a parser asking for one more byte at a time does not copy the window on each. Capped at [max_size]: past that the request goes unmet, which callers already have to handle because end of input meets requests the same way. *)let grow c need = let size = ref (Stdlib.Bytes.length c.buf) in while !size < need && !size < c.max_size do size := !size * 2 done; let size = min (max !size need) c.max_size in if size > Stdlib.Bytes.length c.buf then begin let b = Stdlib.Bytes.create size in Stdlib.Bytes.blit c.buf 0 b 0 c.len; c.buf <- b end
(* Compact the window onto the bytes not yet read, then append one slice.
What does not fit is pushed back rather than making the window grow to accommodate it: a reader over a whole string offers the whole string as one slice, and a cursor that took it would hold the document -- the thing the window exists not to do. bytesrw's [push_back] rewinds the reader's position and does not replay through a [tap], so what a reader reports having handed out stays equal to what the cursor took. *)let refill c = if not c.eof then begin if c.pos > 0 then begin let remaining = c.len - c.pos in if remaining > 0 then Stdlib.Bytes.blit c.buf c.pos c.buf 0 remaining; c.discarded <- c.discarded + c.pos; c.pos <- 0; c.len <- remaining end; let slice = Bytes.Reader.read c.reader in if Bytes.Slice.is_eod slice then c.eof <- true else begin let src = Bytes.Slice.bytes slice in let first = Bytes.Slice.first slice in let n = Bytes.Slice.length slice in let take = min n (Stdlib.Bytes.length c.buf - c.len) in if take > 0 then begin Stdlib.Bytes.blit src first c.buf c.len take; c.on_bytes c.buf c.len take; c.len <- c.len + take end; if take < n then Bytes.Reader.push_back c.reader (Bytes.Slice.make src ~first:(first + take) ~length:(n - take)) end end
let ensure_slow c n = begin if n > Stdlib.Bytes.length c.buf then grow c n; (* A refill that adds nothing without reporting end of input means the window is at [max_size] and the reader still has bytes: stop rather than spin, and let the caller read the short window as it reads a short one at end of input. *) let progressing = ref true in while !progressing && c.len - c.pos < n && not c.eof do let before = c.len - c.pos in refill c; progressing := c.len - c.pos > before done end
(* Split so that the test a parser makes on every byte is a pair of field loads the caller can inline, and the refill it almost never reaches is not. *)let[@inline] ensure c n = if c.len - c.pos < n && not c.eof then ensure_slow c n
let ensure_all c = let progressing = ref true in while !progressing && not c.eof do let before = c.len - c.pos in if c.len = Stdlib.Bytes.length c.buf && c.pos = 0 then grow c (2 * Stdlib.Bytes.length c.buf); refill c; progressing := c.eof || c.len - c.pos > before done