Something went wrong. Try again.
Sliding byte-reader window
Something went wrong. Try again.
7.7 kB · 199 lines
OCaml
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200(*--------------------------------------------------------------------------- Copyright (c) 2026 Thomas Gazagnaire. All rights reserved. SPDX-License-Identifier: ISC ---------------------------------------------------------------------------*)
(** Tests for {!Bytecursor}. The properties worth pinning are the ones a parser built on it relies on and cannot check for itself: the window stays the size it was asked for however the reader slices the input, a stream offset means the same thing before and after the window slides, and the observer is handed every byte exactly once in order. *)
module Reader = Bytesrw.Bytes.Readermodule Slice = Bytesrw.Bytes.Slice
let doc n = String.init n (fun i -> Char.chr (Char.code 'a' + (i mod 26)))
(* A reader handing out slices of exactly [n] bytes, which is how a slice boundary is made to fall in the middle of whatever is being read. *)let chopped n s = let i = ref 0 in Reader.make ~slice_length:n (fun () -> if !i >= String.length s then Slice.eod else begin let length = Int.min n (String.length s - !i) in let slice = Slice.make (Bytes.of_string (String.sub s !i length)) ~first:0 ~length in i := !i + length; slice end)
(* Read the whole stream one byte at a time through the cursor. *)let drain c = let b = Buffer.create 64 in let rec loop () = Bytecursor.ensure c 1; if Bytecursor.available c > 0 then begin Buffer.add_char b (Bytecursor.get c (Bytecursor.pos c)); Bytecursor.skip c 1; loop () end in loop (); Buffer.contents b
(* Whatever the reader's slice length, the cursor delivers the stream. *)let test_reads_the_whole_stream () = let s = doc 5000 in List.iter (fun slice_length -> let c = Bytecursor.v ~size:64 (chopped slice_length s) in Alcotest.(check string) (Fmt.str "slices of %d" slice_length) s (drain c)) [ 1; 3; 64; 65; 4096; 100_000 ]
(* The window is the size it was asked for, not the size of the slices offered to it: a reader over a whole string offers the whole string. *)let test_window_stays_bounded () = let s = doc 200_000 in let c = Bytecursor.v ~size:512 (Reader.of_string s) in let widest = ref 0 in let rec loop () = Bytecursor.ensure c 1; if Bytecursor.length c > !widest then widest := Bytecursor.length c; if Bytecursor.available c > 0 then begin Bytecursor.skip c (Bytecursor.available c); loop () end in loop (); Alcotest.(check bool) (Fmt.str "widest window %d over %d bytes" !widest (String.length s)) true (!widest <= 512)
(* A stream offset survives the window sliding: [discarded + pos] names the same byte throughout, which is the only coordinate an error location can use. *)let test_stream_offset_is_stable () = let s = doc 10_000 in let c = Bytecursor.v ~size:128 (chopped 37 s) in let n = ref 0 in let rec loop () = Bytecursor.ensure c 1; if Bytecursor.available c > 0 then begin let offset = Bytecursor.discarded c + Bytecursor.pos c in Alcotest.(check int) "stream offset" !n offset; Alcotest.(check char) "byte at offset" s.[!n] (Bytecursor.get c (Bytecursor.pos c)); incr n; Bytecursor.skip c 1; loop () end in loop (); Alcotest.(check int) "bytes seen" (String.length s) !n
(* The observer is the hook a streaming validator hangs on, so it must see every byte of the stream, once, in order -- not once per refill round and not again for what a push back replays. *)let test_observer_sees_every_byte_once () = let s = doc 30_000 in let seen = Buffer.create 64 in let on_bytes b first length = Buffer.add_subbytes seen b first length in let c = Bytecursor.v ~size:256 ~on_bytes (Reader.of_string s) in let got = drain c in Alcotest.(check string) "stream" s got; Alcotest.(check string) "observed" s (Buffer.contents seen)
(* What the reader reports having handed out is what the cursor took: the remainder of an oversized slice goes back rather than into the window. *)let test_reader_is_not_over_pulled () = let s = doc 100_000 in let r = Reader.of_string s in let c = Bytecursor.v ~size:1024 r in Bytecursor.ensure c 16; Alcotest.(check int) "pulled" 1024 (Reader.pos r); Bytecursor.skip c 1024; Bytecursor.ensure c 1; Alcotest.(check int) "pulled after a second fill" 2048 (Reader.pos r)
(* [ensure] widens the window past its usual size when asked for more than it holds, and stops asking at [max_size] rather than spinning against a reader whose bytes it cannot take. *)let test_ensure_grows_and_stops () = let s = doc 100_000 in let c = Bytecursor.v ~size:64 (chopped 100 s) in Bytecursor.ensure c 5000; Alcotest.(check int) "available" 5000 (Bytecursor.available c); Alcotest.(check string) "content" (String.sub s 0 5000) (Bytecursor.sub_string c ~first:(Bytecursor.pos c) ~length:5000); let capped = Bytecursor.v ~size:64 ~max_size:1000 (chopped 100 s) in Bytecursor.ensure capped 5000; Alcotest.(check int) "capped available" 1000 (Bytecursor.available capped); Alcotest.(check bool) "not at end of input" false (Bytecursor.eof capped)
(* At end of input a request goes unmet rather than blocking, and [eof] is what says so. *)let test_short_at_end_of_input () = let c = Bytecursor.v ~size:64 (chopped 3 "abcde") in Bytecursor.ensure c 100; Alcotest.(check bool) "eof" true (Bytecursor.eof c); Alcotest.(check int) "available" 5 (Bytecursor.available c); Bytecursor.skip c 5; Bytecursor.ensure c 1; Alcotest.(check int) "drained" 0 (Bytecursor.available c)
(* [ensure_all] holds the rest of the stream whatever the window was sized for, which is what a caller needing the remaining input as one string asks for. *)let test_ensure_all () = let s = doc 70_000 in let c = Bytecursor.v ~size:64 (chopped 997 s) in Bytecursor.ensure c 1; Bytecursor.skip c 1; Bytecursor.ensure_all c; Alcotest.(check bool) "eof" true (Bytecursor.eof c); Alcotest.(check string) "rest of the stream" (String.sub s 1 (String.length s - 1)) (Bytecursor.sub_string c ~first:(Bytecursor.pos c) ~length:(Bytecursor.available c))
(* A token straddling a refill is kept by copying it out before the window slides, which is the one thing a caller has to do for itself. *)let test_token_across_a_refill () = let s = doc 5000 in let c = Bytecursor.v ~size:64 (chopped 17 s) in let b = Buffer.create 64 in let rec loop () = Bytecursor.ensure c 1; let n = Bytecursor.available c in if n > 0 then begin Bytecursor.add_to_buffer c b ~first:(Bytecursor.pos c) ~length:n; Bytecursor.skip c n; loop () end in loop (); Alcotest.(check string) "reassembled" s (Buffer.contents b)
let suite = ( "bytecursor", [ Alcotest.test_case "reads the whole stream" `Quick test_reads_the_whole_stream; Alcotest.test_case "window stays bounded" `Quick test_window_stays_bounded; Alcotest.test_case "stream offset is stable" `Quick test_stream_offset_is_stable; Alcotest.test_case "observer sees every byte once" `Quick test_observer_sees_every_byte_once; Alcotest.test_case "reader is not over-pulled" `Quick test_reader_is_not_over_pulled; Alcotest.test_case "ensure grows and stops" `Quick test_ensure_grows_and_stops; Alcotest.test_case "short at end of input" `Quick test_short_at_end_of_input; Alcotest.test_case "ensure_all" `Quick test_ensure_all; Alcotest.test_case "token across a refill" `Quick test_token_across_a_refill; ] )