Something went wrong. Try again.
An experimental ocaml library bindings for simdutf https://github.com/simdutf/simdutf
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118(* Heap allocation per call, in words. Requires native code with cross-module inlining, so that optional arguments are not boxed at call sites. *)
open Simdutfmodule B = Bigstring
let iterations = 10_000let failures = ref 0
let words_per_call f = ignore (Sys.opaque_identity (f ())); let before = Gc.minor_words () in for _ = 1 to iterations do ignore (Sys.opaque_identity (f ())) done; (Gc.minor_words () -. before) /. float iterations
let expect name ~words f = let actual = words_per_call f in (* Tolerates the boxed floats of the measurement itself. *) if Float.abs (actual -. float words) > 0.01 then ( incr failures; Printf.printf "FAIL %s: %.3f words per call, expected %d\n" name actual words) else Printf.printf "ok %s: %d words\n" name words
(* Words of a string of [n] bytes, including its header. *)let string_words n = (n / (Sys.word_size / 8)) + 2let ok_block = 2
let () = let text = String.make 7 '!' ^ String.make 100 'a' ^ "\xff" in let off = Sys.opaque_identity 7 and len = Sys.opaque_identity 100 in let wide = String.make 200 '\x00' in let encoded = "\r\n" ^ String.concat "\r\n" (List.init 5 (fun _ -> Base64.encode (String.make 54 'x'))) in let decoded_len = Base64.decoded_length encoded in let output = Bytes.create decoded_len in let big_text = B.of_string text and big_encoded = B.of_string encoded in let large = B.of_string (String.make 100_000 'a') in let valid = String.sub text 0 107 in expect "validate" ~words:0 (fun () -> validate valid); expect "validate ~off ~len" ~words:0 (fun () -> validate ~off ~len text); expect "validate invalid (error record)" ~words:6 (fun () -> validate ~off ~len:101 text); expect "validate UTF-16LE ~off ~len" ~words:0 (fun () -> validate ~encoding:Utf16le ~off:2 ~len:100 wide); expect "is_valid ~off ~len" ~words:0 (fun () -> is_valid ~off ~len text); expect "is_valid ~encoding ~off ~len" ~words:0 (fun () -> is_valid ~encoding:Utf32le ~off:4 ~len:100 wide); expect "validate_ascii ~off ~len" ~words:0 (fun () -> validate_ascii ~off ~len text); expect "is_ascii ~off ~len" ~words:0 (fun () -> is_ascii ~off ~len text); expect "count_codepoints ~off ~len (Ok block)" ~words:ok_block (fun () -> count_codepoints ~off ~len text); expect "find ~off ~len" ~words:0 (fun () -> find ~off ~len text '\xff'); expect "find_utf16 ~off ~len" ~words:0 (fun () -> find_utf16 ~encoding:Utf16le ~off:1 ~len:100 wide 0x0a); expect "Bigstring.find ~off ~len" ~words:0 (fun () -> B.find ~off ~len big_text 'z'); expect "Bigstring.find_utf16, lock released" ~words:0 (fun () -> B.find_utf16 ~encoding:Utf16be large 0x0a); expect "Base64.decoded_length" ~words:0 (fun () -> Base64.decoded_length encoded); expect "Base64.decoded_length ~off ~len" ~words:0 (fun () -> Base64.decoded_length ~off:2 ~len:(String.length encoded - 2) encoded); expect "Base64.decode_into (Ok block)" ~words:ok_block (fun () -> Base64.decode_into ~off:2 ~output encoded); expect "Base64.decode_into ~output_off ~output_len (Ok block)" ~words:ok_block (fun () -> Base64.decode_into ~off:2 ~len:(String.length encoded - 2) ~output_off:0 ~output_len:decoded_len ~output encoded); expect "Base64.decode (exact output and Ok block)" ~words:(string_words decoded_len + ok_block) (fun () -> Base64.decode ~off:2 encoded); expect "Bigstring.validate ~off ~len" ~words:0 (fun () -> B.validate ~off ~len big_text); expect "Bigstring.is_valid ~off ~len" ~words:0 (fun () -> B.is_valid ~off ~len big_text); expect "Bigstring.is_valid, lock released" ~words:0 (fun () -> B.is_valid large); expect "Bigstring.Base64.decoded_length ~off" ~words:0 (fun () -> B.Base64.decoded_length ~off:2 big_encoded); expect "Bigstring.Base64.decode_to_bytes (Ok block)" ~words:ok_block (fun () -> B.Base64.decode_to_bytes ~off:2 ~output big_encoded); expect "Bigstring.sub_string (exact string)" ~words:(string_words 100) (fun () -> B.sub_string ~off ~len big_text); (* Large results live in the major heap. *) let raw = String.init (1 lsl 20) (fun i -> Char.chr (i land 255)) in let encoded = let e = Base64.encode raw in "!" ^ String.concat "\r\n" (List.init ((String.length e / 76) + 1) (fun i -> String.sub e (i * 76) (min 76 (String.length e - (i * 76))))) in let before = Gc.allocated_bytes () in let decoded = Sys.opaque_identity (Base64.decode_exn ~off:1 encoded) in let allocated = Gc.allocated_bytes () -. before in let exact = float (string_words (String.length raw) * (Sys.word_size / 8)) in assert (decoded = raw); if allocated > exact +. 256. then ( incr failures; Printf.printf "FAIL Base64.decode 1 MiB: %.0f bytes allocated, output is %.0f\n" allocated exact) else Printf.printf "ok Base64.decode 1 MiB: %.0f bytes allocated, output is %.0f\n" allocated exact; if !failures > 0 then exit 1