simdjsont #
OCaml bindings to simdjson, a JSON parser that uses SIMD instructions.
This library includes a vendored copy of simdjson 5.0.1, which is Copyright 2018-2026 The simdjson authors and licensed under the Apache License 2.0. Source provenance and checksums are recorded in lib/simdjson_vendor.md.
Installation #
System requirements #
This package builds vendored C++ code (simdjson) via Dune foreign_stubs, so you need a C++17-capable compiler toolchain available on your system.
Opam #
opam install simdjsont
Usage #
Quick Start #
(* Validate JSON *)
let () = assert (Simdjsont.Validate.is_valid {|{"name": "Alice", "age": 30}|})
(* Extract a value by path *)
let name = Simdjsont.Extract.string {|{"user": {"name": "Bob"}}|} ~pointer:"/user/name"
(* name = Ok "Bob" *)
(* Decode to a typed value *)
let numbers = Simdjsont.Decode.decode_string Simdjsont.Decode.(list int) "[1, 2, 3]"
(* numbers = Ok [1; 2; 3] *)
(* Encode to JSON string *)
let json = Simdjsont.Encode.to_string Simdjsont.Decode.(list string) ["a"; "b"; "c"]
(* json = {|["a","b","c"]|} *)
(* Work directly with bigstrings (zero-copy, no string round-trip) *)
let buf = Simdjsont.bigstring_of_string "[1, 2, 3]" in
let nums = Simdjsont.decode_bigstring Simdjsont.Decode.(list int) buf ~len:9
(* nums = Ok [1; 2; 3] *)
(* Decode an NDJSON / JSON-Lines stream into typed values *)
let events =
Simdjsont.Ndjson.decode_string_seq Simdjsont.Decode.int "1\n2\n3\n"
|> Seq.filter_map Result.to_option |> List.of_seq
(* events = [1; 2; 3] *)
Validation #
(* Check if JSON is valid *)
let () =
if Simdjsont.Validate.is_valid "[1, 2, 3]" then
print_endline "Valid JSON"
else
print_endline "Invalid JSON"
(* Get error details for invalid JSON *)
let () =
match Simdjsont.Validate.check "{invalid" with
| Ok () -> print_endline "Valid"
| Error msg -> print_endline ("Invalid: " ^ msg)
Extracting Values #
Use JSON Pointer (RFC 6901) syntax to extract values without parsing the entire document:
NDJSON / streaming #
simdjsont can parse newline-delimited JSON (NDJSON) (and more generally, concatenated JSON documents). This is useful when you want to process large streams of JSON documents (logs, events, data pipelines) without splitting the input yourself.
Note: this is NDJSON / JSON Lines, not JSON-LD (W3C linked data).
The high-level Simdjsont.Ndjson module decodes each document with a codec and yields a lazy sequence of results. Parsing stays streaming and zero-copy:
type event = { id : int; name : string }
let event_codec =
let open Simdjsont.Decode in
Obj.field (fun id name -> { id; name })
|> Obj.mem "id" int ~enc:(fun e -> e.id)
|> Obj.mem "name" string ~enc:(fun e -> e.name)
|> Obj.finish
let () =
let ndjson = "{\"id\":1,\"name\":\"a\"}\n{\"id\":2,\"name\":\"b\"}\n" in
Simdjsont.Ndjson.decode_string_seq event_codec ndjson
|> Seq.iter (function
| Ok e -> Printf.printf "id=%d name=%s\n" e.id e.name
| Error msg -> Printf.eprintf "decode error: %s\n" msg)
Simdjsont.Ndjson.decode_seq accepts a bigstring (Simdjsont.Raw.buffer) directly for zero-copy input, and both functions take an optional ?batch_size.
If you just want the JSON values without defining a codec, use Simdjsont.Ndjson.values / values_of_string, which yield a sequence of Simdjsont.Json.t:
let () =
Simdjsont.Ndjson.values_of_string "1\n\"two\"\n[3, 4]\n"
|> Seq.iter (function
| Ok json -> print_endline (Simdjsont.Json.to_string json)
| Error msg -> prerr_endline msg)
(* prints: 1 / "two" / [3,4] *)
For byte offsets, truncation detection, or untyped access, drop down to the low-level streaming API Simdjsont.Raw.Stream:
let ndjson = "{\"id\":1}\n{\"id\":2}\n{broken\n{\"id\":3}\n" in
let parser = Simdjsont.Raw.create_parser () in
let buf = Simdjsont.Raw.buffer_of_string ndjson in
let len = String.length ndjson in
match Simdjsont.Raw.Stream.create parser buf ~len with
| Error e -> Printf.eprintf "Failed to create stream: %s\n" e.message
| Ok stream ->
Simdjsont.Raw.Stream.to_seq stream
|> Seq.iter (function
| Ok (elt, byte_offset) ->
Printf.printf "Doc at %d: %s\n" byte_offset
(Simdjsont.Raw.element_to_string elt)
| Error (e, byte_offset) ->
Printf.printf "Parse error at %d: %s\n" byte_offset e.message);
let truncated = Simdjsont.Raw.Stream.truncated_bytes stream in
if truncated > 0 then
Printf.printf "Warning: %d truncated bytes at end of stream\n" truncated
Error semantics: a malformed document terminates the stream — iteration yields the documents up to and including the failing one (as an Error) and then stops. Documents after a parse error are not reached.
Lifetime note: values returned from Simdjsont.Raw.Stream reference memory owned by the parser/buffer. Keep the parser and buf alive for the whole iteration, and don't reuse the parser for other parses while you still hold elements.
let json = {|{
"users": [
{"id": 1, "name": "Alice", "active": true},
{"id": 2, "name": "Bob", "active": false}
],
"count": 2
}|}
(* Extract a string *)
let () =
match Simdjsont.Extract.string json ~pointer:"/users/0/name" with
| Ok name -> print_endline name (* prints: Alice *)
| Error e -> print_endline ("Error: " ^ e)
(* Extract an integer *)
let () =
match Simdjsont.Extract.int json ~pointer:"/count" with
| Ok n -> Printf.printf "Count: %d\n" n (* prints: Count: 2 *)
| Error e -> print_endline ("Error: " ^ e)
(* Extract from array by index *)
let () =
match Simdjsont.Extract.int json ~pointer:"/users/1/id" with
| Ok id -> Printf.printf "Second user ID: %d\n" id (* prints: Second user ID: 2 *)
| Error e -> print_endline ("Error: " ^ e)
(* Extract a boolean *)
let () =
match Simdjsont.Extract.bool json ~pointer:"/users/0/active" with
| Ok b -> Printf.printf "Active: %b\n" b (* prints: Active: true *)
| Error e -> print_endline ("Error: " ^ e)
(* Check if a value is null *)
let () =
match Simdjsont.Extract.is_null {|{"value": null}|} ~pointer:"/value" with
| Ok true -> print_endline "Value is null"
| Ok false -> print_endline "Value is not null"
| Error e -> print_endline ("Error: " ^ e)
Decoding to Types #
Define codecs to decode JSON directly to OCaml types:
(* Primitive types *)
let () =
match Simdjsont.Decode.decode_string Simdjsont.Decode.bool "true" with
| Ok b -> Printf.printf "Bool: %b\n" b
| Error e -> print_endline e
let () =
match Simdjsont.Decode.decode_string Simdjsont.Decode.int "42" with
| Ok n -> Printf.printf "Int: %d\n" n
| Error e -> print_endline e
let () =
match Simdjsont.Decode.decode_string Simdjsont.Decode.float "3.14" with
| Ok f -> Printf.printf "Float: %f\n" f
| Error e -> print_endline e
let () =
match Simdjsont.Decode.decode_string Simdjsont.Decode.string {|"hello"|} with
| Ok s -> Printf.printf "String: %s\n" s
| Error e -> print_endline e
(* Lists *)
let () =
match Simdjsont.Decode.decode_string Simdjsont.Decode.(list int) "[1, 2, 3, 4, 5]" with
| Ok nums -> List.iter (Printf.printf "%d ") nums
| Error e -> print_endline e
(* Optional values *)
let () =
match Simdjsont.Decode.decode_string Simdjsont.Decode.(optional string) "null" with
| Ok None -> print_endline "No value"
| Ok (Some s) -> print_endline s
| Error e -> print_endline e
Decoding Records #
Use the builder API to decode JSON objects to records:
type point = { x: int; y: int }
let point_codec =
let open Simdjsont.Decode in
Obj.field (fun x y -> { x; y })
|> Obj.mem "x" int ~enc:(fun p -> p.x)
|> Obj.mem "y" int ~enc:(fun p -> p.y)
|> Obj.finish
let () =
match Simdjsont.Decode.decode_string point_codec {|{"x": 10, "y": 20}|} with
| Ok p -> Printf.printf "Point: (%d, %d)\n" p.x p.y
| Error e -> print_endline e
type user = {
id: int;
name: string;
email: string option;
}
let user_codec =
let open Simdjsont.Decode in
Obj.field (fun id name email -> { id; name; email })
|> Obj.mem "id" int ~enc:(fun u -> u.id)
|> Obj.mem "name" string ~enc:(fun u -> u.name)
|> Obj.opt_mem "email" string ~enc:(fun u -> u.email)
|> Obj.finish
let () =
let json = {|{"id": 1, "name": "Alice", "email": null}|} in
match Simdjsont.Decode.decode_string user_codec json with
| Ok user ->
Printf.printf "User %d: %s\n" user.id user.name;
(match user.email with
| Some e -> Printf.printf "Email: %s\n" e
| None -> print_endline "No email")
| Error e -> print_endline e
Encoding #
Encode OCaml values to JSON strings:
(* Encode primitives *)
let () =
print_endline (Simdjsont.Encode.to_string Simdjsont.Decode.bool true); (* true *)
print_endline (Simdjsont.Encode.to_string Simdjsont.Decode.int 42); (* 42 *)
print_endline (Simdjsont.Encode.to_string Simdjsont.Decode.string "hello"); (* "hello" *)
print_endline (Simdjsont.Encode.to_string Simdjsont.Decode.(list int) [1;2;3]) (* [1,2,3] *)
(* Encode records using the same codec *)
type point = { x: int; y: int }
let point_codec =
let open Simdjsont.Decode in
Obj.field (fun x y -> { x; y })
|> Obj.mem "x" int ~enc:(fun p -> p.x)
|> Obj.mem "y" int ~enc:(fun p -> p.y)
|> Obj.finish
let () =
let p = { x = 100; y = 200 } in
print_endline (Simdjsont.Encode.to_string point_codec p)
(* {"x":100,"y":200} *)
PPX deriver (ppx_simdjsont) #
The ppx_simdjsont package (built on ppxlib, so it works across OCaml
versions) generates codecs from type definitions, removing the builder
boilerplate above:
type point = { x : int; y : int } [@@deriving simdjsont]
(* generates:
val point_codec : point Simdjsont.Codec.t
val point_of_string : string -> (point, string) result
val point_to_string : point -> string *)
type user = {
id : int;
name : string;
email : string option; (* missing or null -> None *)
}
[@@deriving simdjsont]
Add it to your dune file:
(libraries simdjsont)
(preprocess (pps ppx_simdjsont))
and to your .opam/project dependencies (ppx_simdjsont).
Supported shapes: records, variants (nullary constructors as strings,
payload constructors as ["Tag", args...]), tuples (as arrays), option,
list, array, primitives (bool, int, int32, int64, nativeint,
float, string, bytes, char, unit, Simdjsont.Json.t), aliases, type
parameters ('a box generates box_codec : 'a Codec.t -> 'a box Codec.t), and
(single/mutual) recursive types.
Field attributes:
[@key "name"](alias[@name]) renames the JSON member.[@default expr]decodes a missing member asexpr(always encodes it).[@codec expr]overrides the field codec. For'a optionfields it must be the inner codec (for'a).
Constructor attributes: [@name "Tag"] (alias [@key]) renames the JSON tag.
For anything else (polymorphic variants, GADTs, custom mappings), use [@codec ...] for the field or write a manual Simdjsont.Codec (see Codec.make,
Codec.decode_with_path, Codec.encode_into, Codec.fix/fix2).
Worked example: nested records with renames, defaults, and options #
One annotation replaces the whole Obj.field |> Obj.mem ... |> Obj.finish
chain, including for nested types — each nested record just derives its own
codec:
type tls = { enabled : bool; cert : string option } [@@deriving simdjsont]
type server = {
host : string; [@key "hostName"]
port : int; [@default 80]
tls : tls;
tags : string list;
}
[@@deriving simdjsont]
Decoding fills in the default and accepts a missing cert as None:
(* Ok { host = "db.internal"; port = 80;
tls = { enabled = true; cert = None };
tags = ["primary"; "eu"] } *)
let srv =
server_of_string
{|{"hostName":"db.internal","tls":{"enabled":true},"tags":["primary","eu"]}|}
Encoding always writes port (defaults only affect decoding) and omits cert
when it is None:
(* {"hostName":"db.internal","port":80,"tls":{"enabled":true},"tags":["primary"]} *)
let json =
server_to_string
{
host = "db.internal";
port = 80;
tls = { enabled = true; cert = None };
tags = [ "primary" ];
}
The equivalent hand-written codec is ~15 lines of builder calls; the deriver keeps it to two type definitions.
Worked example: variants and renamed tags #
type shape = Circle of float | Rect of float * float | Point
[@@deriving simdjsont]
type msg = Ping [@name "ping"] | Text of string [@name "text"]
[@@deriving simdjsont]
| OCaml | JSON |
|---|---|
Point |
"Point" |
Circle 1.5 |
["Circle",1.5] |
Rect (1.5, 2.5) |
["Rect",1.5,2.5] |
Ping |
"ping" |
Text "hello" |
["text","hello"] |
So shape_to_string (Rect (1.5, 2.5)) = {|["Rect",1.5,2.5]|}, and
msg_of_string {|["text","hello"]|} = Ok (Text "hello").
Worked example: generics #
Each type parameter becomes a codec argument, so one definition serves every element type:
type 'a page = { items : 'a list; total : int } [@@deriving simdjsont]
(* val page_codec : 'a Simdjsont.Codec.t -> 'a page Simdjsont.Codec.t *)
(* {"items":[1,2],"total":2} *)
let json = page_to_string Simdjsont.Codec.int { items = [ 1; 2 ]; total = 2 }
(* Ok { items = [1; 2]; total = 2 } *)
let parsed = page_of_string Simdjsont.Codec.int json
Worked example: recursive and mutually recursive types #
Recursion (even through variants) just works — no fix boilerplate in user
code:
type expr = Lit of int | Add of expr * expr | Var of string
[@@deriving simdjsont]
(* ["Add",["Lit",1],["Var","x"]] *)
let json = expr_to_string (Add (Lit 1, Var "x"))
(* Ok (Add (Lit 1, Var "x")) *)
let back = expr_of_string json
Mutually recursive groups derive together:
type author = { name : string; books : book list }
and book = { title : string; author : author option }
[@@deriving simdjsont]
(* Ok { name = "Ada";
books = [{ title = "Notes"; author = None }; ...] } *)
let ada =
author_of_string
{|{"name":"Ada","books":[{"title":"Notes"},
{"title":"Sketches","author":{"name":"Co","books":[]}}]}|}
Errors keep JSON paths #
Derived codecs report failures with the dotted path of the offending value, using the JSON member names:
(* Error "tls.enabled: expected bool" *)
let bad =
server_of_string {|{"hostName":"h","tls":{"enabled":"yes"},"tags":[]}|}
(* Error "missing field: hostName" *)
let missing = server_of_string {|{"tls":{"enabled":true},"tags":[]}|}
These snippets are exercised verbatim by ppx/test/test_ppx_doc.ml, so the
documentation stays in sync with the deriver.
Bigstring API #
Every consume/produce entry point has a bigstring counterpart that works on the standard bigstring type (Simdjsont.Raw.buffer, i.e. (char, int8_unsigned_elt, c_layout) Bigarray.Array1.t — the same type used by Bigstringaf, Lwt_bytes, and Core.Bigstring). This lets you parse data you already hold in a bigstring (sockets, mmap, network buffers) and produce parse-ready bigstrings, without copying through an OCaml string.
simdjson requires Simdjsont.padding bytes of slack after the data. Parsing is zero-copy when the bigstring already has that capacity; otherwise the input is transparently copied into a padded buffer.
(* Decode straight from a bigstring *)
let buf = Simdjsont.bigstring_of_string {|{"x":1,"y":2}|} in
let len = 13 in
match Simdjsont.decode_bigstring point_codec buf ~len with
| Ok p -> Printf.printf "x=%d y=%d\n" p.x p.y
| Error msg -> prerr_endline msg
(* Validate and extract by pointer on a bigstring *)
let _ = Simdjsont.validate_bigstring buf ~len
let _ = Simdjsont.Extract.int_bigstring buf ~len ~pointer:"/x"
(* Encode directly into a parse-ready bigstring (no intermediate string).
The returned buffer can be fed straight back into decode_bigstring. *)
let out, out_len = Simdjsont.encode_bigstring point_codec { x = 100; y = 200 } in
match Simdjsont.decode_bigstring point_codec out ~len:out_len with
| Ok p -> assert (p.x = 100)
| Error _ -> ()
Helpers: Simdjsont.padding, Simdjsont.create_bigstring, Simdjsont.bigstring_of_string. Codec-level entry points live in Simdjsont.Codec (decode_bigstring, encode_to_bigstring) and Simdjsont.Json (to_bigstring).
Extracting with Codecs #
Combine extraction and decoding:
type item = { name: string; price: float }
let item_codec =
let open Simdjsont.Decode in
Obj.field (fun name price -> { name; price })
|> Obj.mem "name" string ~enc:(fun i -> i.name)
|> Obj.mem "price" float ~enc:(fun i -> i.price)
|> Obj.finish
let json = {|{
"order": {
"items": [
{"name": "Book", "price": 12.99},
{"name": "Pen", "price": 1.50}
]
}
}|}
let () =
match Simdjsont.Extract.at item_codec json ~pointer:"/order/items/0" with
| Ok item -> Printf.printf "%s: $%.2f\n" item.name item.price
| Error e -> print_endline e
(* prints: Book: $12.99 *)
Working with Dynamic JSON #
Use Simdjsont.Json.t for dynamic JSON values:
(* Build JSON programmatically *)
let json = Simdjsont.Json.(Object [
("name", String "Alice");
("age", Int 30L);
("scores", Array [Int 95L; Int 87L; Int 92L]);
("active", Bool true);
])
let () = print_endline (Simdjsont.Json.to_string json)
(* {"name":"Alice","age":30,"scores":[95,87,92],"active":true} *)
(* Decode to dynamic JSON *)
let () =
match Simdjsont.Decode.decode_string Simdjsont.Decode.value {|{"key": [1, 2, 3]}|} with
| Ok v -> print_endline (Simdjsont.Json.to_string v)
| Error e -> print_endline e
Converting to/from Yojson (example) #
simdjsont does not depend on yojson, but you can convert values when you need interoperability.
(* Requires adding `yojson` to your own project dependencies. *)
let rec simdjsont_to_yojson (v : Simdjsont.Json.t) : Yojson.Safe.t =
match v with
| Null -> `Null
| Bool b -> `Bool b
| Int i -> `Intlit (Int64.to_string i)
| Float f -> `Float f
| String s -> `String s
| Array xs -> `List (List.map simdjsont_to_yojson xs)
| Object fields ->
`Assoc
(List.map (fun (k, x) -> (k, simdjsont_to_yojson x)) fields)
let rec yojson_to_simdjsont (v : Yojson.Safe.t) : Simdjsont.Json.t =
match v with
| `Null -> Null
| `Bool b -> Bool b
| `Int n -> Int (Int64.of_int n)
| `Intlit s -> Int (Int64.of_string s)
| `Float f -> Float f
| `String s -> String s
| `List xs -> Array (List.map yojson_to_simdjsont xs)
| `Assoc fields ->
Object (List.map (fun (k, x) -> (k, yojson_to_simdjsont x)) fields)
| `Tuple xs -> Array (List.map yojson_to_simdjsont xs)
| `Variant (tag, arg_opt) ->
Object
[
( "variant",
Object
(
[ ("tag", String tag) ]
@
match arg_opt with
| None -> []
| Some a -> [ ("arg", yojson_to_simdjsont a) ] ) );
]
let () =
match Simdjsont.Decode.decode_string Simdjsont.Decode.value {|{"name": "test", "value": 123}|} with
| Ok json ->
let yojson = simdjsont_to_yojson json in
let back = yojson_to_simdjsont yojson in
assert (Simdjsont.Json.to_string back = Simdjsont.Json.to_string json)
| Error e -> failwith e
Low-Level API #
For maximum control, use the low-level bindings:
let () =
let open Simdjsont.Raw in
let parser = create_parser () in
match parse_string parser {|{"users": [{"name": "Alice"}, {"name": "Bob"}]}|} with
| Ok root ->
(* Navigate to nested value *)
(match at_pointer root "/users/1/name" with
| Ok elt -> print_endline (string_exn elt) (* prints: Bob *)
| Error e -> print_endline e.message);
(* Iterate over arrays *)
(match at_pointer root "/users" with
| Ok elt ->
let arr = array_exn elt in
array_iter (fun user ->
let obj = object_exn user in
match object_find obj "name" with
| Ok name -> print_endline (string_exn name)
| Error _ -> ()
) arr
| Error e -> print_endline e.message)
| Error e ->
print_endline ("Parse error: " ^ e.message)
CBOR #
simdjsont includes support for CBOR (RFC 8949), using the same codecs as JSON. It decodes CBOR into the same internal Simdjsont.Json.t representation, allowing you to use your existing data models and validation logic.
CBOR Quick Start #
type point = { x: int; y: int }
let point_codec =
let open Simdjsont.Decode in
Obj.field (fun x y -> { x; y })
|> Obj.mem "x" int ~enc:(fun p -> p.x)
|> Obj.mem "y" int ~enc:(fun p -> p.y)
|> Obj.finish
(* Encode to CBOR binary string *)
let cbor = Simdjsont.Cbor.encode_string point_codec { x = 10; y = 20 }
(* Decode from CBOR *)
match Simdjsont.Cbor.decode_string point_codec cbor with
| Ok p -> Printf.printf "Point from CBOR: (%d, %d)\n" p.x p.y
| Error e -> print_endline e
CBOR Streaming #
CBOR often contains multiple concatenated items. Use Simdjsont.Cbor.to_seq to parse these as a sequence:
let cbor_data =
Simdjsont.Cbor.encode_string Simdjsont.Decode.int 1 ^
Simdjsont.Cbor.encode_string Simdjsont.Decode.int 2 ^
Simdjsont.Cbor.encode_string Simdjsont.Decode.int 3
let () =
Simdjsont.Cbor.to_seq Simdjsont.Decode.int cbor_data
|> Seq.iter (function
| Ok n -> Printf.printf "Decoded: %d\n" n
| Error e -> Printf.printf "Error: %s\n" e)
CBOR Support Model #
CBOR is richer than JSON, while simdjsont codecs describe JSON-compatible OCaml values. The CBOR decoder therefore maps supported CBOR items onto the same value model used by Simdjsont.Json.t and Simdjsont.Codec.t.
| CBOR feature | Status | Mapping |
|---|---|---|
| Unsigned and negative integers | Supported within the target range | int, int64, or Json.Int; out-of-range values return a decode error |
| Half, single, and double floats | Supported | float or Json.Float |
| Text strings | Supported | string; indefinite-length text is concatenated |
| Byte strings | Supported | string; indefinite-length bytes are concatenated |
| Arrays | Supported | list, array, or Json.Array; indefinite arrays are supported |
| Maps | Supported for text/byte-string keys | object codecs or Json.Object; indefinite maps are supported |
| Tags | Parsed as transparent wrappers | tag number is ignored and the tagged item is decoded normally |
true, false, null |
Supported | bool, unit, options, or Json values |
undefined |
Supported as JSON-compatible null | null / None / Json.Null |
The following CBOR features are not represented as distinct semantic values:
- Semantic tags: dates, URIs, encoded CBOR, and similar tagged values are not interpreted specially; the tag is ignored.
- Bignums: tags 2 and 3 are not converted to arbitrary-precision integers. Their byte-string payload is decoded as an OCaml string, following the transparent-tag rule. Use a custom binary/string layer if your data relies on CBOR bignums.
- Integer map keys: JSON objects require string keys, so CBOR maps with integer keys return a decode error.
- Other simple values: simple values besides booleans, null, and undefined are rejected.
If you need to preserve every CBOR-specific distinction, use a dedicated CBOR AST in your application. Simdjsont.Cbor is intended for high-performance interchange of JSON-compatible data over CBOR bytes.
Json.Int and the int64 codec accept only signed 64-bit integers; the int
codec checks the platform's OCaml integer range. The float codec can also read
unsigned 64-bit integers, with normal floating-point rounding. CBOR negative
integers below Int64.min_int are rejected during parsing.
License #
ISC