diff --git a/dune-project b/dune-project index 57d8bfb..03dbd9e 100644 --- a/dune-project +++ b/dune-project @@ -29,7 +29,6 @@ dune pprint crunch - containers ppx_deriving js_of_ocaml js_of_ocaml-compiler diff --git a/lib/00_core/Pinc_Core.ml b/lib/00_core/Pinc_Core.ml index fe3ca71..d5d2945 100644 --- a/lib/00_core/Pinc_Core.ml +++ b/lib/00_core/Pinc_Core.ml @@ -2,6 +2,7 @@ include StdlibExtension module StringMap = StringMap module StringSet = StringSet module Identifier = Identifier +module Utf8String = Utf8String module Dedent = struct let indentation = @@ -25,12 +26,12 @@ module Dedent = struct let lines string = let lines = String.split_on_char '\n' string in - let lines = List.map Containers.String.rtrim lines in + let lines = List.map String.trim_right lines in let lines = match lines with | [] -> [] | "" :: rest -> drop_indentation rest - | first :: rest -> Containers.String.ltrim first :: drop_indentation rest + | first :: rest -> String.trim_left first :: drop_indentation rest in lines ;; diff --git a/lib/00_core/StdlibExtension.ml b/lib/00_core/StdlibExtension.ml index af910e1..38ff91c 100644 --- a/lib/00_core/StdlibExtension.ml +++ b/lib/00_core/StdlibExtension.ml @@ -119,4 +119,30 @@ module String = struct in take_first index s ;; + + let trim_left s = + let len = length s in + let i = ref 0 in + while !i < len && Char.is_whitespace (unsafe_get s !i) do + incr i + done; + let j = len - 1 in + if j >= !i then + sub s !i (j - !i + 1) + else + empty + ;; + + let trim_right s = + let len = length s in + let i = 0 in + let j = ref (len - 1) in + while !j >= i && Char.is_whitespace (unsafe_get s !j) do + decr j + done; + if !j >= i then + sub s i (!j - i + 1) + else + empty + ;; end diff --git a/lib/00_core/Utf8String.ml b/lib/00_core/Utf8String.ml new file mode 100644 index 0000000..9c7aa66 --- /dev/null +++ b/lib/00_core/Utf8String.ml @@ -0,0 +1,197 @@ +type t = String.t + +let to_string x = x + +(** State for decoding *) +module Dec = struct + type t = { + s : string; + len : int; + mutable i : int; + } + + let make ?(idx = 0) (s : string) : t = { s; i = idx; len = String.length s } +end + +(** Malformed string at given offset *) +exception Malformed of string * int + +(* decode next char. Mutate state, calls [yield c] if a char [c] is + read, [stop ()] otherwise. + @raise Malformed if an invalid substring is met *) +let next_ (type a) (st : Dec.t) ~(yield : Uchar.t -> a) ~(stop : unit -> a) () : a = + let open Dec in + let malformed st = raise (Malformed (st.s, st.i)) in + (* read a multi-byte character. + @param acc the accumulator (containing the first byte of the char) + @param n_bytes number of bytes to read (i.e. [width char - 1]) + @param overlong minimal bound on second byte (to detect overlong encoding) + *) + let read_multi ?(overlong = 0) n_bytes acc = + (* inner loop j = 1..jmax *) + let rec aux j acc = + let c = Char.code st.s.[st.i + j] in + (* check that c is in 0b10xxxxxx *) + if c lsr 6 <> 0b10 then + malformed st; + (* overlong encoding? *) + if j = 1 && overlong <> 0 && c land 0b111111 < overlong then + malformed st; + (* except for first, each char gives 6 bits *) + let next = (acc lsl 6) lor (c land 0b111111) in + if j = n_bytes then + if + (* done reading the codepoint *) + Uchar.is_valid next + then ( + st.i <- st.i + j + 1; + (* +1 for first char *) + yield (Uchar.unsafe_of_int next)) + else + malformed st + else + aux (j + 1) next + in + assert (n_bytes >= 1); + (* is the string long enough to contain the whole codepoint? *) + if st.i + n_bytes < st.len then + aux 1 acc + (* start with j=1, first char is already processed! *) + else + (* char is truncated *) + malformed st + in + if st.i >= st.len then + stop () + else ( + let c = st.s.[st.i] in + (* find leading byte, and detect some impossible cases + according to https://en.wikipedia.org/wiki/Utf8#Codepage_layout *) + match c with + | '\000' .. '\127' -> + st.i <- 1 + st.i; + yield (Uchar.of_int @@ Char.code c) (* 0xxxxxxx *) + | '\194' .. '\223' -> read_multi 1 (Char.code c land 0b11111) (* 110yyyyy *) + | '\225' .. '\239' -> read_multi 2 (Char.code c land 0b1111) (* 1110zzzz *) + | '\241' .. '\244' -> read_multi 3 (Char.code c land 0b111) (* 11110uuu *) + | '\224' -> + (* overlong: if next byte is < than [0b001000000] then the char + would fit in 1 byte *) + read_multi ~overlong:0b00100000 2 (Char.code c land 0b1111) + (* 1110zzzz *) + | '\240' -> + (* overlong: if next byte is < than [0b000100000] then the char + would fit in 2 bytes *) + read_multi ~overlong:0b00010000 3 (Char.code c land 0b111) + (* 11110uuu *) + | '\128' .. '\193' (* 192,193 are forbidden *) | '\245' .. '\255' -> malformed st) +;; + +let fold ?idx f acc s = + let st = Dec.make ?idx s in + let rec aux acc = + next_ + st + ~yield:(fun x -> + let acc = f acc x in + aux acc) + ~stop:(fun () -> acc) + () + in + aux acc +;; + +let n_chars = fold (fun x _ -> x + 1) 0 + +let to_list ?(idx = 0) s : Uchar.t list = + fold ~idx (fun acc x -> x :: acc) [] s |> List.rev +;; + +(* Convert a code point (int) into a string; + There are various equally trivial versions of this around. +*) + +let[@inline] uchar_to_bytes (c : Uchar.t) (f : char -> unit) : unit = + let c = Uchar.to_int c in + let mask = 0b111111 in + assert (Uchar.is_valid c); + if c <= 0x7f then + f (Char.unsafe_chr c) + else if c <= 0x7ff then ( + f (Char.unsafe_chr (0xc0 lor (c lsr 6))); + f (Char.unsafe_chr (0x80 lor (c land mask)))) + else if c <= 0xffff then ( + f (Char.unsafe_chr (0xe0 lor (c lsr 12))); + f (Char.unsafe_chr (0x80 lor ((c lsr 6) land mask))); + f (Char.unsafe_chr (0x80 lor (c land mask)))) + else if c <= 0x1fffff then ( + f (Char.unsafe_chr (0xf0 lor (c lsr 18))); + f (Char.unsafe_chr (0x80 lor ((c lsr 12) land mask))); + f (Char.unsafe_chr (0x80 lor ((c lsr 6) land mask))); + f (Char.unsafe_chr (0x80 lor (c land mask)))) + else ( + f (Char.unsafe_chr (0xf8 lor (c lsr 24))); + f (Char.unsafe_chr (0x80 lor ((c lsr 18) land mask))); + f (Char.unsafe_chr (0x80 lor ((c lsr 12) land mask))); + f (Char.unsafe_chr (0x80 lor ((c lsr 6) land mask))); + f (Char.unsafe_chr (0x80 lor (c land mask)))) +;; + +(* number of bytes required to encode this codepoint. A skeleton version + of {!uchar_to_bytes}. *) +let[@inline] uchar_num_bytes (c : Uchar.t) : int = + let c = Uchar.to_int c in + if c <= 0x7f then + 1 + else if c <= 0x7ff then + 2 + else if c <= 0xffff then + 3 + else if c <= 0x1fffff then + 4 + else + 5 +;; + +let of_list l : t = + let len = List.fold_left (fun n c -> n + uchar_num_bytes c) 0 l in + if len > Sys.max_string_length then + invalid_arg "CCUtf8_string.of_list: string size limit exceeded"; + let buf = Bytes.make len '\000' in + let i = ref 0 in + List.iter + (fun c -> + uchar_to_bytes c (fun byte -> + Bytes.unsafe_set buf !i byte; + incr i)) + l; + assert (!i = len); + Bytes.unsafe_to_string buf +;; + +let is_valid (s : string) : bool = + let exception Stop in + try + let st = Dec.make s in + while true do + next_ st ~yield:(fun _ -> ()) ~stop:(fun () -> raise Stop) () + done; + assert false + with + | Malformed _ -> false + | Stop -> true +;; + +let of_string_exn s = + if is_valid s then + s + else + invalid_arg "CCUtf8_string.of_string_exn" +;; + +let of_string s = + if is_valid s then + Some s + else + None +;; diff --git a/lib/00_core/Utf8String.mli b/lib/00_core/Utf8String.mli new file mode 100644 index 0000000..19b1e4b --- /dev/null +++ b/lib/00_core/Utf8String.mli @@ -0,0 +1,8 @@ +type t + +val n_chars : t -> int +val to_string : t -> string +val to_list : ?idx:int -> t -> Uchar.t list +val of_list : Uchar.t list -> t +val of_string_exn : string -> t +val of_string : string -> t option diff --git a/lib/00_core/dune b/lib/00_core/dune index 92a269b..a007f84 100644 --- a/lib/00_core/dune +++ b/lib/00_core/dune @@ -1,4 +1,4 @@ (library (name Pinc_Core) (public_name pinc-lang.core) - (libraries containers)) + (libraries)) diff --git a/lib/pinc_backend/Externals.ml b/lib/pinc_backend/Externals.ml index ff69a01..e81e47a 100644 --- a/lib/pinc_backend/Externals.ml +++ b/lib/pinc_backend/Externals.ml @@ -43,8 +43,7 @@ module PincString = struct value_loc (Printf.sprintf "The argument given to the String.length function is not of type string") - | Ok v -> - v |> Containers.Utf8_string.of_string_exn |> Containers.Utf8_string.n_chars + | Ok v -> v |> Pinc_Core.Utf8String.of_string_exn |> Pinc_Core.Utf8String.n_chars in let output = Helpers.Value.int result in @@ -100,12 +99,12 @@ module PincString = struct let result = str - |> Containers.Utf8_string.of_string_exn - |> Containers.Utf8_string.to_list - |> Containers.List.drop offset - |> Containers.List.take length - |> Containers.Utf8_string.of_list - |> Containers.Utf8_string.to_string + |> Pinc_Core.Utf8String.of_string_exn + |> Pinc_Core.Utf8String.to_list + |> List.drop offset + |> List.take length + |> Pinc_Core.Utf8String.of_list + |> Pinc_Core.Utf8String.to_string in let output = Helpers.Value.string result in diff --git a/lib/pinc_backend/Interpreter.ml b/lib/pinc_backend/Interpreter.ml index bcfa786..631f00a 100644 --- a/lib/pinc_backend/Interpreter.ml +++ b/lib/pinc_backend/Interpreter.ml @@ -839,8 +839,8 @@ and eval_binary_bracket_access ~state left right = let output = try a - |> CCUtf8_string.of_string_exn - |> CCUtf8_string.to_list + |> Pinc_Core.Utf8String.of_string_exn + |> Pinc_Core.Utf8String.to_list |> Fun.flip List.nth b |> Helpers.Value.char ~loc:(Location.merge ~s:left.expression_loc ~e:right.expression_loc ()) @@ -1116,16 +1116,17 @@ and eval_for_in ~state ~index_ident ~ident ~reverse ~iterable body = let map s = if reverse then s - |> CCUtf8_string.to_seq - |> CCSeq.to_rev_list - |> List.to_seq - |> Seq.map (fun c -> c |> Helpers.Value.char ~loc:iterable_value.value_loc) + |> Pinc_Core.Utf8String.to_list + |> List.rev + |> List.map (fun c -> c |> Helpers.Value.char ~loc:iterable_value.value_loc) else s - |> CCUtf8_string.to_seq - |> Seq.map (fun c -> c |> Helpers.Value.char ~loc:iterable_value.value_loc) + |> Pinc_Core.Utf8String.to_list + |> List.map (fun c -> c |> Helpers.Value.char ~loc:iterable_value.value_loc) + in + let state, res = + s |> Pinc_Core.Utf8String.of_string_exn |> map |> List.to_seq |> loop ~state [] in - let state, res = s |> CCUtf8_string.of_string_exn |> map |> loop ~state [] in state |> State.add_output ~output:(res |> Helpers.Value.list ~loc:body.expression_loc) | Portal l -> diff --git a/lib/pinc_backend/dune b/lib/pinc_backend/dune index 0dee9d1..2fc2b20 100644 --- a/lib/pinc_backend/dune +++ b/lib/pinc_backend/dune @@ -2,13 +2,7 @@ (name Pinc_Backend) (public_name pinc-lang.backend) (flags :standard -open Pinc_Core) - (libraries - unix - Pinc_Core - Pinc_Source - Pinc_Diagnostics - Pinc_Parser - containers) + (libraries Pinc_Core Pinc_Source Pinc_Diagnostics Pinc_Parser) (preprocess (pps ppx_deriving.show)) (instrumentation diff --git a/pinc-lang.opam b/pinc-lang.opam index 177bc38..7ed46f6 100644 --- a/pinc-lang.opam +++ b/pinc-lang.opam @@ -12,7 +12,6 @@ depends: [ "dune" {>= "3.17"} "pprint" "crunch" - "containers" "ppx_deriving" "js_of_ocaml" "js_of_ocaml-compiler" diff --git a/pinc-lang.opam.locked b/pinc-lang.opam.locked index d093ff1..79536c0 100644 --- a/pinc-lang.opam.locked +++ b/pinc-lang.opam.locked @@ -9,77 +9,81 @@ homepage: "https://github.com/pinc-official/pinc-lang" bug-reports: "https://github.com/pinc-official/pinc-lang/issues" depends: [ "astring" {= "0.8.5" & with-dev-setup} - "base" {= "v0.17.1" & with-dev-setup} + "base" {= "v0.17.3" & with-dev-setup} "base-bigarray" {= "base"} - "base-bytes" {= "base" & with-dev-setup} "base-domains" {= "base"} "base-effects" {= "base"} "base-nnp" {= "base"} "base-threads" {= "base"} "base-unix" {= "base"} "camlp-streams" {= "5.0.1" & with-dev-setup} - "chrome-trace" {= "3.17.2" & with-dev-setup} - "cmdliner" {= "1.3.0"} - "containers" {= "3.17"} + "chrome-trace" {= "3.23.1" & with-dev-setup} + "cmdliner" {= "2.1.1"} "cppo" {= "1.8.0"} "crunch" {= "4.0.0"} - "csexp" {= "1.5.2"} - "dune" {= "3.21.0"} - "dune-build-info" {= "3.17.2" & with-dev-setup} - "dune-configurator" {= "3.17.2"} - "dune-rpc" {= "3.21.0" & with-dev-setup} - "dyn" {= "3.21.0" & with-dev-setup} - "either" {= "1.0.0"} + "csexp" {= "1.5.2" & with-dev-setup} + "dune" {= "3.23.1"} + "dune-build-info" {= "3.23.1" & with-dev-setup} + "dune-configurator" {= "3.23.1" & with-dev-setup} + "dune-rpc" {= "3.23.1" & with-dev-setup} + "dyn" {= "3.23.1" & with-dev-setup} + "either" {= "1.0.0" & with-dev-setup} "fiber" {= "3.7.0" & with-dev-setup} - "fix" {= "20230505" & with-dev-setup} - "fmt" {= "0.9.0"} + "fix" {= "20250919" & with-dev-setup} + "fmt" {= "0.11.0"} "fpath" {= "0.7.3" & with-dev-setup} - "fs-io" {= "3.21.0" & with-dev-setup} - "jsonrpc" {= "1.25.0" & with-dev-setup} - "lsp" {= "1.25.0" & with-dev-setup} - "menhir" {= "20240715" & with-dev-setup} - "menhirCST" {= "20240715" & with-dev-setup} - "menhirLib" {= "20240715" & with-dev-setup} - "menhirSdk" {= "20240715" & with-dev-setup} - "merlin-lib" {= "5.6.1-504" & with-dev-setup} - "ocaml" {= "5.4.0"} - "ocaml-compiler" {= "5.4.0"} + "fs-io" {= "3.23.1" & with-dev-setup} + "gen" {= "1.1"} + "js_of_ocaml" {= "6.3.2"} + "js_of_ocaml-compiler" {= "6.3.2"} + "jsonrpc" {= "1.26.0" & with-dev-setup} + "lsp" {= "1.26.0" & with-dev-setup} + "menhir" {= "20260209"} + "menhirCST" {= "20260209"} + "menhirGLR" {= "20260209"} + "menhirLib" {= "20260209"} + "menhirSdk" {= "20260209"} + "merlin-lib" {= "5.7.1-504" & with-dev-setup} + "ocaml" {= "5.4.1"} + "ocaml-base-compiler" {= "5.4.1"} + "ocaml-compiler" {= "5.4.1"} "ocaml-compiler-libs" {= "v0.17.0"} "ocaml-config" {= "3"} - "ocaml-index" {= "5.6.1-504" & with-dev-setup} - "ocaml-lsp-server" {= "1.25.0" & with-dev-setup} - "ocaml-variants" {= "5.4.0+options"} - "ocaml-version" {= "4.0.3" & with-dev-setup} - "ocaml_intrinsics_kernel" {= "v0.17.1" & with-dev-setup} - "ocamlbuild" {= "0.15.0"} - "ocamlc-loc" {= "3.21.0" & with-dev-setup} + "ocaml-index" {= "5.7.1-504" & with-dev-setup} + "ocaml-lsp-server" {= "1.26.0" & with-dev-setup} + "ocaml-options-vanilla" {= "1"} + "ocaml-version" {= "4.1.1" & with-dev-setup} + "ocaml_intrinsics_kernel" {= "v0.17.2" & with-dev-setup} + "ocamlbuild" {= "0.16.1"} + "ocamlc-loc" {= "3.23.1" & with-dev-setup} "ocamlfind" {= "1.9.8"} - "ocamlformat" {= "0.28.1" & with-dev-setup} - "ocamlformat-lib" {= "0.28.1" & with-dev-setup} - "ocamlformat-rpc-lib" {= "0.27.0" & with-dev-setup} - "ocp-indent" {= "1.8.1" & with-dev-setup} - "ordering" {= "3.21.0" & with-dev-setup} + "ocamlformat" {= "0.29.0" & with-dev-setup} + "ocamlformat-lib" {= "0.29.0" & with-dev-setup} + "ocamlformat-rpc-lib" {= "0.29.0" & with-dev-setup} + "ocp-indent" {= "1.9.0" & with-dev-setup} + "ordering" {= "3.23.1" & with-dev-setup} "pp" {= "2.0.0" & with-dev-setup} "pprint" {= "20230830"} "ppx_derivers" {= "1.2.1"} "ppx_deriving" {= "6.1.1"} "ppx_yojson_conv_lib" {= "v0.17.0" & with-dev-setup} - "ppxlib" {= "0.37.0"} + "ppxlib" {= "0.38.0"} "ptime" {= "1.2.0"} - "re" {= "1.12.0" & with-dev-setup} - "seq" {= "base" & with-dev-setup} + "re" {= "1.14.0" & with-dev-setup} + "sedlex" {= "3.7"} + "seq" {= "base"} "sexplib0" {= "v0.17.0"} "spawn" {= "v0.17.0" & with-dev-setup} "stdio" {= "v0.17.0" & with-dev-setup} "stdlib-shims" {= "0.3.0"} - "stdune" {= "3.21.0" & with-dev-setup} - "top-closure" {= "3.21.0" & with-dev-setup} - "topkg" {= "1.0.7"} - "uucp" {= "16.0.0" & with-dev-setup} - "uuseg" {= "16.0.0" & with-dev-setup} - "uutf" {= "1.0.3" & with-dev-setup} - "xdg" {= "3.17.2" & with-dev-setup} - "yojson" {= "2.2.2" & with-dev-setup} + "stdune" {= "3.23.1" & with-dev-setup} + "top-closure" {= "3.23.1" & with-dev-setup} + "topkg" {= "1.1.1"} + "uucp" {= "17.0.0" & with-dev-setup} + "uuseg" {= "17.0.0" & with-dev-setup} + "uutf" {= "1.0.4" & with-dev-setup} + "xdg" {= "3.23.1" & with-dev-setup} + "yojson" {= "3.0.0"} ] build: [ ["dune" "subst"] {dev}