diff --git a/lib/compiler/URI_util.ml b/lib/compiler/URI_util.ml index ec3ba6d..7c8a9db 100644 --- a/lib/compiler/URI_util.ml +++ b/lib/compiler/URI_util.ml @@ -26,11 +26,11 @@ let next_uri ~(prefix : string option) ~(mode : [< `Random | `Sequential]) ~(for let config = forest.config in let keys = List.of_seq @@ - let@ uri = Seq.filter_map @~ addrs in - if URI.host config.url = URI.host uri then - let@ prefix', key = Option.bind @@ URI_scheme.split_addr uri in - if prefix = prefix' then Some key else None - else None + let@ uri = Seq.filter_map @~ addrs in + if URI.host config.url = URI.host uri then + let@ prefix', key = Option.bind @@ URI_scheme.split_addr uri in + if prefix = prefix' then Some key else None + else None in let next = match mode with diff --git a/lib/compiler/test/Test_compiler.ml b/lib/compiler/test/Test_compiler.ml index 94b8832..78a87b6 100644 --- a/lib/compiler/test/Test_compiler.ml +++ b/lib/compiler/test/Test_compiler.ml @@ -28,16 +28,16 @@ let raw_trees = let test_batch_run ~env () = let forest, history = - with_test_forest - ~raw_trees - ~env - ~config - (fun path -> - Sys.chdir (Eio.Path.native_exn path); - let@ () = Reporter.easy_run in - let forest = State.make ~env ~config ~dev: false () in - Driver.run_with_history Load_all_configured_dirs forest - ) + let@ path = + with_test_forest + ~raw_trees + ~env + ~config + in + Sys.chdir (Eio.Path.native_exn path); + let@ () = Reporter.easy_run in + let forest = State.make ~env ~config ~dev: false () in + Driver.run_with_history Load_all_configured_dirs forest in Alcotest.(check @@ list action) "all actions have run" diff --git a/lib/compiler/test/Test_diagnostic_store.ml b/lib/compiler/test/Test_diagnostic_store.ml index e69de29..8b13789 100644 --- a/lib/compiler/test/Test_diagnostic_store.ml +++ b/lib/compiler/test/Test_diagnostic_store.ml @@ -0,0 +1 @@ + diff --git a/lib/compiler/test/Test_expansion.ml b/lib/compiler/test/Test_expansion.ml index 6ecc32c..028d4ac 100644 --- a/lib/compiler/test/Test_expansion.ml +++ b/lib/compiler/test/Test_expansion.ml @@ -61,7 +61,8 @@ let test_subtree ~env () = let@ () = Reporter.easy_run in let forest = State.make ~env ~config ~dev: false () in let expanded = - expand ~forest + expand + ~forest {| \subtree[foo]{ \title{Hello} @@ -79,7 +80,7 @@ let test_subtree ~env () = evaluated let test_visible ~env () = - let forest = State.make ~env ~config ~dev: false () in + let forest = State.make ~env ~config ~dev: false () in let code = Result.get_ok @@ parse_string diff --git a/lib/compiler/test/Test_import_graph.ml b/lib/compiler/test/Test_import_graph.ml index 77bc684..4ffd476 100644 --- a/lib/compiler/test/Test_import_graph.ml +++ b/lib/compiler/test/Test_import_graph.ml @@ -14,7 +14,7 @@ open struct module T = Types end let config = {(Config.default ()) with trees = ["imports"]} -let mk_vertex v = T.Uri_vertex (URI_scheme.named_uri ~base:config.url v) +let mk_vertex v = T.Uri_vertex (URI_scheme.named_uri ~base: config.url v) let has_edge g v w = Forest_graph.mem_edge g (mk_vertex v) (mk_vertex w) diff --git a/lib/frontend/Config_parser.ml b/lib/frontend/Config_parser.ml index 2fce169..20b4741 100644 --- a/lib/frontend/Config_parser.ml +++ b/lib/frontend/Config_parser.ml @@ -23,18 +23,19 @@ end let keys (tbl : Toml.Types.value Toml.Types.Table.t) = let rec go current keys tbl = List.fold_left - (fun acc (key, value) -> - match value with - | Toml.Types.TBool _ - | TInt _ - | TFloat _ - | TString _ - | TDate _ - | TArray _ -> - (key :: current) :: acc - | TTable tbl -> - go (key :: current) acc tbl - ) + begin + fun acc (key, value) -> + match value with + | Toml.Types.TBool _ + | TInt _ + | TFloat _ + | TString _ + | TDate _ + | TArray _ -> + (key :: current) :: acc + | TTable tbl -> + go (key :: current) acc tbl + end keys (Toml.Types.Table.to_list tbl) in diff --git a/lib/search/Index.ml b/lib/search/Index.ml index d46b05b..699627f 100644 --- a/lib/search/Index.ml +++ b/lib/search/Index.ml @@ -4,6 +4,7 @@ * SPDX-License-Identifier: GPL-3.0-or-later *) +open Forester_prelude open Forester_core open Spelll @@ -21,14 +22,10 @@ type t = { number_of_docs: int; } -let average_doc_length - : t -> float -= fun {number_of_tokens; number_of_docs; _} -> +let average_doc_length {number_of_tokens; number_of_docs; _} : float = Float.of_int number_of_tokens /. Float.of_int number_of_docs -let add_one - : T.content T.article -> t -> t -= fun article ({index; number_of_tokens; number_of_docs;} as t) -> +let add_one (article : T.content T.article) ({index; number_of_tokens; number_of_docs;} as t) : t = if Option.is_none T.(article.frontmatter.uri) then t else let tokens_in_article = Tokenizer.tokenize_article article in @@ -36,20 +33,21 @@ let add_one let new_tokens = ref 0 in let new_index = List.fold_left - (fun index (ocurrences, token) -> - match Index.retrieve_l ~limit: 0 index token with - | [] -> - (* Unseen token*) - (* TODO: add to list of ocurrences*) - let ocurrence = Ocurrences.singleton ([ocurrences], uri) in - new_tokens := !new_tokens + 1; - Index.add index token ocurrence - | ids :: [] -> - Index.add index token (Ocurrences.add ([ocurrences], uri) ids) - | _ -> - (* We are using limit=0, so this shouldn't happen*) - assert false - ) + begin + fun index (ocurrences, token) -> + match Index.retrieve_l ~limit: 0 index token with + | [] -> + (* Unseen token*) + (* TODO: add to list of ocurrences*) + let ocurrence = Ocurrences.singleton ([ocurrences], uri) in + new_tokens := !new_tokens + 1; + Index.add index token ocurrence + | ids :: [] -> + Index.add index token (Ocurrences.add ([ocurrences], uri) ids) + | _ -> + (* We are using limit=0, so this shouldn't happen*) + assert false + end index tokens_in_article in @@ -59,20 +57,13 @@ let add_one number_of_tokens = number_of_tokens + !new_tokens } -let add - : T.content T.article list -> t -> t - = +let add : T.content T.article list -> t -> t = List.fold_right add_one -let search - : ?fuzz: int -> t -> string -> (int list list * URI.t) list -= fun ?(fuzz = 0) index term -> - Tokenizer.tokenize term - |> List.concat_map - (fun str -> - List.concat_map Ocurrences.to_list @@ - Index.retrieve_l ~limit: fuzz index.index str - ) +let search ?(fuzz = 0) (index : t) (term : string) : (int list list * URI.t) list = + let@ str = List.concat_map @~ Tokenizer.tokenize term in + List.concat_map Ocurrences.to_list @@ + Index.retrieve_l ~limit: fuzz index.index str module BM_25 = struct let sum = List.fold_left (+.) 0. @@ -87,30 +78,25 @@ module BM_25 = struct List.length @@ Tokenizer.tokenize_article d - let score - : T.content T.article -> string -> t -> float - = fun d q index -> + let score (d : T.content T.article) (q : string) (index : t) : float = let tokens = Tokenizer.tokenize q in assert (List.length tokens > 0); let avg_len = average_doc_length index in let k_1 = 1.5 in let b = 0.75 in sum @@ - List.map - (fun q_i -> - let num_occurrences = - Float.of_int @@ - List.length @@ search index q_i - in - (* Format.printf "num_occurrences: %f" num_occurrences; *) - idf q index *. - begin - (num_occurrences *. k_1 +. 1.) /. - (num_occurrences +. k_1 *. (1. -. b +. (b *. doc_length d /. avg_len))) +. - 1. - end - ) - tokens + let@ q_i = List.map @~ tokens in + let num_occurrences = + Float.of_int @@ + List.length @@ search index q_i + in + (* Format.printf "num_occurrences: %f" num_occurrences; *) + idf q index *. + begin + (num_occurrences *. k_1 +. 1.) /. + (num_occurrences +. k_1 *. (1. -. b +. (b *. doc_length d /. avg_len))) +. + 1. + end end let create articles = @@ -124,12 +110,10 @@ let create articles = let marshal (v : t) filename = let oc = open_out_bin filename in - Fun.protect - ~finally: (fun () -> close_out oc) - (fun () -> Marshal.to_channel oc v []) + let@ () = Fun.protect ~finally: (fun () -> close_out oc) in + Marshal.to_channel oc v [] let unmarshal filename : t = let ic = open_in_bin filename in - Fun.protect - ~finally: (fun () -> close_in ic) - (fun () -> Marshal.from_channel ic) + let@ () = Fun.protect ~finally: (fun () -> close_in ic) in + Marshal.from_channel ic diff --git a/lib/search/Search_engine.ml b/lib/search/Search_engine.ml index 3f8dad8..a8ea5ce 100644 --- a/lib/search/Search_engine.ml +++ b/lib/search/Search_engine.ml @@ -10,7 +10,6 @@ open Forester_compiler open Forester_search open Forester_frontend - open struct module T = Types end let ranked_search