diff --git a/lib/language_server/Completion_context.ml b/lib/language_server/Completion_context.ml index 780834b..43ca7d0 100644 --- a/lib/language_server/Completion_context.ml +++ b/lib/language_server/Completion_context.ml @@ -90,16 +90,12 @@ let purpose_for ~pending_head ~enclosing_purpose ~prev_sibling_purpose | _ -> enclosing_purpose end -let dispatch lexbuf = - match Stack.top Lexer.mode_stack with - | Lexer.Main -> Lexer.token lexbuf - | Ident_init -> Lexer.ident_init lexbuf - | Ident_fragments -> Lexer.ident_fragments lexbuf - | Verbatim (herald, buffer) -> Lexer.verbatim herald buffer lexbuf - -let reset_lexer_state () = - Stack.clear Lexer.mode_stack; - Stack.push Lexer.Main Lexer.mode_stack +let dispatch stack lexbuf = + match Stack.top stack with + | Lexer.Main -> Lexer.token stack lexbuf + | Ident_init -> Lexer.ident_init stack lexbuf + | Ident_fragments -> Lexer.ident_fragments stack lexbuf + | Verbatim (herald, buffer) -> Lexer.verbatim stack herald buffer lexbuf let classify ~(document : Lsp.Text_document.t) ~(position : L.Position.t) : t = let text = Lsp.Text_document.text document in @@ -146,12 +142,12 @@ let classify ~(document : Lsp.Text_document.t) ~(position : L.Position.t) : t = pending_head := None | tok -> pending_head := token_head tok in - reset_lexer_state (); + let stack = Lexer.create_mode_stack () in let lexbuf = Lexing.from_string text in (try let continue_ = ref true in while !continue_ do - let toks = dispatch lexbuf in + let toks = dispatch stack lexbuf in if Lexing.lexeme_end lexbuf > cursor then continue_ := false else begin List.iter handle_token toks; @@ -159,5 +155,4 @@ let classify ~(document : Lsp.Text_document.t) ~(position : L.Position.t) : t = end done with _ -> ()); - reset_lexer_state (); {enclosing = List.filter_map (fun {frame; _} -> frame) !levels} diff --git a/lib/parser/Lexer.mll b/lib/parser/Lexer.mll index d43fdf3..522d267 100644 --- a/lib/parser/Lexer.mll +++ b/lib/parser/Lexer.mll @@ -8,12 +8,14 @@ open Forester_prelude type mode = Main | Ident_init | Ident_fragments | Verbatim of string * Buffer.t - let mode_stack = Stack.of_seq @@ List.to_seq [Main] + type mode_stack = mode Stack.t - let push_mode mode = Stack.push mode mode_stack - let drop_mode () = Stack.drop mode_stack - let set_mode mode = drop_mode(); push_mode mode - let push_verbatim_mode herald = push_mode @@ Verbatim (herald, Buffer.create 2000) + let create_mode_stack () : mode_stack = Stack.of_seq @@ List.to_seq [Main] + + let push_mode stack mode = Stack.push mode stack + let drop_mode stack = Stack.drop stack + let set_mode stack mode = drop_mode stack; push_mode stack mode + let push_verbatim_mode stack herald = push_mode stack @@ Verbatim (herald, Buffer.create 2000) exception SyntaxError of string @@ -34,9 +36,9 @@ let newline_followed_by_ws = (newline) (wschar)* let text = [^' ' '%' '#' '\\' '{' '}' '[' ']' '(' ')' '\r' '\n']+ let verbatim_herald = [^' ' '\t' '\r' '\n' '|']+ -rule token = parse - | "\\" { push_mode Ident_init; [] } - | "%" { comment lexbuf } +rule token stack = parse + | "\\" { push_mode stack Ident_init; [] } + | "%" { comment stack lexbuf } | "##{" { [Tokens.HASH_HASH_LBRACE] } | "#{" { [Tokens.HASH_LBRACE] } | "'" { [Tokens.TICK] } @@ -57,48 +59,48 @@ rule token = parse | eof { [Tokens.EOF] } | _ { raise_err lexbuf } -and ident_init = parse - | "verb" (verbatim_herald as herald) '|' { drop_mode (); push_verbatim_mode herald; [] } - | "startverb" { drop_mode (); push_verbatim_mode "\\stopverb"; [] } - | "scope" { drop_mode (); [Tokens.SCOPE] } - | "put" { drop_mode (); [Tokens.PUT] } - | "put?" { drop_mode (); [Tokens.DEFAULT] } - | "get" { drop_mode (); [Tokens.GET] } - | "import" { drop_mode (); [Tokens.IMPORT] } - | "export" { drop_mode (); [Tokens.EXPORT] } - | "namespace" { drop_mode (); [Tokens.NAMESPACE] } - | "open" { drop_mode (); [Tokens.OPEN] } - | "def" { drop_mode (); [Tokens.DEF] } - | "alloc" { drop_mode (); [Tokens.ALLOC] } - | "let" { drop_mode (); [Tokens.LET] } - | "fun" { drop_mode (); [Tokens.FUN] } - | "subtree" { drop_mode (); [Tokens.SUBTREE] } - | "object" { drop_mode (); [Tokens.OBJECT] } - | "patch" { drop_mode (); [Tokens.PATCH] } - | "call" { drop_mode (); [Tokens.CALL] } - | "datalog" { drop_mode (); [Tokens.DATALOG] } - | "<" (xml_base_ident as prefix) ':' (xml_base_ident as uname) ">" { drop_mode (); [XML_ELT_IDENT (Some prefix, uname)] } - | "<" (xml_base_ident as uname) ">" { drop_mode (); [XML_ELT_IDENT (None, uname)] } - | "xmlns:" (xml_base_ident as str) { drop_mode (); [DECL_XMLNS str] } - | "%" { drop_mode (); [Tokens.TEXT "%"] } - | (simple_name as s) "/" { set_mode Ident_fragments; [Tokens.IDENT s; Tokens.SLASH] } - | simple_name as s { drop_mode (); [Tokens.IDENT s] } - | special_name as c { drop_mode (); [Tokens.IDENT (String.make 1 c)] } - | newline { drop_mode (); Lexing.new_line lexbuf; raise_err lexbuf } +and ident_init stack = parse + | "verb" (verbatim_herald as herald) '|' { drop_mode stack; push_verbatim_mode stack herald; [] } + | "startverb" { drop_mode stack; push_verbatim_mode stack "\\stopverb"; [] } + | "scope" { drop_mode stack; [Tokens.SCOPE] } + | "put" { drop_mode stack; [Tokens.PUT] } + | "put?" { drop_mode stack; [Tokens.DEFAULT] } + | "get" { drop_mode stack; [Tokens.GET] } + | "import" { drop_mode stack; [Tokens.IMPORT] } + | "export" { drop_mode stack; [Tokens.EXPORT] } + | "namespace" { drop_mode stack; [Tokens.NAMESPACE] } + | "open" { drop_mode stack; [Tokens.OPEN] } + | "def" { drop_mode stack; [Tokens.DEF] } + | "alloc" { drop_mode stack; [Tokens.ALLOC] } + | "let" { drop_mode stack; [Tokens.LET] } + | "fun" { drop_mode stack; [Tokens.FUN] } + | "subtree" { drop_mode stack; [Tokens.SUBTREE] } + | "object" { drop_mode stack; [Tokens.OBJECT] } + | "patch" { drop_mode stack; [Tokens.PATCH] } + | "call" { drop_mode stack; [Tokens.CALL] } + | "datalog" { drop_mode stack; [Tokens.DATALOG] } + | "<" (xml_base_ident as prefix) ':' (xml_base_ident as uname) ">" { drop_mode stack; [XML_ELT_IDENT (Some prefix, uname)] } + | "<" (xml_base_ident as uname) ">" { drop_mode stack; [XML_ELT_IDENT (None, uname)] } + | "xmlns:" (xml_base_ident as str) { drop_mode stack; [DECL_XMLNS str] } + | "%" { drop_mode stack; [Tokens.TEXT "%"] } + | (simple_name as s) "/" { set_mode stack Ident_fragments; [Tokens.IDENT s; Tokens.SLASH] } + | simple_name as s { drop_mode stack; [Tokens.IDENT s] } + | special_name as c { drop_mode stack; [Tokens.IDENT (String.make 1 c)] } + | newline { drop_mode stack; Lexing.new_line lexbuf; raise_err lexbuf } | _ { raise_err lexbuf } -and ident_fragments = parse +and ident_fragments stack = parse | (simple_name as s) "/" { [Tokens.IDENT s; Tokens.SLASH] } - | simple_name as s { drop_mode (); [Tokens.IDENT s] } - | newline { drop_mode (); Lexing.new_line lexbuf; raise_err lexbuf } + | simple_name as s { drop_mode stack; [Tokens.IDENT s] } + | newline { drop_mode stack; Lexing.new_line lexbuf; raise_err lexbuf } | _ { raise_err lexbuf } -and comment = parse - | newline_followed_by_ws { Lexing.new_line lexbuf; token lexbuf } +and comment stack = parse + | newline_followed_by_ws { Lexing.new_line lexbuf; token stack lexbuf } | eof { [Tokens.EOF] } - | _ { comment lexbuf } + | _ { comment stack lexbuf } -and verbatim herald buffer = parse +and verbatim stack herald buffer = parse | newline as c { Lexing.new_line lexbuf; @@ -117,7 +119,7 @@ and verbatim herald buffer = parse String_util.trim_newlines @@ Buffer.sub buffer 0 offset in - drop_mode (); + drop_mode stack; [Tokens.VERBATIM text] else [] diff --git a/lib/parser/Parse.ml b/lib/parser/Parse.ml index 1b3e791..acd6b32 100644 --- a/lib/parser/Parse.ml +++ b/lib/parser/Parse.ml @@ -24,19 +24,20 @@ let buffer_lexer lexer = in loop -let lexer = - let@ lexbuf = buffer_lexer in - match Stack.top @@ Lexer.mode_stack with - | Main -> Lexer.token lexbuf - | Ident_init -> Lexer.ident_init lexbuf - | Ident_fragments -> Lexer.ident_fragments lexbuf - | Verbatim (herald, buffer) -> Lexer.verbatim herald buffer lexbuf +let make_lexer () = + let stack = Lexer.create_mode_stack () in + buffer_lexer @@ fun lexbuf -> + match Stack.top stack with + | Main -> Lexer.token stack lexbuf + | Ident_init -> Lexer.ident_init stack lexbuf + | Ident_fragments -> Lexer.ident_fragments stack lexbuf + | Verbatim (herald, buffer) -> Lexer.verbatim stack herald buffer lexbuf let parse source lexbuf = let module Parser = Grammar.Make (struct let v = source end) in - try ok @@ Parser.main lexer lexbuf with + try ok @@ Parser.main (make_lexer ()) lexbuf with | Parser.Error -> let range = Range.of_lexbuf ~source lexbuf in error @@ {msg = "parse error"; range}