From ed5dcd71dad4b9966cd1f8a5c0d7f69ee074bfa9 Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Sat, 29 Jun 2019 18:58:07 -0700 Subject: [PATCH] vtparse: document things --- vtparse/Cargo.toml | 6 +++ vtparse/README.md | 20 ++++++++++ vtparse/build.rs | 4 +- vtparse/src/lib.rs | 92 ++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 vtparse/README.md diff --git a/vtparse/Cargo.toml b/vtparse/Cargo.toml index 70856d0e4..9b1a5ccb7 100644 --- a/vtparse/Cargo.toml +++ b/vtparse/Cargo.toml @@ -3,6 +3,12 @@ authors = ["Wez Furlong "] name = "vtparse" version = "0.1.0" edition = "2018" +repository = "https://github.com/wez/wezterm" +description = "Low level escape sequence parser" +license = "MIT" +documentation = "https://docs.rs/termwiz" +keywords = ["terminal", "escape", "ansi", "sequence", "parser"] +readme = "README.md" [dependencies] utf8parse = "0.1" diff --git a/vtparse/README.md b/vtparse/README.md new file mode 100644 index 000000000..fa01355ff --- /dev/null +++ b/vtparse/README.md @@ -0,0 +1,20 @@ +# vtparse + +This is an implementation of a parser for escape and control sequences. +It is based on the [DEC ANSI Parser](https://vt100.net/emu/dec_ansi_parser). + +It has been modified slightly to support UTF-8 sequences. + +`vtparse` is the lowest level parser; it categorizes the basic +types of sequences but does not ascribe any semantic meaning +to them. + +You may wish to look at `termwiz::escape::parser::Parser` in the +[termwiz](https://docs.rs/termwiz) crate if you're looking for semantic +parsing. + +## Comparison with the `vte` crate + +`vtparse` has support for dynamically sized OSC buffers, which makes +it suitable for processing large escape sequences, such as those +used by the `iTerm2` image protocol. diff --git a/vtparse/build.rs b/vtparse/build.rs index 4ffcf4e1e..f8381f588 100644 --- a/vtparse/build.rs +++ b/vtparse/build.rs @@ -335,7 +335,7 @@ fn pack(action: Action, state: State) -> u8 { fn write_tables(dest_path: std::path::PathBuf, tables: &Tables) -> std::io::Result<()> { let mut f = std::fs::File::create(&dest_path)?; - writeln!(f, "pub static TRANSITIONS: [[u8; 256]; 14] = [")?; + writeln!(f, "static TRANSITIONS: [[u8; 256]; 14] = [")?; for state_num in State::Ground as u8..State::Anywhere as u8 { let this_state = State::from_u8(state_num); writeln!(f, " // State: {:?}", this_state)?; @@ -365,7 +365,7 @@ fn event_table( f: &mut std::fs::File, table: &HashMap, ) -> std::io::Result<()> { - writeln!(f, "pub static {}: [Action; 14] = [", label)?; + writeln!(f, "static {}: [Action; 14] = [", label)?; for state_num in State::Ground as u8..State::Anywhere as u8 { let this_state = State::from_u8(state_num); let action = table.get(&this_state).cloned().unwrap_or(Action::None); diff --git a/vtparse/src/lib.rs b/vtparse/src/lib.rs index 732b4ca0e..40e59addf 100644 --- a/vtparse/src/lib.rs +++ b/vtparse/src/lib.rs @@ -1,5 +1,15 @@ //! An implementation of the state machine described by -//! https://vt100.net/emu/dec_ansi_parser +//! [DEC ANSI Parser](https://vt100.net/emu/dec_ansi_parser), modified to support UTF-8. +//! +//! This is sufficient to broadly categorize ANSI/ECMA-48 escape sequences that are +//! commonly used in terminal emulators. It does not ascribe semantic meaning to +//! those escape sequences; for example, if you wish to parse the SGR sequence +//! that makes text bold, you will need to know which codes correspond to bold +//! in your implementation of `VTActor`. +//! +//! You may wish to use `termwiz::escape::parser::Parser` in the +//! [termwiz](https://docs.rs/termwiz/) crate if you don't want to have to research +//! all those possible escape sequences for yourself. use utf8parse::Parser as Utf8Parser; mod enums; use crate::enums::*; @@ -26,10 +36,40 @@ fn lookup_exit(state: State) -> Action { unsafe { *EXIT.get_unchecked(state as usize) } } -/// Terminology: an intermediate is a character in the range 0x20-0x2f +/// `VTActor` is a trait that allows the host application to process +/// the different kinds of sequence as they are parsed from the input +/// stream. +/// +/// The functions defined by this trait correspond to the actions defined +/// in the [state machine](https://vt100.net/emu/dec_ansi_parser). +/// +/// ## Terminology: +/// An intermediate is a character in the range 0x20-0x2f that +/// occurs before the final character in an escape sequence. +/// +/// `ignored_excess_intermediates` is a boolean that is set in the case +/// where there were more than two intermediate characters; no standard +/// defines any codes with more than two. Intermediates after +/// the second will set this flag and are discarded. +/// +/// `params` in most of the functions of this trait are decimal integer parameters in escape +/// sequences. They are separated by semicolon characters. An omitted parameter is returned in +/// this interface as a zero, which represents the default value for that parameter. +/// +/// Other jargon used here is defined in +/// [ECMA-48](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-48,%202nd%20Edition,%20August%201979.pdf). pub trait VTActor { /// The current code should be mapped to a glyph according to the character set mappings and - /// shift states in effect, and that glyph should be displayed. GL characters (20 to 7F) are + /// shift states in effect, and that glyph should be displayed. + /// + /// If the input was UTF-8 then it will have been mapped to a unicode code point. Invalid + /// sequences are represented here using the unicode REPLACEMENT_CHARACTER. + /// + /// Otherwise the parameter will be a 7-bit printable value and may be subject to mapping + /// depending on other state maintained by the embedding application. + /// + /// ## Some commentary from the state machine documentation: + /// GL characters (20 to 7F) are /// printed. 20 (SP) and 7F (DEL) are included in this area, although both codes have special /// behaviour. If a 94-character set is mapped into GL, 20 will cause a space to be displayed, /// and 7F will be ignored. When a 96-character set is mapped into GL, both 20 and 7F may cause @@ -44,12 +84,18 @@ pub trait VTActor { /// The C0 or C1 control function should be executed, which may have any one of a variety of /// effects, including changing the cursor position, suspending or resuming communications or /// changing the shift states in effect. + /// + /// See [ECMA-48](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-48,%202nd%20Edition,%20August%201979.pdf) + /// for more information on C0 and C1 control functions. fn execute_c0_or_c1(&mut self, control: u8); /// invoked when a final character arrives in the first part of a device control string. It /// determines the control function from the private marker, intermediate character(s) and /// final character, and executes it, passing in the parameter list. It also selects a handler /// function for the rest of the characters in the control string. + /// + /// See [ECMA-48](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-48,%202nd%20Edition,%20August%201979.pdf) + /// for more information on device control strings. fn dcs_hook( &mut self, params: &[i64], @@ -60,10 +106,24 @@ pub trait VTActor { /// This action passes characters from the data string part of a device control string to a /// handler that has previously been selected by the dcs_hook action. C0 controls are also /// passed to the handler. + /// + /// See [ECMA-48](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-48,%202nd%20Edition,%20August%201979.pdf) + /// for more information on device control strings. fn dcs_put(&mut self, byte: u8); + /// When a device control string is terminated by ST, CAN, SUB or ESC, this action calls the + /// previously selected handler function with an “end of data” parameter. This allows the + /// handler to finish neatly. + /// + /// See [ECMA-48](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-48,%202nd%20Edition,%20August%201979.pdf) + /// for more information on device control strings. fn dcs_unhook(&mut self); + /// The final character of an escape sequence has arrived, so determine the control function + /// to be executed from the intermediate character(s) and final character, and execute it. + /// + /// See [ECMA-48](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-48,%202nd%20Edition,%20August%201979.pdf) + /// for more information on escape sequences. fn esc_dispatch( &mut self, params: &[i64], @@ -71,6 +131,13 @@ pub trait VTActor { ignored_excess_intermediates: bool, byte: u8, ); + + /// A final character of a Control Sequence Initiator has arrived, so determine the control function to be executed from + /// private marker, intermediate character(s) and final character, and execute it, passing in + /// the parameter list. + /// + /// See [ECMA-48](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-48,%202nd%20Edition,%20August%201979.pdf) + /// for more information on control functions. fn csi_dispatch( &mut self, params: &[i64], @@ -78,9 +145,18 @@ pub trait VTActor { ignored_excess_intermediates: bool, byte: u8, ); + + /// Called when an OSC string is terminated by ST, CAN, SUB or ESC. + /// + /// `params` is an array of byte strings (which may also be valid utf-8) + /// that were passed as semicolon separated parameters to the operating + /// system command. fn osc_dispatch(&mut self, params: &[&[u8]]); } +/// `VTAction` is an alternative way to work with the parser; rather +/// than implementing the VTActor trait you can use `CollectingVTActor` +/// to capture the sequence of events into a `Vec`. #[derive(Debug, Clone, Eq, PartialEq)] pub enum VTAction { Print(char), @@ -107,6 +183,10 @@ pub enum VTAction { OscDispatch(Vec>), } +/// This is an implementation of `VTActor` that captures the events +/// into an internal vector. +/// It can be iterated via `into_iter` or have the internal +/// vector extracted via `into_vec`. #[derive(Default)] pub struct CollectingVTActor { actions: Vec, @@ -198,6 +278,7 @@ const MAX_INTERMEDIATES: usize = 2; const MAX_OSC: usize = 16; const MAX_PARAMS: usize = 16; +/// The virtual terminal parser. It works together with an implementation of `VTActor`. pub struct VTParser { state: State, @@ -402,6 +483,8 @@ impl VTParser { self.utf8_parser.advance(&mut decoder, byte); } + /// Parse a single byte. This may result in a call to one of the + /// methods on the provided `actor`. #[inline(always)] pub fn parse_byte(&mut self, byte: u8, actor: &mut dyn VTActor) { // While in utf-8 parsing mode, co-opt the vt state @@ -425,6 +508,9 @@ impl VTParser { } } + /// Parse a sequence of bytes. The sequence need not be complete. + /// This may result in some number of calls to the methods on the + /// provided `actor`. pub fn parse(&mut self, bytes: &[u8], actor: &mut dyn VTActor) { for b in bytes { self.parse_byte(*b, actor); -- 2.51.2