diff --git a/README.md b/README.md index 4900517..bbe47da 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,26 @@ -# tim +# william -[![Package Version](https://img.shields.io/hexpm/v/tim)](https://hex.pm/packages/tim) -[![Hex Docs](https://img.shields.io/badge/hex-docs-ffaff3)](https://hexdocs.pm/tim/) +[![Package Version](https://img.shields.io/hexpm/v/william)](https://hex.pm/packages/william) +[![Hex Docs](https://img.shields.io/badge/hex-docs-ffaff3)](https://hexdocs.pm/william/) -`tim` is a TOML lexer and syntax highlighter for Gleam. It parses TOML source +`william` is a TOML lexer and syntax highlighter for Gleam. It parses TOML source into a recoverable token stream and can render those tokens as ANSI or HTML. +> The name comes from William Hanna, one of the original Tom & Jerry creators +> alongside Joseph Barbera. + ```sh -gleam add tim@1 +gleam add william@1 ``` ```gleam import gleam/io -import tim +import william pub fn main() -> Nil { let html = "name = \"TOML\"\n" - |> tim.highlight - |> tim.to_html + |> william.highlight + |> william.to_html io.println(html) } @@ -56,7 +59,7 @@ pre code { } ``` -Further documentation can be found at . +Further documentation can be found at . ## Development diff --git a/dev/tim_dev.gleam b/dev/william_dev.gleam similarity index 81% rename from dev/tim_dev.gleam rename to dev/william_dev.gleam index 8749c7d..634e4b4 100644 --- a/dev/tim_dev.gleam +++ b/dev/william_dev.gleam @@ -1,7 +1,7 @@ import argv import gleam/io import simplifile -import tim +import william pub fn main() -> Nil { let source_path = case argv.load() { @@ -12,7 +12,7 @@ pub fn main() -> Nil { io.println( source - |> tim.highlight - |> tim.to_ansi, + |> william.highlight + |> william.to_ansi, ) } diff --git a/gleam.toml b/gleam.toml index 5c9679c..5fedebd 100644 --- a/gleam.toml +++ b/gleam.toml @@ -1,4 +1,4 @@ -name = "tim" +name = "william" version = "1.0.0" # Fill out these fields if you intend to generate HTML documentation or publish @@ -6,7 +6,7 @@ version = "1.0.0" # description = "A TOML lexer and syntax highlighter for Gleam" licences = ["BSD-3-Clause"] -repository = { type = "gitlab", user = "arkandos", repo = "tim" } +repository = { type = "gitlab", user = "arkandos", repo = "william" } links = [ { title = "contour", href = "https://hexdocs.pm/contour" }, { title = "just", href = "https://hexdocs.pm/just" }, diff --git a/src/tim.gleam b/src/william.gleam similarity index 100% rename from src/tim.gleam rename to src/william.gleam diff --git a/test/tim_test.gleam b/test/tim_test.gleam deleted file mode 100644 index 88ee862..0000000 --- a/test/tim_test.gleam +++ /dev/null @@ -1,493 +0,0 @@ -import gleam/string -import gleam_community/ansi -import gleeunit -import tim - -pub fn main() -> Nil { - gleeunit.main() -} - -pub fn key_value_test() { - assert_tokens("title = \"TOML Example\"", [ - tim.BareKey("title"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.String(tim.BasicString, "TOML Example"), - ]) -} - -pub fn dotted_and_quoted_keys_test() { - assert_tokens("site.\"google.com\".'owner' = true", [ - tim.BareKey("site"), - tim.Dot, - tim.String(tim.BasicString, "google.com"), - tim.Dot, - tim.String(tim.LiteralString, "owner"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.Boolean("true"), - ]) -} - -pub fn numeric_keys_stay_keys_test() { - assert_tokens("1234 = 5678", [ - tim.BareKey("1234"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.Integer(tim.Decimal, "5678"), - ]) -} - -pub fn tables_test() { - assert_tokens("[servers.alpha]\n[[products]]", [ - tim.OpenTable, - tim.BareKey("servers"), - tim.Dot, - tim.BareKey("alpha"), - tim.CloseTable, - tim.EndOfLine("\n"), - tim.OpenArrayTable, - tim.BareKey("products"), - tim.CloseArrayTable, - ]) -} - -pub fn values_test() { - assert_tokens("enabled = false\nports = [ 8000, 0xDEAD_BEEF, inf ]", [ - tim.BareKey("enabled"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.Boolean("false"), - tim.EndOfLine("\n"), - tim.BareKey("ports"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.OpenBracket, - tim.Whitespace(" "), - tim.Integer(tim.Decimal, "8000"), - tim.Comma, - tim.Whitespace(" "), - tim.Integer(tim.Hexadecimal, "0xDEAD_BEEF"), - tim.Comma, - tim.Whitespace(" "), - tim.Float("inf"), - tim.Whitespace(" "), - tim.CloseBracket, - ]) -} - -pub fn decimal_integer_grammar_test() { - assert_value("0", tim.Integer(tim.Decimal, "0")) - assert_value("1", tim.Integer(tim.Decimal, "1")) - assert_value("+234", tim.Integer(tim.Decimal, "+234")) - assert_value("-234", tim.Integer(tim.Decimal, "-234")) - assert_value("00", tim.Integer(tim.Decimal, "00")) - assert_value("01", tim.Integer(tim.Decimal, "01")) - assert_value("+01", tim.Integer(tim.Decimal, "+01")) - assert_value("-01", tim.Integer(tim.Decimal, "-01")) - assert_value("1_000_009", tim.Integer(tim.Decimal, "1_000_009")) - - assert_invalid_value("+") - assert_invalid_value("-") - assert_invalid_value("1_") - assert_invalid_value("1__2") - assert_invalid_value("1_000__009") -} - -pub fn based_integer_grammar_test() { - assert_value("0b101010", tim.Integer(tim.Binary, "0b101010")) - assert_value("0b1_010_10", tim.Integer(tim.Binary, "0b1_010_10")) - assert_value("0o1234567", tim.Integer(tim.Octal, "0o1234567")) - assert_value("0o1_23_45_67", tim.Integer(tim.Octal, "0o1_23_45_67")) - assert_value("0xdeadbeef", tim.Integer(tim.Hexadecimal, "0xdeadbeef")) - assert_value("0xDEADBEEF", tim.Integer(tim.Hexadecimal, "0xDEADBEEF")) - assert_value( - "0xd_e_a_d_b_e_e_f", - tim.Integer(tim.Hexadecimal, "0xd_e_a_d_b_e_e_f"), - ) - - assert_invalid_value("0b") - assert_invalid_value("0b_1") - assert_invalid_value("0b102") - assert_invalid_value("0b1__0") - assert_invalid_value("0b1_") - assert_invalid_value("+0b101010") - assert_invalid_value("-0b101010") - assert_invalid_value("0o8") - assert_invalid_value("0o1__7") - assert_invalid_value("+0o1234567") - assert_invalid_value("-0o1234567") - assert_invalid_value("0x") - assert_invalid_value("0x_DEAD") - assert_invalid_value("0xDEAD_") - assert_invalid_value("0xdead__beef") - assert_invalid_value("+0xDEADBEEF") - assert_invalid_value("-0xDEADBEEF") -} - -pub fn float_grammar_test() { - assert_value("1.0", tim.Float("1.0")) - assert_value("+1.0", tim.Float("+1.0")) - assert_value("-1.0", tim.Float("-1.0")) - assert_value("123456789.9876", tim.Float("123456789.9876")) - assert_value("1e6", tim.Float("1e6")) - assert_value("1E6", tim.Float("1E6")) - assert_value("5e+22", tim.Float("5e+22")) - assert_value("-2e-22", tim.Float("-2e-22")) - assert_value("6.626e25", tim.Float("6.626e25")) - assert_value("6.626e+25", tim.Float("6.626e+25")) - assert_value("6.626e-25", tim.Float("6.626e-25")) - assert_value("01.0", tim.Float("01.0")) - assert_value("01e2", tim.Float("01e2")) - assert_value("224_617.445_991_228", tim.Float("224_617.445_991_228")) - - assert_invalid_value(".7") - assert_invalid_value("7.") - assert_invalid_value("3.e+20") - assert_invalid_value("1e") - assert_invalid_value("1e+") - assert_invalid_value("1e_2") - assert_invalid_value("1_e2") - assert_invalid_value("1_.0") - assert_invalid_value("1._0") - assert_invalid_value("1.0_") - assert_invalid_value("1__0.0") - assert_invalid_value("1.0__0") -} - -pub fn special_float_grammar_test() { - assert_value("nan", tim.Float("nan")) - assert_value("+nan", tim.Float("+nan")) - assert_value("-nan", tim.Float("-nan")) - assert_value("inf", tim.Float("inf")) - assert_value("+inf", tim.Float("+inf")) - assert_value("-inf", tim.Float("-inf")) -} - -pub fn inline_tables_test() { - assert_tokens("point = { x = 1, y = 2 }", [ - tim.BareKey("point"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.OpenBrace, - tim.Whitespace(" "), - tim.BareKey("x"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.Integer(tim.Decimal, "1"), - tim.Comma, - tim.Whitespace(" "), - tim.BareKey("y"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.Integer(tim.Decimal, "2"), - tim.Whitespace(" "), - tim.CloseBrace, - ]) -} - -pub fn date_time_test() { - assert_tokens("dob = 1979-05-27T07:32:00Z\nlocal = 1979-05-27 07:32", [ - tim.BareKey("dob"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.DateTime("1979-05-27T07:32:00Z"), - tim.EndOfLine("\n"), - tim.BareKey("local"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.DateTime("1979-05-27 07:32"), - ]) -} - -pub fn date_time_grammar_test() { - assert_value("1979-05-27", tim.DateTime("1979-05-27")) - assert_value("1980-02-29", tim.DateTime("1980-02-29")) - assert_value("1979-02-29", tim.DateTime("1979-02-29")) - assert_value("1979-13-01", tim.DateTime("1979-13-01")) - assert_value("1979-00-01", tim.DateTime("1979-00-01")) - assert_value("1979-05-00", tim.DateTime("1979-05-00")) - assert_value("1979-04-31", tim.DateTime("1979-04-31")) - assert_value("07:32:01", tim.DateTime("07:32:01")) - assert_value("07:00:01", tim.DateTime("07:00:01")) - assert_value("24:00", tim.DateTime("24:00")) - assert_value("07:60", tim.DateTime("07:60")) - assert_value("07:32:61", tim.DateTime("07:32:61")) - assert_value("07:32:01.999999", tim.DateTime("07:32:01.999999")) - assert_value("07:32:01.09179", tim.DateTime("07:32:01.09179")) - assert_value("07:32:01.123456789", tim.DateTime("07:32:01.123456789")) - assert_value("07:32:01.1", tim.DateTime("07:32:01.1")) - assert_value("07:32:01.001", tim.DateTime("07:32:01.001")) - assert_value("07:32:01.000000789", tim.DateTime("07:32:01.000000789")) - assert_value("07:32", tim.DateTime("07:32")) - assert_value("1979-05-27T07:32:00", tim.DateTime("1979-05-27T07:32:00")) - assert_value("1979-05-27t07:32:00", tim.DateTime("1979-05-27t07:32:00")) - assert_value("1979-05-27 07:00:01", tim.DateTime("1979-05-27 07:00:01")) - assert_value("1979-05-27T07:32:00Z", tim.DateTime("1979-05-27T07:32:00Z")) - assert_value("1979-05-27T07:32:00z", tim.DateTime("1979-05-27T07:32:00z")) - assert_value("1979-05-27 07:00:01Z", tim.DateTime("1979-05-27 07:00:01Z")) - assert_value( - "1979-05-27 07:00:01+07:40", - tim.DateTime("1979-05-27 07:00:01+07:40"), - ) - assert_value( - "1979-05-27 07:00:01-07:01", - tim.DateTime("1979-05-27 07:00:01-07:01"), - ) - assert_value("1979-05-27T07:32Z", tim.DateTime("1979-05-27T07:32Z")) - assert_value("1979-05-27T07:32+00:00", tim.DateTime("1979-05-27T07:32+00:00")) - assert_value("1979-05-27T24:00", tim.DateTime("1979-05-27T24:00")) - assert_value("1979-05-27T07:32+24:00", tim.DateTime("1979-05-27T07:32+24:00")) - assert_value("1979-05-27T07:32+00:60", tim.DateTime("1979-05-27T07:32+00:60")) - - assert_invalid_value("1979-5-27") - assert_invalid_value("1979-05-2") - assert_invalid_value("7:32") - assert_invalid_value("07:3") - assert_invalid_value("07:32:") - assert_invalid_value("07:32:01.") - assert_invalid_value("07:32Z") - assert_invalid_value("1979-05-27T") - assert_invalid_value("1979-05-27T07:32+0:00") - assert_invalid_value("1979-05-27T07:32+00:0") - assert_invalid_value("1979-05-27T07:32+0000") -} - -pub fn multiline_strings_test() { - assert_tokens("poem = \"\"\"\nroses\n\"\"\"", [ - tim.BareKey("poem"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.String(tim.MultilineBasicString, "\nroses\n"), - ]) - - assert_tokens("regex = '''\\d+'''", [ - tim.BareKey("regex"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.String(tim.MultilineLiteralString, "\\d+"), - ]) -} - -pub fn ported_key_and_string_fixtures_test() { - assert_tokens("\" \" = true\n'' = true", [ - tim.String(tim.BasicString, " "), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.Boolean("true"), - tim.EndOfLine("\n"), - tim.String(tim.LiteralString, ""), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.Boolean("true"), - ]) - - assert_tokens("one . two . three = true", [ - tim.BareKey("one"), - tim.Whitespace(" "), - tim.Dot, - tim.Whitespace(" "), - tim.BareKey("two"), - tim.Whitespace(" "), - tim.Dot, - tim.Whitespace(" "), - tim.BareKey("three"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.Boolean("true"), - ]) - - assert_tokens("field = \"#\"\nregex = '\\n'", [ - tim.BareKey("field"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.String(tim.BasicString, "#"), - tim.EndOfLine("\n"), - tim.BareKey("regex"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.String(tim.LiteralString, "\\n"), - ]) -} - -pub fn ported_multiline_string_fixtures_test() { - assert_tokens("a = \"\"\"\nThis string contains a #hash character\n\"\"\"", [ - tim.BareKey("a"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.String( - tim.MultilineBasicString, - "\nThis string contains a #hash character\n", - ), - ]) - - assert_tokens("a = '''\nThis string contains a #hash character\n'''", [ - tim.BareKey("a"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.String( - tim.MultilineLiteralString, - "\nThis string contains a #hash character\n", - ), - ]) -} - -pub fn comment_and_roundtrip_test() { - let source = "# hello\nname = \"Tom\" # inline\n" - let tokens = tim.tokenise(tim.new(), source) - - assert tokens - == [ - tim.Comment("# hello"), - tim.EndOfLine("\n"), - tim.BareKey("name"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.String(tim.BasicString, "Tom"), - tim.Whitespace(" "), - tim.Comment("# inline"), - tim.EndOfLine("\n"), - ] - - assert tim.to_source(tokens) == source -} - -pub fn trailing_and_unexpected_fixtures_test() { - assert_tokens("a = 1", [ - tim.BareKey("a"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.Integer(tim.Decimal, "1"), - ]) - - assert_tokens("a = 1 ", [ - tim.BareKey("a"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.Integer(tim.Decimal, "1"), - tim.Whitespace(" "), - ]) - - assert_tokens("a = 1 b", [ - tim.BareKey("a"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - tim.Integer(tim.Decimal, "1"), - tim.Whitespace(" "), - tim.Unexpected("b"), - ]) -} - -pub fn highlighter_test() { - assert tim.highlight("name = 1") - == [ - tim.HighlightKey("name"), - tim.HighlightWhitespace(" "), - tim.HighlightOperator("="), - tim.HighlightWhitespace(" "), - tim.HighlightNumber("1"), - ] - - assert tim.highlight("[dependencies]\nname = 1") - == [ - tim.HighlightPunctuation("["), - tim.HighlightTable("dependencies"), - tim.HighlightPunctuation("]"), - tim.HighlightWhitespace("\n"), - tim.HighlightKey("name"), - tim.HighlightWhitespace(" "), - tim.HighlightOperator("="), - tim.HighlightWhitespace(" "), - tim.HighlightNumber("1"), - ] - - let html = - "name = \"\"" - |> tim.highlight - |> tim.to_html - - assert string.contains(html, "hl-attribute") - assert string.contains(html, "<Tom>") -} - -pub fn ansi_theme_matches_tear_test() { - let highlighted = - [ - tim.HighlightKey("key"), - tim.HighlightOperator("="), - tim.HighlightTable("table"), - tim.HighlightString("\"str\""), - tim.HighlightNumber("1"), - tim.HighlightDateTime("1979-05-27"), - tim.HighlightLiteral("true"), - tim.HighlightComment("# c"), - tim.HighlightError("!"), - tim.HighlightPunctuation("["), - tim.HighlightWhitespace(" "), - ] - |> tim.to_ansi - - assert highlighted - == ansi.yellow("key") - <> ansi.magenta("=") - <> ansi.cyan("table") - <> ansi.green("\"str\"") - <> ansi.green("1") - <> ansi.blue("1979-05-27") - <> ansi.green("true") - <> ansi.italic(ansi.gray("# c")) - <> ansi.bg_bright_red(ansi.white("!")) - <> "[" - <> " " -} - -pub fn ignore_options_test() { - assert tim.tokenise( - tim.new() - |> tim.ignore_whitespace - |> tim.ignore_comments, - "# comment\nname = true", - ) - == [tim.BareKey("name"), tim.Equal, tim.Boolean("true")] -} - -fn assert_tokens(str: String, tokens: List(tim.Token)) { - assert tim.tokenise(tim.new(), str) == tokens as string.inspect(str) -} - -fn assert_value(value: String, token: tim.Token) { - assert_tokens("a = " <> value, [ - tim.BareKey("a"), - tim.Whitespace(" "), - tim.Equal, - tim.Whitespace(" "), - token, - ]) -} - -fn assert_invalid_value(value: String) { - assert_value(value, tim.InvalidNumber(value)) -} diff --git a/test/william_test.gleam b/test/william_test.gleam new file mode 100644 index 0000000..ab61214 --- /dev/null +++ b/test/william_test.gleam @@ -0,0 +1,502 @@ +import gleam/string +import gleam_community/ansi +import gleeunit +import william + +pub fn main() -> Nil { + gleeunit.main() +} + +pub fn key_value_test() { + assert_tokens("title = \"TOML Example\"", [ + william.BareKey("title"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.String(william.BasicString, "TOML Example"), + ]) +} + +pub fn dotted_and_quoted_keys_test() { + assert_tokens("site.\"google.com\".'owner' = true", [ + william.BareKey("site"), + william.Dot, + william.String(william.BasicString, "google.com"), + william.Dot, + william.String(william.LiteralString, "owner"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.Boolean("true"), + ]) +} + +pub fn numeric_keys_stay_keys_test() { + assert_tokens("1234 = 5678", [ + william.BareKey("1234"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.Integer(william.Decimal, "5678"), + ]) +} + +pub fn tables_test() { + assert_tokens("[servers.alpha]\n[[products]]", [ + william.OpenTable, + william.BareKey("servers"), + william.Dot, + william.BareKey("alpha"), + william.CloseTable, + william.EndOfLine("\n"), + william.OpenArrayTable, + william.BareKey("products"), + william.CloseArrayTable, + ]) +} + +pub fn values_test() { + assert_tokens("enabled = false\nports = [ 8000, 0xDEAD_BEEF, inf ]", [ + william.BareKey("enabled"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.Boolean("false"), + william.EndOfLine("\n"), + william.BareKey("ports"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.OpenBracket, + william.Whitespace(" "), + william.Integer(william.Decimal, "8000"), + william.Comma, + william.Whitespace(" "), + william.Integer(william.Hexadecimal, "0xDEAD_BEEF"), + william.Comma, + william.Whitespace(" "), + william.Float("inf"), + william.Whitespace(" "), + william.CloseBracket, + ]) +} + +pub fn decimal_integer_grammar_test() { + assert_value("0", william.Integer(william.Decimal, "0")) + assert_value("1", william.Integer(william.Decimal, "1")) + assert_value("+234", william.Integer(william.Decimal, "+234")) + assert_value("-234", william.Integer(william.Decimal, "-234")) + assert_value("00", william.Integer(william.Decimal, "00")) + assert_value("01", william.Integer(william.Decimal, "01")) + assert_value("+01", william.Integer(william.Decimal, "+01")) + assert_value("-01", william.Integer(william.Decimal, "-01")) + assert_value("1_000_009", william.Integer(william.Decimal, "1_000_009")) + + assert_invalid_value("+") + assert_invalid_value("-") + assert_invalid_value("1_") + assert_invalid_value("1__2") + assert_invalid_value("1_000__009") +} + +pub fn based_integer_grammar_test() { + assert_value("0b101010", william.Integer(william.Binary, "0b101010")) + assert_value("0b1_010_10", william.Integer(william.Binary, "0b1_010_10")) + assert_value("0o1234567", william.Integer(william.Octal, "0o1234567")) + assert_value("0o1_23_45_67", william.Integer(william.Octal, "0o1_23_45_67")) + assert_value("0xdeadbeef", william.Integer(william.Hexadecimal, "0xdeadbeef")) + assert_value("0xDEADBEEF", william.Integer(william.Hexadecimal, "0xDEADBEEF")) + assert_value( + "0xd_e_a_d_b_e_e_f", + william.Integer(william.Hexadecimal, "0xd_e_a_d_b_e_e_f"), + ) + + assert_invalid_value("0b") + assert_invalid_value("0b_1") + assert_invalid_value("0b102") + assert_invalid_value("0b1__0") + assert_invalid_value("0b1_") + assert_invalid_value("+0b101010") + assert_invalid_value("-0b101010") + assert_invalid_value("0o8") + assert_invalid_value("0o1__7") + assert_invalid_value("+0o1234567") + assert_invalid_value("-0o1234567") + assert_invalid_value("0x") + assert_invalid_value("0x_DEAD") + assert_invalid_value("0xDEAD_") + assert_invalid_value("0xdead__beef") + assert_invalid_value("+0xDEADBEEF") + assert_invalid_value("-0xDEADBEEF") +} + +pub fn float_grammar_test() { + assert_value("1.0", william.Float("1.0")) + assert_value("+1.0", william.Float("+1.0")) + assert_value("-1.0", william.Float("-1.0")) + assert_value("123456789.9876", william.Float("123456789.9876")) + assert_value("1e6", william.Float("1e6")) + assert_value("1E6", william.Float("1E6")) + assert_value("5e+22", william.Float("5e+22")) + assert_value("-2e-22", william.Float("-2e-22")) + assert_value("6.626e25", william.Float("6.626e25")) + assert_value("6.626e+25", william.Float("6.626e+25")) + assert_value("6.626e-25", william.Float("6.626e-25")) + assert_value("01.0", william.Float("01.0")) + assert_value("01e2", william.Float("01e2")) + assert_value("224_617.445_991_228", william.Float("224_617.445_991_228")) + + assert_invalid_value(".7") + assert_invalid_value("7.") + assert_invalid_value("3.e+20") + assert_invalid_value("1e") + assert_invalid_value("1e+") + assert_invalid_value("1e_2") + assert_invalid_value("1_e2") + assert_invalid_value("1_.0") + assert_invalid_value("1._0") + assert_invalid_value("1.0_") + assert_invalid_value("1__0.0") + assert_invalid_value("1.0__0") +} + +pub fn special_float_grammar_test() { + assert_value("nan", william.Float("nan")) + assert_value("+nan", william.Float("+nan")) + assert_value("-nan", william.Float("-nan")) + assert_value("inf", william.Float("inf")) + assert_value("+inf", william.Float("+inf")) + assert_value("-inf", william.Float("-inf")) +} + +pub fn inline_tables_test() { + assert_tokens("point = { x = 1, y = 2 }", [ + william.BareKey("point"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.OpenBrace, + william.Whitespace(" "), + william.BareKey("x"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.Integer(william.Decimal, "1"), + william.Comma, + william.Whitespace(" "), + william.BareKey("y"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.Integer(william.Decimal, "2"), + william.Whitespace(" "), + william.CloseBrace, + ]) +} + +pub fn date_time_test() { + assert_tokens("dob = 1979-05-27T07:32:00Z\nlocal = 1979-05-27 07:32", [ + william.BareKey("dob"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.DateTime("1979-05-27T07:32:00Z"), + william.EndOfLine("\n"), + william.BareKey("local"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.DateTime("1979-05-27 07:32"), + ]) +} + +pub fn date_time_grammar_test() { + assert_value("1979-05-27", william.DateTime("1979-05-27")) + assert_value("1980-02-29", william.DateTime("1980-02-29")) + assert_value("1979-02-29", william.DateTime("1979-02-29")) + assert_value("1979-13-01", william.DateTime("1979-13-01")) + assert_value("1979-00-01", william.DateTime("1979-00-01")) + assert_value("1979-05-00", william.DateTime("1979-05-00")) + assert_value("1979-04-31", william.DateTime("1979-04-31")) + assert_value("07:32:01", william.DateTime("07:32:01")) + assert_value("07:00:01", william.DateTime("07:00:01")) + assert_value("24:00", william.DateTime("24:00")) + assert_value("07:60", william.DateTime("07:60")) + assert_value("07:32:61", william.DateTime("07:32:61")) + assert_value("07:32:01.999999", william.DateTime("07:32:01.999999")) + assert_value("07:32:01.09179", william.DateTime("07:32:01.09179")) + assert_value("07:32:01.123456789", william.DateTime("07:32:01.123456789")) + assert_value("07:32:01.1", william.DateTime("07:32:01.1")) + assert_value("07:32:01.001", william.DateTime("07:32:01.001")) + assert_value("07:32:01.000000789", william.DateTime("07:32:01.000000789")) + assert_value("07:32", william.DateTime("07:32")) + assert_value("1979-05-27T07:32:00", william.DateTime("1979-05-27T07:32:00")) + assert_value("1979-05-27t07:32:00", william.DateTime("1979-05-27t07:32:00")) + assert_value("1979-05-27 07:00:01", william.DateTime("1979-05-27 07:00:01")) + assert_value("1979-05-27T07:32:00Z", william.DateTime("1979-05-27T07:32:00Z")) + assert_value("1979-05-27T07:32:00z", william.DateTime("1979-05-27T07:32:00z")) + assert_value("1979-05-27 07:00:01Z", william.DateTime("1979-05-27 07:00:01Z")) + assert_value( + "1979-05-27 07:00:01+07:40", + william.DateTime("1979-05-27 07:00:01+07:40"), + ) + assert_value( + "1979-05-27 07:00:01-07:01", + william.DateTime("1979-05-27 07:00:01-07:01"), + ) + assert_value("1979-05-27T07:32Z", william.DateTime("1979-05-27T07:32Z")) + assert_value( + "1979-05-27T07:32+00:00", + william.DateTime("1979-05-27T07:32+00:00"), + ) + assert_value("1979-05-27T24:00", william.DateTime("1979-05-27T24:00")) + assert_value( + "1979-05-27T07:32+24:00", + william.DateTime("1979-05-27T07:32+24:00"), + ) + assert_value( + "1979-05-27T07:32+00:60", + william.DateTime("1979-05-27T07:32+00:60"), + ) + + assert_invalid_value("1979-5-27") + assert_invalid_value("1979-05-2") + assert_invalid_value("7:32") + assert_invalid_value("07:3") + assert_invalid_value("07:32:") + assert_invalid_value("07:32:01.") + assert_invalid_value("07:32Z") + assert_invalid_value("1979-05-27T") + assert_invalid_value("1979-05-27T07:32+0:00") + assert_invalid_value("1979-05-27T07:32+00:0") + assert_invalid_value("1979-05-27T07:32+0000") +} + +pub fn multiline_strings_test() { + assert_tokens("poem = \"\"\"\nroses\n\"\"\"", [ + william.BareKey("poem"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.String(william.MultilineBasicString, "\nroses\n"), + ]) + + assert_tokens("regex = '''\\d+'''", [ + william.BareKey("regex"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.String(william.MultilineLiteralString, "\\d+"), + ]) +} + +pub fn ported_key_and_string_fixtures_test() { + assert_tokens("\" \" = true\n'' = true", [ + william.String(william.BasicString, " "), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.Boolean("true"), + william.EndOfLine("\n"), + william.String(william.LiteralString, ""), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.Boolean("true"), + ]) + + assert_tokens("one . two . three = true", [ + william.BareKey("one"), + william.Whitespace(" "), + william.Dot, + william.Whitespace(" "), + william.BareKey("two"), + william.Whitespace(" "), + william.Dot, + william.Whitespace(" "), + william.BareKey("three"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.Boolean("true"), + ]) + + assert_tokens("field = \"#\"\nregex = '\\n'", [ + william.BareKey("field"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.String(william.BasicString, "#"), + william.EndOfLine("\n"), + william.BareKey("regex"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.String(william.LiteralString, "\\n"), + ]) +} + +pub fn ported_multiline_string_fixtures_test() { + assert_tokens("a = \"\"\"\nThis string contains a #hash character\n\"\"\"", [ + william.BareKey("a"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.String( + william.MultilineBasicString, + "\nThis string contains a #hash character\n", + ), + ]) + + assert_tokens("a = '''\nThis string contains a #hash character\n'''", [ + william.BareKey("a"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.String( + william.MultilineLiteralString, + "\nThis string contains a #hash character\n", + ), + ]) +} + +pub fn comment_and_roundtrip_test() { + let source = "# hello\nname = \"Tom\" # inline\n" + let tokens = william.tokenise(william.new(), source) + + assert tokens + == [ + william.Comment("# hello"), + william.EndOfLine("\n"), + william.BareKey("name"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.String(william.BasicString, "Tom"), + william.Whitespace(" "), + william.Comment("# inline"), + william.EndOfLine("\n"), + ] + + assert william.to_source(tokens) == source +} + +pub fn trailing_and_unexpected_fixtures_test() { + assert_tokens("a = 1", [ + william.BareKey("a"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.Integer(william.Decimal, "1"), + ]) + + assert_tokens("a = 1 ", [ + william.BareKey("a"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.Integer(william.Decimal, "1"), + william.Whitespace(" "), + ]) + + assert_tokens("a = 1 b", [ + william.BareKey("a"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + william.Integer(william.Decimal, "1"), + william.Whitespace(" "), + william.Unexpected("b"), + ]) +} + +pub fn highlighter_test() { + assert william.highlight("name = 1") + == [ + william.HighlightKey("name"), + william.HighlightWhitespace(" "), + william.HighlightOperator("="), + william.HighlightWhitespace(" "), + william.HighlightNumber("1"), + ] + + assert william.highlight("[dependencies]\nname = 1") + == [ + william.HighlightPunctuation("["), + william.HighlightTable("dependencies"), + william.HighlightPunctuation("]"), + william.HighlightWhitespace("\n"), + william.HighlightKey("name"), + william.HighlightWhitespace(" "), + william.HighlightOperator("="), + william.HighlightWhitespace(" "), + william.HighlightNumber("1"), + ] + + let html = + "name = \"\"" + |> william.highlight + |> william.to_html + + assert string.contains(html, "hl-attribute") + assert string.contains(html, "<Tom>") +} + +pub fn ansi_theme_matches_tear_test() { + let highlighted = + [ + william.HighlightKey("key"), + william.HighlightOperator("="), + william.HighlightTable("table"), + william.HighlightString("\"str\""), + william.HighlightNumber("1"), + william.HighlightDateTime("1979-05-27"), + william.HighlightLiteral("true"), + william.HighlightComment("# c"), + william.HighlightError("!"), + william.HighlightPunctuation("["), + william.HighlightWhitespace(" "), + ] + |> william.to_ansi + + assert highlighted + == ansi.yellow("key") + <> ansi.magenta("=") + <> ansi.cyan("table") + <> ansi.green("\"str\"") + <> ansi.green("1") + <> ansi.blue("1979-05-27") + <> ansi.green("true") + <> ansi.italic(ansi.gray("# c")) + <> ansi.bg_bright_red(ansi.white("!")) + <> "[" + <> " " +} + +pub fn ignore_options_test() { + assert william.tokenise( + william.new() + |> william.ignore_whitespace + |> william.ignore_comments, + "# comment\nname = true", + ) + == [william.BareKey("name"), william.Equal, william.Boolean("true")] +} + +fn assert_tokens(str: String, tokens: List(william.Token)) { + assert william.tokenise(william.new(), str) == tokens as string.inspect(str) +} + +fn assert_value(value: String, token: william.Token) { + assert_tokens("a = " <> value, [ + william.BareKey("a"), + william.Whitespace(" "), + william.Equal, + william.Whitespace(" "), + token, + ]) +} + +fn assert_invalid_value(value: String) { + assert_value(value, william.InvalidNumber(value)) +}