diff --git a/pkg/harfbuzz/buffer.zig b/pkg/harfbuzz/buffer.zig index 035120f1a..b97c1bef4 100644 --- a/pkg/harfbuzz/buffer.zig +++ b/pkg/harfbuzz/buffer.zig @@ -238,6 +238,12 @@ pub const Buffer = struct { pub fn guessSegmentProperties(self: Buffer) void { c.hb_buffer_guess_segment_properties(self.handle); } + + /// Sets the cluster level of a buffer. The `ClusterLevel` dictates one + /// aspect of how HarfBuzz will treat non-base characters during shaping. + pub fn setClusterLevel(self: Buffer, level: ClusterLevel) void { + c.hb_buffer_set_cluster_level(self.handle, @intFromEnum(level)); + } }; /// The type of hb_buffer_t contents. @@ -252,6 +258,40 @@ pub const ContentType = enum(u2) { glyphs = c.HB_BUFFER_CONTENT_TYPE_GLYPHS, }; +/// Data type for holding HarfBuzz's clustering behavior options. The cluster +/// level dictates one aspect of how HarfBuzz will treat non-base characters +/// during shaping. +pub const ClusterLevel = enum(u2) { + /// In `monotone_graphemes`, non-base characters are merged into the + /// cluster of the base character that precedes them. There is also cluster + /// merging every time the clusters will otherwise become non-monotone. + /// This is the default cluster level. + monotone_graphemes = c.HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES, + + /// In `monotone_characters`, non-base characters are initially assigned + /// their own cluster values, which are not merged into preceding base + /// clusters. This allows HarfBuzz to perform additional operations like + /// reorder sequences of adjacent marks. The output is still monotone, but + /// the cluster values are more granular. + monotone_characters = c.HB_BUFFER_CLUSTER_LEVEL_MONOTONE_CHARACTERS, + + /// In `characters`, non-base characters are assigned their own cluster + /// values, which are not merged into preceding base clusters. Moreover, + /// the cluster values are not merged into monotone order. This is the most + /// granular cluster level, and it is useful for clients that need to know + /// the exact cluster values of each character, but is harder to use for + /// clients, since clusters might appear in any order. + characters = c.HB_BUFFER_CLUSTER_LEVEL_CHARACTERS, + + /// In `graphemes`, non-base characters are merged into the cluster of the + /// base character that precedes them. This is similar to the Unicode + /// Grapheme Cluster algorithm, but it is not exactly the same. The output + /// is not forced to be monotone. This is useful for clients that want to + /// use HarfBuzz as a cheap implementation of the Unicode Grapheme Cluster + /// algorithm. + graphemes = c.HB_BUFFER_CLUSTER_LEVEL_GRAPHEMES, +}; + /// The hb_glyph_info_t is the structure that holds information about the /// glyphs and their relation to input text. pub const GlyphInfo = extern struct { diff --git a/src/font/shaper/harfbuzz.zig b/src/font/shaper/harfbuzz.zig index 0acf06bbf..744f98254 100644 --- a/src/font/shaper/harfbuzz.zig +++ b/src/font/shaper/harfbuzz.zig @@ -266,6 +266,12 @@ pub const Shaper = struct { // Reset the buffer for our current run self.shaper.hb_buf.reset(); self.shaper.hb_buf.setContentType(.unicode); + + // We set the cluster level to `characters` to give us the most + // granularity, matching the CoreText shaper, and allowing us + // to use our same ligature detection heuristics. + self.shaper.hb_buf.setClusterLevel(.characters); + self.shaper.codepoints.clearRetainingCapacity(); // We don't support RTL text because RTL in terminals is messy. @@ -325,12 +331,13 @@ pub const Shaper = struct { break :blk false; } else false; - const formatted_cps = if (positions_differ or + const formatted_cps: ?[]u8 = if (positions_differ or y_offset_differs or cluster_differs or extra_debugging) blk: { var allocating = std.Io.Writer.Allocating.init(self.alloc); + defer allocating.deinit(); const writer = &allocating.writer; const codepoints = self.codepoints.items; var last_cluster: ?u32 = null; @@ -364,7 +371,8 @@ pub const Shaper = struct { } } break :blk try allocating.toOwnedSlice(); - } else ""; + } else null; + defer if (formatted_cps) |cps| self.alloc.free(cps); if (extra_debugging) { log.warn("extra debugging of positions index={d} cell_offset.cluster={d} cluster={d} run_offset.cluster={d} diff={d} pos=({d},{d}) run_offset=({d},{d}) cell_offset.x={d} is_prev_prepend={} cps = {s}", .{ @@ -379,7 +387,7 @@ pub const Shaper = struct { run_offset.y, cell_offset.x, is_previous_codepoint_prepend, - formatted_cps, + formatted_cps.?, }); } @@ -392,19 +400,19 @@ pub const Shaper = struct { advance_y_offset, x_offset_diff, y_offset_diff, - formatted_cps, + formatted_cps.?, }); } if (y_offset_differs) { - log.warn("y_offset differs from zero: cluster={d} pos=({d},{d}) run_offset=({d},{d}) cell_offset.x={d} cps = {s}", .{ + log.warn("run_offset.y differs from zero: cluster={d} pos=({d},{d}) run_offset=({d},{d}) cell_offset.x={d} cps = {s}", .{ cluster, x_offset, y_offset, run_offset.x, run_offset.y, cell_offset.x, - formatted_cps, + formatted_cps.?, }); } @@ -420,7 +428,7 @@ pub const Shaper = struct { run_offset.y, cell_offset.x, is_previous_codepoint_prepend, - formatted_cps, + formatted_cps.?, }); } } @@ -966,7 +974,7 @@ test "shape with empty cells in between" { } } -test "shape Chinese characters" { +test "shape Combining characters" { const testing = std.testing; const alloc = testing.allocator; @@ -1015,22 +1023,89 @@ test "shape Chinese characters" { try testing.expectEqual(@as(usize, 1), count); } -test "shape box glyphs" { +// This test exists because the string it uses causes HarfBuzz to output a +// non-monotonic run with our cluster level set to `characters`, which we need +// to handle by tracking the max cluster for the run. +test "shape Devanagari string" { const testing = std.testing; const alloc = testing.allocator; - var testdata = try testShaper(alloc); + // We need a font that supports devanagari for this to work, if we can't + // find Arial Unicode MS, which is a system font on macOS, we just skip + // the test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Arial Unicode MS", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Disable grapheme clustering + t.modes.set(.grapheme_cluster, false); + + var s = t.vtStream(); + defer s.deinit(); + try s.nextSlice("अपार्टमेंट"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + + const run = try it.next(alloc); + try testing.expect(run != null); + const cells = try shaper.shape(run.?); + + try testing.expectEqual(@as(usize, 8), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 1), cells[1].x); + try testing.expectEqual(@as(u16, 2), cells[2].x); + try testing.expectEqual(@as(u16, 3), cells[3].x); + try testing.expectEqual(@as(u16, 4), cells[4].x); + try testing.expectEqual(@as(u16, 5), cells[5].x); + try testing.expectEqual(@as(u16, 5), cells[6].x); + try testing.expectEqual(@as(u16, 6), cells[7].x); + + try testing.expect(try it.next(alloc) == null); +} + +test "shape Tai Tham vowels (position differs from advance)" { + // Note that while this test was necessary for CoreText, the old logic was + // working for HarfBuzz. Still we keep it to ensure it has the correct + // behavior. + const testing = std.testing; + const alloc = testing.allocator; + + // We need a font that supports Tai Tham for this to work, if we can't find + // Noto Sans Tai Tham, which is a system font on macOS, we just skip the + // test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Noto Sans Tai Tham", + ) catch return error.SkipZigTest; defer testdata.deinit(); var buf: [32]u8 = undefined; var buf_idx: usize = 0; - buf_idx += try std.unicode.utf8Encode(0x2500, buf[buf_idx..]); // horiz line - buf_idx += try std.unicode.utf8Encode(0x2501, buf[buf_idx..]); // + buf_idx += try std.unicode.utf8Encode(0x1a2F, buf[buf_idx..]); // ᨯ + buf_idx += try std.unicode.utf8Encode(0x1a70, buf[buf_idx..]); // ᩰ // Make a screen with some data - var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); defer t.deinit(alloc); + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + var s = t.vtStream(); defer s.deinit(); try s.nextSlice(buf[0..buf_idx]); @@ -1048,382 +1123,408 @@ test "shape box glyphs" { var count: usize = 0; while (try it.next(alloc)) |run| { count += 1; - try testing.expectEqual(@as(u32, 2), shaper.hb_buf.getLength()); + const cells = try shaper.shape(run); + const cell_width = run.grid.metrics.cell_width; try testing.expectEqual(@as(usize, 2), cells.len); - try testing.expectEqual(@as(u32, 0x2500), cells[0].glyph_index); try testing.expectEqual(@as(u16, 0), cells[0].x); - try testing.expectEqual(@as(u32, 0x2501), cells[1].glyph_index); - try testing.expectEqual(@as(u16, 1), cells[1].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + + // The first glyph renders in the next cell + try testing.expectEqual(@as(i16, @intCast(cell_width)), cells[0].x_offset); + try testing.expectEqual(@as(i16, 0), cells[1].x_offset); } try testing.expectEqual(@as(usize, 1), count); } -test "shape selection boundary" { +test "shape Tibetan characters" { const testing = std.testing; const alloc = testing.allocator; - var testdata = try testShaper(alloc); + // We need a font that has multiple glyphs for this codepoint to reproduce + // the old broken behavior, and Noto Serif Tibetan is one of them. It's not + // a default Mac font, and if we can't find it we just skip the test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Noto Serif Tibetan", + ) catch return error.SkipZigTest; defer testdata.deinit(); + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x0f00, buf[buf_idx..]); // ༀ + // Make a screen with some data - var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); defer t.deinit(alloc); + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + var s = t.vtStream(); defer s.deinit(); - try s.nextSlice("a1b2c3d4e5"); + try s.nextSlice(buf[0..buf_idx]); var state: terminal.RenderState = .empty; defer state.deinit(alloc); try state.update(alloc, &t); - // Full line selection - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - .selection = .{ 0, 9 }, - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 1), count); - } + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; - // Offset x, goes to end of line selection - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - .selection = .{ 2, 9 }, - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 2), count); - } + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 2), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); - // Offset x, starts at beginning of line - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - .selection = .{ 0, 3 }, - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 2), count); + // The second glyph renders at the correct location + try testing.expect(cells[1].x_offset < 2); } + try testing.expectEqual(@as(usize, 1), count); +} - // Selection only subset of line - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - .selection = .{ 1, 3 }, - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 3), count); - } +test "shape Tai Tham letters (run_offset.y differs from zero)" { + const testing = std.testing; + const alloc = testing.allocator; - // Selection only one character - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - .selection = .{ 1, 1 }, - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 3), count); + // We need a font that supports Tai Tham for this to work, if we can't find + // Noto Sans Tai Tham, which is a system font on macOS, we just skip the + // test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Noto Sans Tai Tham", + ) catch return error.SkipZigTest; + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + + // First grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x1a48, buf[buf_idx..]); // MA + buf_idx += try std.unicode.utf8Encode(0x1a60, buf[buf_idx..]); // SAKOT + buf_idx += try std.unicode.utf8Encode(0x1a3f, buf[buf_idx..]); // LOW LA + buf_idx += try std.unicode.utf8Encode(0x1a75, buf[buf_idx..]); // Tone-1 + // Second grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x1a41, buf[buf_idx..]); // HIGH PA + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + + var s = t.vtStream(); + defer s.deinit(); + try s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 3), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + try testing.expectEqual(@as(u16, 0), cells[2].x); // U from second grapheme + + // The U glyph renders at a y below zero + try testing.expectEqual(@as(i16, -3), cells[2].y_offset); } + try testing.expectEqual(@as(usize, 1), count); } -test "shape cursor boundary" { +test "shape Javanese ligatures" { const testing = std.testing; const alloc = testing.allocator; - var testdata = try testShaper(alloc); + // We need a font that supports Javanese for this to work, if we can't find + // Noto Sans Javanese Regular, which is a system font on macOS, we just + // skip the test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Noto Sans Javanese", + ) catch return error.SkipZigTest; defer testdata.deinit(); + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + + // First grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0xa9a4, buf[buf_idx..]); // NA + buf_idx += try std.unicode.utf8Encode(0xa9c0, buf[buf_idx..]); // PANGKON + // Second grapheme cluster, combining with the first in a ligature: + buf_idx += try std.unicode.utf8Encode(0xa9b2, buf[buf_idx..]); // HA + buf_idx += try std.unicode.utf8Encode(0xa9b8, buf[buf_idx..]); // Vowel sign SUKU + // Make a screen with some data - var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); defer t.deinit(alloc); + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + var s = t.vtStream(); defer s.deinit(); - try s.nextSlice("a1b2c3d4e5"); + try s.nextSlice(buf[0..buf_idx]); var state: terminal.RenderState = .empty; defer state.deinit(alloc); try state.update(alloc, &t); - // No cursor is full line - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 1), count); - } + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; - { - // Cursor at index 0 is two runs - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - .cursor_x = 0, - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 2), count); - } - // And without cursor splitting remains one - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 1), count); - } - } + const cells = try shaper.shape(run); + const cell_width = run.grid.metrics.cell_width; + try testing.expectEqual(@as(usize, 3), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + try testing.expectEqual(@as(u16, 0), cells[2].x); - { - // Cursor at index 1 is three runs - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - .cursor_x = 1, - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 3), count); - } - // And without cursor splitting remains one - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 1), count); - } - } - { - // Cursor at last col is two runs - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - .cursor_x = 9, - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 2), count); - } - // And without cursor splitting remains one - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 1), count); - } + // The vowel sign SUKU renders with correct x_offset + try testing.expect(cells[2].x_offset > 3 * cell_width); } + try testing.expectEqual(@as(usize, 1), count); } -test "shape cursor boundary and colored emoji" { +test "shape Chakma vowel sign with ligature (vowel sign renders first)" { const testing = std.testing; const alloc = testing.allocator; - var testdata = try testShaper(alloc); + // We need a font that supports Chakma for this to work, if we can't find + // Noto Sans Chakma Regular, which is a system font on macOS, we just skip + // the test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Noto Sans Chakma", + ) catch return error.SkipZigTest; defer testdata.deinit(); + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + + // First grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x1111d, buf[buf_idx..]); // BAA + // Second grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x11116, buf[buf_idx..]); // TAA + buf_idx += try std.unicode.utf8Encode(0x11133, buf[buf_idx..]); // Virama + // Third grapheme cluster, combining with the second in a ligature: + buf_idx += try std.unicode.utf8Encode(0x11120, buf[buf_idx..]); // YYAA + buf_idx += try std.unicode.utf8Encode(0x1112c, buf[buf_idx..]); // Vowel Sign U + // Make a screen with some data - var t = try terminal.Terminal.init( - alloc, - .{ .cols = 3, .rows = 10 }, - ); + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); defer t.deinit(alloc); + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + var s = t.vtStream(); defer s.deinit(); - try s.nextSlice("👍🏼"); + try s.nextSlice(buf[0..buf_idx]); var state: terminal.RenderState = .empty; defer state.deinit(alloc); try state.update(alloc, &t); - // No cursor is full line - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 1), count); - } + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; - // Cursor on emoji does not split it - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - .cursor_x = 0, - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 1), count); - } - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 1), count); - } - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - .cursor_x = 1, - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 1), count); - } - { - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - _ = try shaper.shape(run); - } - try testing.expectEqual(@as(usize, 1), count); + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 4), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + // See the giant "We need to reset the `cell_offset`" comment, but here + // we should technically have the rest of these be `x` of 1, but that + // would require going back in the stream to adjust past cells, and + // we don't take on that complexity. + try testing.expectEqual(@as(u16, 0), cells[1].x); + try testing.expectEqual(@as(u16, 0), cells[2].x); + try testing.expectEqual(@as(u16, 0), cells[3].x); + + // The vowel sign U renders before the TAA: + try testing.expect(cells[1].x_offset < cells[2].x_offset); } + try testing.expectEqual(@as(usize, 1), count); } -test "shape cell attribute change" { +test "shape Bengali ligatures with out of order vowels" { const testing = std.testing; const alloc = testing.allocator; - var testdata = try testShaper(alloc); + // We need a font that supports Bengali for this to work, if we can't find + // Arial Unicode MS, which is a system font on macOS, we just skip the + // test. + var testdata = testShaperWithDiscoveredFont( + alloc, + "Arial Unicode MS", + ) catch return error.SkipZigTest; defer testdata.deinit(); - // Plain >= should shape into 1 run - { - var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); - defer t.deinit(alloc); + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; - var s = t.vtStream(); - defer s.deinit(); - try s.nextSlice(">="); + // First grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x09b0, buf[buf_idx..]); // RA + buf_idx += try std.unicode.utf8Encode(0x09be, buf[buf_idx..]); // Vowel sign AA + // Second grapheme cluster: + buf_idx += try std.unicode.utf8Encode(0x09b7, buf[buf_idx..]); // SSA + buf_idx += try std.unicode.utf8Encode(0x09cd, buf[buf_idx..]); // Virama + // Third grapheme cluster, combining with the second in a ligature: + buf_idx += try std.unicode.utf8Encode(0x099f, buf[buf_idx..]); // TTA + buf_idx += try std.unicode.utf8Encode(0x09cd, buf[buf_idx..]); // Virama + // Fourth grapheme cluster, combining with the previous two in a ligature: + buf_idx += try std.unicode.utf8Encode(0x09b0, buf[buf_idx..]); // RA + buf_idx += try std.unicode.utf8Encode(0x09c7, buf[buf_idx..]); // Vowel sign E - var state: terminal.RenderState = .empty; - defer state.deinit(alloc); - try state.update(alloc, &t); + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + defer t.deinit(alloc); + + // Enable grapheme clustering + t.modes.set(.grapheme_cluster, true); + + var s = t.vtStream(); + defer s.deinit(); + try s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 8), cells.len); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u16, 0), cells[1].x); + // See the giant "We need to reset the `cell_offset`" comment, but here + // we should technically have the rest of these be `x` of 1, but that + // would require going back in the stream to adjust past cells, and + // we don't take on that complexity. + try testing.expectEqual(@as(u16, 0), cells[2].x); + try testing.expectEqual(@as(u16, 0), cells[3].x); + try testing.expectEqual(@as(u16, 0), cells[4].x); + try testing.expectEqual(@as(u16, 0), cells[5].x); + try testing.expectEqual(@as(u16, 0), cells[6].x); + try testing.expectEqual(@as(u16, 0), cells[7].x); + + // The vowel sign E renders before the SSA: + try testing.expect(cells[2].x_offset < cells[3].x_offset); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape box glyphs" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + var buf: [32]u8 = undefined; + var buf_idx: usize = 0; + buf_idx += try std.unicode.utf8Encode(0x2500, buf[buf_idx..]); // horiz line + buf_idx += try std.unicode.utf8Encode(0x2501, buf[buf_idx..]); // + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + try s.nextSlice(buf[0..buf_idx]); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + try testing.expectEqual(@as(u32, 2), shaper.hb_buf.getLength()); + const cells = try shaper.shape(run); + try testing.expectEqual(@as(usize, 2), cells.len); + try testing.expectEqual(@as(u32, 0x2500), cells[0].glyph_index); + try testing.expectEqual(@as(u16, 0), cells[0].x); + try testing.expectEqual(@as(u32, 0x2501), cells[1].glyph_index); + try testing.expectEqual(@as(u16, 1), cells[1].x); + } + try testing.expectEqual(@as(usize, 1), count); +} + +test "shape selection boundary" { + const testing = std.testing; + const alloc = testing.allocator; + + var testdata = try testShaper(alloc); + defer testdata.deinit(); + + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + try s.nextSlice("a1b2c3d4e5"); + + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + // Full line selection + { + // Get our run iterator var shaper = &testdata.shaper; var it = shaper.runIterator(.{ .grid = testdata.grid, .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 0, 9 }, }); var count: usize = 0; while (try it.next(alloc)) |run| { @@ -1433,25 +1534,14 @@ test "shape cell attribute change" { try testing.expectEqual(@as(usize, 1), count); } - // Bold vs regular should split + // Offset x, goes to end of line selection { - var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); - defer t.deinit(alloc); - - var s = t.vtStream(); - defer s.deinit(); - try s.nextSlice(">"); - try s.nextSlice("\x1b[1m"); - try s.nextSlice("="); - - var state: terminal.RenderState = .empty; - defer state.deinit(alloc); - try state.update(alloc, &t); - + // Get our run iterator var shaper = &testdata.shaper; var it = shaper.runIterator(.{ .grid = testdata.grid, .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 2, 9 }, }); var count: usize = 0; while (try it.next(alloc)) |run| { @@ -1461,28 +1551,14 @@ test "shape cell attribute change" { try testing.expectEqual(@as(usize, 2), count); } - // Changing fg color should split + // Offset x, starts at beginning of line { - var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); - defer t.deinit(alloc); - - var s = t.vtStream(); - defer s.deinit(); - // RGB 1, 2, 3 - try s.nextSlice("\x1b[38;2;1;2;3m"); - try s.nextSlice(">"); - // RGB 3, 2, 1 - try s.nextSlice("\x1b[38;2;3;2;1m"); - try s.nextSlice("="); - - var state: terminal.RenderState = .empty; - defer state.deinit(alloc); - try state.update(alloc, &t); - + // Get our run iterator var shaper = &testdata.shaper; var it = shaper.runIterator(.{ .grid = testdata.grid, .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 0, 3 }, }); var count: usize = 0; while (try it.next(alloc)) |run| { @@ -1492,533 +1568,581 @@ test "shape cell attribute change" { try testing.expectEqual(@as(usize, 2), count); } - // Changing bg color should not split + // Selection only subset of line { - var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); - defer t.deinit(alloc); - - var s = t.vtStream(); - defer s.deinit(); - // RGB 1, 2, 3 bg - try s.nextSlice("\x1b[48;2;1;2;3m"); - try s.nextSlice(">"); - // RGB 3, 2, 1 bg - try s.nextSlice("\x1b[48;2;3;2;1m"); - try s.nextSlice("="); - - var state: terminal.RenderState = .empty; - defer state.deinit(alloc); - try state.update(alloc, &t); - + // Get our run iterator var shaper = &testdata.shaper; var it = shaper.runIterator(.{ .grid = testdata.grid, .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 1, 3 }, }); var count: usize = 0; while (try it.next(alloc)) |run| { count += 1; _ = try shaper.shape(run); } - try testing.expectEqual(@as(usize, 1), count); + try testing.expectEqual(@as(usize, 3), count); } - // Same bg color should not split + // Selection only one character { - var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); - defer t.deinit(alloc); - - var s = t.vtStream(); - defer s.deinit(); - // RGB 1, 2, 3 bg - try s.nextSlice("\x1b[48;2;1;2;3m"); - try s.nextSlice(">"); - try s.nextSlice("="); - - var state: terminal.RenderState = .empty; - defer state.deinit(alloc); - try state.update(alloc, &t); - + // Get our run iterator var shaper = &testdata.shaper; var it = shaper.runIterator(.{ .grid = testdata.grid, .cells = state.row_data.get(0).cells.slice(), + .selection = .{ 1, 1 }, }); var count: usize = 0; while (try it.next(alloc)) |run| { count += 1; _ = try shaper.shape(run); } - try testing.expectEqual(@as(usize, 1), count); + try testing.expectEqual(@as(usize, 3), count); } } -const TestShaper = struct { - alloc: Allocator, - shaper: Shaper, - grid: *SharedGrid, - lib: Library, - - pub fn deinit(self: *TestShaper) void { - self.shaper.deinit(); - self.grid.deinit(self.alloc); - self.alloc.destroy(self.grid); - self.lib.deinit(); - } -}; +test "shape cursor boundary" { + const testing = std.testing; + const alloc = testing.allocator; -const TestFont = enum { - inconsolata, - monaspace_neon, - arabic, -}; + var testdata = try testShaper(alloc); + defer testdata.deinit(); -/// Helper to return a fully initialized shaper. -fn testShaper(alloc: Allocator) !TestShaper { - return try testShaperWithFont(alloc, .inconsolata); -} + // Make a screen with some data + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); -fn testShaperWithFont(alloc: Allocator, font_req: TestFont) !TestShaper { - const testEmoji = font.embedded.emoji; - const testEmojiText = font.embedded.emoji_text; - const testFont = switch (font_req) { - .inconsolata => font.embedded.inconsolata, - .monaspace_neon => font.embedded.monaspace_neon, - .arabic => font.embedded.arabic, - }; + var s = t.vtStream(); + defer s.deinit(); + try s.nextSlice("a1b2c3d4e5"); - var lib = try Library.init(alloc); - errdefer lib.deinit(); - - var c = Collection.init(); - c.load_options = .{ .library = lib }; - - // Setup group - _ = try c.add(alloc, try .init( - lib, - testFont, - .{ .size = .{ .points = 12 } }, - ), .{ - .style = .regular, - .fallback = false, - .size_adjustment = .none, - }); + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); - if (comptime !font.options.backend.hasCoretext()) { - // Coretext doesn't support Noto's format - _ = try c.add(alloc, try .init( - lib, - testEmoji, - .{ .size = .{ .points = 12 } }, - ), .{ - .style = .regular, - .fallback = false, - .size_adjustment = .none, - }); - } else { - // On CoreText we want to load Apple Emoji, we should have it. - var disco = font.Discover.init(); - defer disco.deinit(); - var disco_it = try disco.discover(alloc, .{ - .family = "Apple Color Emoji", - .size = 12, - .monospace = false, - }); - defer disco_it.deinit(); - var face = (try disco_it.next()).?; - errdefer face.deinit(); - _ = try c.addDeferred(alloc, face, .{ - .style = .regular, - .fallback = false, - .size_adjustment = .none, + // No cursor is full line + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); } - _ = try c.add(alloc, try .init( - lib, - testEmojiText, - .{ .size = .{ .points = 12 } }, - ), .{ - .style = .regular, - .fallback = false, - .size_adjustment = .none, - }); - - const grid_ptr = try alloc.create(SharedGrid); - errdefer alloc.destroy(grid_ptr); - grid_ptr.* = try .init(alloc, .{ .collection = c }); - errdefer grid_ptr.*.deinit(alloc); - - var shaper = try Shaper.init(alloc, .{ - // Some of our tests rely on dlig being enabled by default - .features = &.{"dlig"}, - }); - errdefer shaper.deinit(); - - return TestShaper{ - .alloc = alloc, - .shaper = shaper, - .grid = grid_ptr, - .lib = lib, - }; -} - -fn testShaperWithDiscoveredFont(alloc: Allocator, font_req: [:0]const u8) !TestShaper { - var lib = try Library.init(alloc); - errdefer lib.deinit(); - - var c = Collection.init(); - c.load_options = .{ .library = lib }; - // Discover and add our font to the collection. { - var disco = font.Discover.init(); - defer disco.deinit(); - var disco_it = try disco.discover(alloc, .{ - .family = font_req, - .size = 12, - .monospace = false, - }); - defer disco_it.deinit(); - var face: font.DeferredFace = (try disco_it.next()) orelse return error.FontNotFound; - errdefer face.deinit(); - _ = try c.add( - alloc, - try face.load(lib, .{ .size = .{ .points = 12 } }), - .{ - .style = .regular, - .fallback = false, - .size_adjustment = .none, - }, - ); + // Cursor at index 0 is two runs + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 0, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + // And without cursor splitting remains one + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } } - const grid_ptr = try alloc.create(SharedGrid); - errdefer alloc.destroy(grid_ptr); - grid_ptr.* = try .init(alloc, .{ .collection = c }); - errdefer grid_ptr.*.deinit(alloc); - - var shaper = try Shaper.init(alloc, .{}); - errdefer shaper.deinit(); - - return TestShaper{ - .alloc = alloc, - .shaper = shaper, - .grid = grid_ptr, - .lib = lib, - }; + { + // Cursor at index 1 is three runs + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 1, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 3), count); + } + // And without cursor splitting remains one + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + } + { + // Cursor at last col is two runs + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 9, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } + // And without cursor splitting remains one + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + } } -test "shape Tai Tham vowels (y_offset differs from zero)" { +test "shape cursor boundary and colored emoji" { const testing = std.testing; const alloc = testing.allocator; - // We need a font that supports Tai Tham for this to work, if we can't find - // Noto Sans Tai Tham, we just skip the test. - var testdata = testShaperWithDiscoveredFont( - alloc, - "Noto Sans Tai Tham", - ) catch return error.SkipZigTest; + var testdata = try testShaper(alloc); defer testdata.deinit(); - var buf: [32]u8 = undefined; - var buf_idx: usize = 0; - buf_idx += try std.unicode.utf8Encode(0x1a2F, buf[buf_idx..]); // ᨯ - buf_idx += try std.unicode.utf8Encode(0x1a70, buf[buf_idx..]); // ᩰ - // Make a screen with some data - var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); + var t = try terminal.Terminal.init( + alloc, + .{ .cols = 3, .rows = 10 }, + ); defer t.deinit(alloc); - // Enable grapheme clustering - t.modes.set(.grapheme_cluster, true); - var s = t.vtStream(); defer s.deinit(); - try s.nextSlice(buf[0..buf_idx]); + try s.nextSlice("👍🏼"); var state: terminal.RenderState = .empty; defer state.deinit(alloc); try state.update(alloc, &t); - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - - const cells = try shaper.shape(run); - const cell_width = run.grid.metrics.cell_width; - try testing.expectEqual(@as(usize, 2), cells.len); - try testing.expectEqual(@as(u16, 0), cells[0].x); - try testing.expectEqual(@as(u16, 0), cells[1].x); + // No cursor is full line + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } - // The first glyph renders in the next cell - try testing.expectEqual(@as(i16, @intCast(cell_width)), cells[0].x_offset); - try testing.expectEqual(@as(i16, 0), cells[1].x_offset); + // Cursor on emoji does not split it + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 0, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + .cursor_x = 1, + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } + { + // Get our run iterator + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); } - try testing.expectEqual(@as(usize, 1), count); } -test "shape Tai Tham letters (y_offset differs from zero)" { +test "shape cell attribute change" { const testing = std.testing; const alloc = testing.allocator; - // We need a font that supports Tai Tham for this to work, if we can't find - // Noto Sans Tai Tham, we just skip the test. - var testdata = testShaperWithDiscoveredFont( - alloc, - "Noto Sans Tai Tham", - ) catch return error.SkipZigTest; + var testdata = try testShaper(alloc); defer testdata.deinit(); - var buf: [32]u8 = undefined; - var buf_idx: usize = 0; - - // First grapheme cluster: - buf_idx += try std.unicode.utf8Encode(0x1a48, buf[buf_idx..]); // MA - buf_idx += try std.unicode.utf8Encode(0x1a60, buf[buf_idx..]); // SAKOT - buf_idx += try std.unicode.utf8Encode(0x1a3f, buf[buf_idx..]); // LOW LA - buf_idx += try std.unicode.utf8Encode(0x1a75, buf[buf_idx..]); // Tone-1 - // Second grapheme cluster: - buf_idx += try std.unicode.utf8Encode(0x1a41, buf[buf_idx..]); // HIGH PA + // Plain >= should shape into 1 run + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 10, .rows = 3 }); + defer t.deinit(alloc); - // Make a screen with some data - var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); - defer t.deinit(alloc); + var s = t.vtStream(); + defer s.deinit(); + try s.nextSlice(">="); - // Enable grapheme clustering - t.modes.set(.grapheme_cluster, true); + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); - var s = t.vtStream(); - defer s.deinit(); - try s.nextSlice(buf[0..buf_idx]); + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } - var state: terminal.RenderState = .empty; - defer state.deinit(alloc); - try state.update(alloc, &t); + // Bold vs regular should split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; + var s = t.vtStream(); + defer s.deinit(); + try s.nextSlice(">"); + try s.nextSlice("\x1b[1m"); + try s.nextSlice("="); - const cells = try shaper.shape(run); - try testing.expectEqual(@as(usize, 3), cells.len); - try testing.expectEqual(@as(u16, 0), cells[0].x); - try testing.expectEqual(@as(u16, 0), cells[1].x); - try testing.expectEqual(@as(u16, 0), cells[2].x); // U from second grapheme + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); - // The U glyph renders at a y below zero - try testing.expectEqual(@as(i16, -3), cells[2].y_offset); + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); } - try testing.expectEqual(@as(usize, 1), count); -} -test "shape Javanese ligatures" { - const testing = std.testing; - const alloc = testing.allocator; + // Changing fg color should split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); - // We need a font that supports Javanese for this to work, if we can't find - // Noto Sans Javanese Regular, we just skip the test. - var testdata = testShaperWithDiscoveredFont( - alloc, - "Noto Sans Javanese", - ) catch return error.SkipZigTest; - defer testdata.deinit(); + var s = t.vtStream(); + defer s.deinit(); + // RGB 1, 2, 3 + try s.nextSlice("\x1b[38;2;1;2;3m"); + try s.nextSlice(">"); + // RGB 3, 2, 1 + try s.nextSlice("\x1b[38;2;3;2;1m"); + try s.nextSlice("="); - var buf: [32]u8 = undefined; - var buf_idx: usize = 0; + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); - // First grapheme cluster: - buf_idx += try std.unicode.utf8Encode(0xa9a4, buf[buf_idx..]); // NA - buf_idx += try std.unicode.utf8Encode(0xa9c0, buf[buf_idx..]); // PANGKON - // Second grapheme cluster, combining with the first in a ligature: - buf_idx += try std.unicode.utf8Encode(0xa9b2, buf[buf_idx..]); // HA - buf_idx += try std.unicode.utf8Encode(0xa9b8, buf[buf_idx..]); // Vowel sign SUKU + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 2), count); + } - // Make a screen with some data - var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); - defer t.deinit(alloc); + // Changing bg color should not split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); - // Enable grapheme clustering - t.modes.set(.grapheme_cluster, true); + var s = t.vtStream(); + defer s.deinit(); + // RGB 1, 2, 3 bg + try s.nextSlice("\x1b[48;2;1;2;3m"); + try s.nextSlice(">"); + // RGB 3, 2, 1 bg + try s.nextSlice("\x1b[48;2;3;2;1m"); + try s.nextSlice("="); - var s = t.vtStream(); - defer s.deinit(); - try s.nextSlice(buf[0..buf_idx]); + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); - var state: terminal.RenderState = .empty; - defer state.deinit(alloc); - try state.update(alloc, &t); + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); + } - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; + // Same bg color should not split + { + var t = try terminal.Terminal.init(alloc, .{ .cols = 3, .rows = 10 }); + defer t.deinit(alloc); - const cells = try shaper.shape(run); - const cell_width = run.grid.metrics.cell_width; - try testing.expectEqual(@as(usize, 3), cells.len); - try testing.expectEqual(@as(u16, 0), cells[0].x); - try testing.expectEqual(@as(u16, 0), cells[1].x); - try testing.expectEqual(@as(u16, 0), cells[2].x); + var s = t.vtStream(); + defer s.deinit(); + // RGB 1, 2, 3 bg + try s.nextSlice("\x1b[48;2;1;2;3m"); + try s.nextSlice(">"); + try s.nextSlice("="); - // The vowel sign SUKU renders with correct x_offset - try testing.expect(cells[2].x_offset > 3 * cell_width); + var state: terminal.RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + var shaper = &testdata.shaper; + var it = shaper.runIterator(.{ + .grid = testdata.grid, + .cells = state.row_data.get(0).cells.slice(), + }); + var count: usize = 0; + while (try it.next(alloc)) |run| { + count += 1; + _ = try shaper.shape(run); + } + try testing.expectEqual(@as(usize, 1), count); } - try testing.expectEqual(@as(usize, 1), count); } -test "shape Chakma vowel sign with ligature (vowel sign renders first)" { - const testing = std.testing; - const alloc = testing.allocator; - - // We need a font that supports Chakma for this to work, if we can't find - // Noto Sans Chakma Regular, we just skip the test. - var testdata = testShaperWithDiscoveredFont( - alloc, - "Noto Sans Chakma", - ) catch return error.SkipZigTest; - defer testdata.deinit(); +const TestShaper = struct { + alloc: Allocator, + shaper: Shaper, + grid: *SharedGrid, + lib: Library, - var buf: [32]u8 = undefined; - var buf_idx: usize = 0; + pub fn deinit(self: *TestShaper) void { + self.shaper.deinit(); + self.grid.deinit(self.alloc); + self.alloc.destroy(self.grid); + self.lib.deinit(); + } +}; - // First grapheme cluster: - buf_idx += try std.unicode.utf8Encode(0x1111d, buf[buf_idx..]); // BAA - // Second grapheme cluster: - buf_idx += try std.unicode.utf8Encode(0x11116, buf[buf_idx..]); // TAA - buf_idx += try std.unicode.utf8Encode(0x11133, buf[buf_idx..]); // Virama - // Third grapheme cluster, combining with the second in a ligature: - buf_idx += try std.unicode.utf8Encode(0x11120, buf[buf_idx..]); // YYAA - buf_idx += try std.unicode.utf8Encode(0x1112c, buf[buf_idx..]); // Vowel Sign U +const TestFont = enum { + inconsolata, + monaspace_neon, + arabic, +}; - // Make a screen with some data - var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); - defer t.deinit(alloc); +/// Helper to return a fully initialized shaper. +fn testShaper(alloc: Allocator) !TestShaper { + return try testShaperWithFont(alloc, .inconsolata); +} - // Enable grapheme clustering - t.modes.set(.grapheme_cluster, true); +fn testShaperWithFont(alloc: Allocator, font_req: TestFont) !TestShaper { + const testEmoji = font.embedded.emoji; + const testEmojiText = font.embedded.emoji_text; + const testFont = switch (font_req) { + .inconsolata => font.embedded.inconsolata, + .monaspace_neon => font.embedded.monaspace_neon, + .arabic => font.embedded.arabic, + }; - var s = t.vtStream(); - defer s.deinit(); - try s.nextSlice(buf[0..buf_idx]); + var lib = try Library.init(alloc); + errdefer lib.deinit(); - var state: terminal.RenderState = .empty; - defer state.deinit(alloc); - try state.update(alloc, &t); + var c = Collection.init(); + c.load_options = .{ .library = lib }; - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), + // Setup group + _ = try c.add(alloc, try .init( + lib, + testFont, + .{ .size = .{ .points = 12 } }, + ), .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; - - const cells = try shaper.shape(run); - try testing.expectEqual(@as(usize, 4), cells.len); - try testing.expectEqual(@as(u16, 0), cells[0].x); - // See the giant "We need to reset the `cell_offset`" comment, but here - // we should technically have the rest of these be `x` of 1, but that - // would require going back in the stream to adjust past cells, and - // we don't take on that complexity. - try testing.expectEqual(@as(u16, 0), cells[1].x); - try testing.expectEqual(@as(u16, 0), cells[2].x); - try testing.expectEqual(@as(u16, 0), cells[3].x); - // The vowel sign U renders before the TAA: - try testing.expect(cells[1].x_offset < cells[2].x_offset); + if (comptime !font.options.backend.hasCoretext()) { + // Coretext doesn't support Noto's format + _ = try c.add(alloc, try .init( + lib, + testEmoji, + .{ .size = .{ .points = 12 } }, + ), .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }); + } else { + // On CoreText we want to load Apple Emoji, we should have it. + var disco = font.Discover.init(); + defer disco.deinit(); + var disco_it = try disco.discover(alloc, .{ + .family = "Apple Color Emoji", + .size = 12, + .monospace = false, + }); + defer disco_it.deinit(); + var face = (try disco_it.next()).?; + errdefer face.deinit(); + _ = try c.addDeferred(alloc, face, .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }); } - try testing.expectEqual(@as(usize, 1), count); -} - -test "shape Bengali ligatures with out of order vowels" { - const testing = std.testing; - const alloc = testing.allocator; - - // We need a font that supports Bengali for this to work, if we can't find - // Arial Unicode MS, we just skip the test. - var testdata = testShaperWithDiscoveredFont( - alloc, - "Arial Unicode MS", - ) catch return error.SkipZigTest; - defer testdata.deinit(); + _ = try c.add(alloc, try .init( + lib, + testEmojiText, + .{ .size = .{ .points = 12 } }, + ), .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }); - var buf: [32]u8 = undefined; - var buf_idx: usize = 0; + const grid_ptr = try alloc.create(SharedGrid); + errdefer alloc.destroy(grid_ptr); + grid_ptr.* = try .init(alloc, .{ .collection = c }); + errdefer grid_ptr.*.deinit(alloc); - // First grapheme cluster: - buf_idx += try std.unicode.utf8Encode(0x09b0, buf[buf_idx..]); // RA - buf_idx += try std.unicode.utf8Encode(0x09be, buf[buf_idx..]); // Vowel sign AA - // Second grapheme cluster: - buf_idx += try std.unicode.utf8Encode(0x09b7, buf[buf_idx..]); // SSA - buf_idx += try std.unicode.utf8Encode(0x09cd, buf[buf_idx..]); // Virama - // Third grapheme cluster, combining with the second in a ligature: - buf_idx += try std.unicode.utf8Encode(0x099f, buf[buf_idx..]); // TTA - buf_idx += try std.unicode.utf8Encode(0x09cd, buf[buf_idx..]); // Virama - // Fourth grapheme cluster, combining with the previous two in a ligature: - buf_idx += try std.unicode.utf8Encode(0x09b0, buf[buf_idx..]); // RA - buf_idx += try std.unicode.utf8Encode(0x09c7, buf[buf_idx..]); // Vowel sign E + var shaper = try Shaper.init(alloc, .{ + // Some of our tests rely on dlig being enabled by default + .features = &.{"dlig"}, + }); + errdefer shaper.deinit(); - // Make a screen with some data - var t = try terminal.Terminal.init(alloc, .{ .cols = 30, .rows = 3 }); - defer t.deinit(alloc); + return TestShaper{ + .alloc = alloc, + .shaper = shaper, + .grid = grid_ptr, + .lib = lib, + }; +} - // Enable grapheme clustering - t.modes.set(.grapheme_cluster, true); +fn testShaperWithDiscoveredFont(alloc: Allocator, font_req: [:0]const u8) !TestShaper { + var lib = try Library.init(alloc); + errdefer lib.deinit(); - var s = t.vtStream(); - defer s.deinit(); - try s.nextSlice(buf[0..buf_idx]); + var c = Collection.init(); + c.load_options = .{ .library = lib }; - var state: terminal.RenderState = .empty; - defer state.deinit(alloc); - try state.update(alloc, &t); + // Discover and add our font to the collection. + { + var disco = font.Discover.init(); + defer disco.deinit(); + var disco_it = try disco.discover(alloc, .{ + .family = font_req, + .size = 12, + .monospace = false, + }); + defer disco_it.deinit(); + var face: font.DeferredFace = (try disco_it.next()) orelse return error.FontNotFound; + errdefer face.deinit(); + _ = try c.add( + alloc, + try face.load(lib, .{ .size = .{ .points = 12 } }), + .{ + .style = .regular, + .fallback = false, + .size_adjustment = .none, + }, + ); + } - // Get our run iterator - var shaper = &testdata.shaper; - var it = shaper.runIterator(.{ - .grid = testdata.grid, - .cells = state.row_data.get(0).cells.slice(), - }); - var count: usize = 0; - while (try it.next(alloc)) |run| { - count += 1; + const grid_ptr = try alloc.create(SharedGrid); + errdefer alloc.destroy(grid_ptr); + grid_ptr.* = try .init(alloc, .{ .collection = c }); + errdefer grid_ptr.*.deinit(alloc); - const cells = try shaper.shape(run); - try testing.expectEqual(@as(usize, 8), cells.len); - try testing.expectEqual(@as(u16, 0), cells[0].x); - try testing.expectEqual(@as(u16, 0), cells[1].x); - // See the giant "We need to reset the `cell_offset`" comment, but here - // we should technically have the rest of these be `x` of 1, but that - // would require going back in the stream to adjust past cells, and - // we don't take on that complexity. - try testing.expectEqual(@as(u16, 0), cells[2].x); - try testing.expectEqual(@as(u16, 0), cells[3].x); - try testing.expectEqual(@as(u16, 0), cells[4].x); - try testing.expectEqual(@as(u16, 0), cells[5].x); - try testing.expectEqual(@as(u16, 0), cells[6].x); - try testing.expectEqual(@as(u16, 0), cells[7].x); + var shaper = try Shaper.init(alloc, .{}); + errdefer shaper.deinit(); - // The vowel sign E renders before the SSA: - try testing.expect(cells[2].x_offset < cells[3].x_offset); - } - try testing.expectEqual(@as(usize, 1), count); + return TestShaper{ + .alloc = alloc, + .shaper = shaper, + .grid = grid_ptr, + .lib = lib, + }; }