diff --git a/README.md b/README.md index d1e5dc1..0e667b9 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ Formatting flags: | `--printWidth` | Target line width (default 80) | | `--indent` | Indentation: a literal like `" "` or `"\t"` | | `--align` | `field`, `assign`, or `disable` | -| `---separator` | Separators per construct (`struct`, `union`, `exception`, `enum`, `argument`, `throws`): `comma`, `semicolon`, `none`, or `preserve` (keep as written) | +| `---separator` | Separators per construct (`struct`, `union`, `exception`, `enum`, `argument`, `throws`, `list`, `map`): `comma`, `semicolon`, `none`, or `preserve` (keep as written) | | `--break-` | Always break the construct's bodies onto multiple lines (same constructs) | | `--config` | Path to a `thriftls.json` config file | | `-I` | Additional include path, like the thrift compiler's `-I` (repeatable) | @@ -230,13 +230,17 @@ the file being formatted or the workspace root (like Biome). Set the "exceptions": "semicolon", "enums": "comma", "arguments": "comma", - "throws": "comma" + "throws": "comma", + "lists": "comma", + "maps": "comma" }, "break": { "structs": true, "unions": true, "exceptions": true, - "enums": true + "enums": true, + "lists": true, + "maps": true }, "includePaths": ["/path/to/base"], "logLevel": 3 @@ -274,14 +278,17 @@ width, comments, and blank lines. Controls trailing separators per construct, independently. The `separators` object has one key per construct: `structs`, `unions`, -`exceptions`, `enums`, `arguments` (function arguments), and `throws` -(throws entries). Each accepts: +`exceptions`, `enums`, `arguments` (function arguments), `throws` +(throws entries), `lists` and `maps` (const list and map values). Each +accepts: - `comma`: Always add trailing commas - `semicolon`: Always add trailing semicolons - `none`: Remove trailing separators - `preserve`: Keep as written (default) +For `lists` and `maps`, the separator appears between the items and after +the last item; `none` removes them entirely (`[1, 2]` becomes `[1 2]`). For example, semicolons in structs and commas in enums: ```json @@ -309,7 +316,7 @@ whose separators are inconsistently present looks broken. Forces layouts that would otherwise collapse to one line to stay multiline, regardless of the source's trailing delimiters. Like `separators`, the `break` object has one key per construct: `structs`, -`unions`, `exceptions`, `enums`, `arguments`, `throws`. +`unions`, `exceptions`, `enums`, `arguments`, `throws`, `lists`, `maps`. All default to `false`. diff --git a/doc/print.go b/doc/print.go index df4ce3f..a8d3d43 100644 --- a/doc/print.go +++ b/doc/print.go @@ -115,6 +115,23 @@ func (p *printer) write(s string) { p.out = append(p.out, s...) } +// lineEnded reports whether the output already ends with a newline +// (ignoring trailing spaces and tabs, i.e. indentation). +func (p *printer) lineEnded() bool { + for i := len(p.out) - 1; i >= 0; i-- { + switch p.out[i] { + case ' ', '\t': + continue + case '\n': + return true + default: + return false + } + } + + return false +} + // trim removes trailing spaces and tabs from the output and returns how many // columns were removed. func (p *printer) trim() int { @@ -242,6 +259,23 @@ func (p *printer) run(d Doc) (string, error) { p.write(newLine) p.position = 0 } else { + // A structural soft line right after a line that + // already ended (a line comment owns its line end) + // must not emit another newline — that would be a + // blank line — but it must re-apply the structural + // indentation: the comment's hard line carried the + // inner indent, and the following content belongs at + // the structural indent (e.g. a closing bracket after + // a comment inside a list). Hard lines always render + // (consecutive hard lines are blank lines). + if !v.Hard && p.lineEnded() { + p.trim() + p.write(cmd.indentation.value) + p.position = cmd.indentation.length + + break + } + p.trim() p.write(newLine + cmd.indentation.value) p.position = cmd.indentation.length diff --git a/formatter/body.go b/formatter/body.go index ffeee27..613f93e 100644 --- a/formatter/body.go +++ b/formatter/body.go @@ -13,13 +13,9 @@ func (f *formatter) structLike(v *syntax.Struct) doc.Doc { close := f.scanKind(open+1, v.TokEnd(), syntax.TokenRBrace) parts := []doc.Doc{ - f.emitTokens(v.TokStart(), open, emitOpts{skipText: []int{open}, breakSkip: true}), + f.emitTokens(v.TokStart(), open, emitOpts{skipText: []int{open}}), f.bracedBody(v.Fields, open, close, close != v.TokEnd(), f.constructOf(v.Kind)), } - if v.Annotations != nil { - parts = append(parts, f.breakBeforeAnnotations(close)) - } - parts = append(parts, f.annotationsDoc(v.Annotations, v.Annotations != nil && v.Annotations.TokEnd() == v.TokEnd())) parts = append(parts, f.afterAnnotations(v.Annotations, v.TokEnd())) @@ -32,13 +28,9 @@ func (f *formatter) enum(v *syntax.Enum) doc.Doc { close := f.scanKind(open+1, v.TokEnd(), syntax.TokenRBrace) parts := []doc.Doc{ - f.emitTokens(v.TokStart(), open, emitOpts{skipText: []int{open}, breakSkip: true}), + f.emitTokens(v.TokStart(), open, emitOpts{skipText: []int{open}}), f.bracedEnumBody(v.Values, open, close, close != v.TokEnd()), } - if v.Annotations != nil { - parts = append(parts, f.breakBeforeAnnotations(close)) - } - parts = append(parts, f.annotationsDoc(v.Annotations, v.Annotations != nil && v.Annotations.TokEnd() == v.TokEnd())) parts = append(parts, f.afterAnnotations(v.Annotations, v.TokEnd())) @@ -78,15 +70,19 @@ func (f *formatter) constructOf(kind syntax.StructKind) Construct { func (f *formatter) bracedBody(fields []*syntax.Field, open, close int, closeTrailing bool, c Construct) doc.Doc { bodyID := f.id() sepMode := f.opts.Separator.Get(c) - inner := append([]doc.Doc{doc.Line, f.fieldList(fields, bodyID, sepMode)}, f.closingTriviaAt(close)...) + inner := append([]doc.Doc{doc.Line, f.fieldList(fields, bodyID, sepMode)}, f.ownLineComments(close)...) closeBreak := doc.IfBreak(doc.SoftLine, doc.Text(" ")) if len(fields) == 0 { - inner = append([]doc.Doc{}, f.closingTriviaAt(close)...) + inner = append([]doc.Doc{}, f.ownLineComments(close)...) closeBreak = doc.IfBreak(doc.SoftLine, doc.Concat{}) } - openDoc := append([]doc.Doc{doc.Text(" {")}, f.openTriviaAt(open)...) + openComments := f.sameLineComments(open) + openDoc := append([]doc.Doc{doc.Text(" {")}, openComments...) + if len(openComments) > 0 { + openDoc = append(openDoc, doc.BreakParent) + } content := doc.Concat{ doc.Concat(openDoc), @@ -105,15 +101,19 @@ func (f *formatter) bracedBody(fields []*syntax.Field, open, close int, closeTra // bracedEnumBody is bracedBody for enum values. func (f *formatter) bracedEnumBody(values []*syntax.EnumValue, open, close int, closeTrailing bool) doc.Doc { bodyID := f.id() - inner := append([]doc.Doc{doc.Line, f.enumValueList(values, bodyID)}, f.closingTriviaAt(close)...) + inner := append([]doc.Doc{doc.Line, f.enumValueList(values, bodyID)}, f.ownLineComments(close)...) closeBreak := doc.IfBreak(doc.SoftLine, doc.Text(" ")) if len(values) == 0 { - inner = append([]doc.Doc{}, f.closingTriviaAt(close)...) + inner = append([]doc.Doc{}, f.ownLineComments(close)...) closeBreak = doc.IfBreak(doc.SoftLine, doc.Concat{}) } - openDoc := append([]doc.Doc{doc.Text(" {")}, f.openTriviaAt(open)...) + openComments := f.sameLineComments(open) + openDoc := append([]doc.Doc{doc.Text(" {")}, openComments...) + if len(openComments) > 0 { + openDoc = append(openDoc, doc.BreakParent) + } content := doc.Concat{ doc.Concat(openDoc), @@ -128,51 +128,6 @@ func (f *formatter) bracedEnumBody(values []*syntax.EnumValue, open, close int, return doc.GroupID(bodyID, content) } -// closingTriviaAt returns the comments attached before the closing token, -// as docs each ending with a hard line. The leading hard line forces the -// body to break so the comments stay inside; the caller's closing line -// provides the newline after the last one. -func (f *formatter) closingTriviaAt(close int) []doc.Doc { - tok := f.token(close) - - parts := make([]doc.Doc, 0, 8) - if len(tok.Leading) > 0 { - parts = append(parts, doc.HardLine) - - prevBlank := 0 - for i, c := range tok.Leading { - parts = append(parts, f.blankLineDocs(c.BlankLinesBefore-prevBlank, doc.HardLine)...) - prevBlank = c.BlankLinesBefore - - parts = append(parts, doc.Text(trimComment(c.Text))) - if i < len(tok.Leading)-1 { - parts = append(parts, doc.HardLine) - } - } - } - - return parts -} - -// openTriviaAt returns the trailing comments of the opening token as -// line-suffix docs, plus a break parent so the body goes multiline. Empty -// when there are none. -func (f *formatter) openTriviaAt(open int) []doc.Doc { - tok := f.token(open) - - parts := make([]doc.Doc, 0, 8) - - if len(tok.Trailing) > 0 { - for _, c := range tok.Trailing { - parts = append(parts, doc.LineSuffix(doc.Text(" "+trimComment(c.Text)))) - } - - parts = append(parts, doc.BreakParent) - } - - return parts -} - // service formats a service declaration. The body is always multiline: // functions are too complex to flatten. func (f *formatter) service(v *syntax.Service) doc.Doc { @@ -183,49 +138,91 @@ func (f *formatter) service(v *syntax.Service) doc.Doc { for i, fn := range v.Functions { if i > 0 { - parts = append(parts, doc.HardLineNoBreak) - parts = append(parts, f.blankLines(fn, doc.HardLineNoBreak)...) + // The separator line collapses when the previous function + // ended with a line comment (which owns its line end). + parts = append(parts, doc.Line) + parts = append(parts, f.blankLines(fn, doc.HardLine)...) } else { - parts = append(parts, f.blankLines(fn, doc.HardLineNoBreak)...) + parts = append(parts, f.blankLines(fn, doc.HardLine)...) } parts = append(parts, f.function(fn)) } - inner := doc.Concat{doc.Concat(parts), doc.Concat(f.closingTriviaAt(close))} + inner := doc.Concat{doc.Concat(parts), doc.Concat(f.ownLineComments(close))} if len(v.Functions) > 0 { // The first function starts its own line; the closing trivia // provides the break before it for empty bodies, so the blank // count does not double. - inner = doc.Concat{doc.HardLineNoBreak, doc.Concat(parts), doc.Concat(f.closingTriviaAt(close))} - } else if len(f.token(close).Leading) == 0 && f.token(close).BlankLinesBefore > 0 { + inner = doc.Concat{doc.Line, doc.Concat(parts), doc.Concat(f.ownLineComments(close))} + } else if !f.hasOwnLineComments(close) && f.token(close).BlankLinesBefore > 0 { // Empty body with blank lines before the close and no comments: - // closingTriviaAt emits nothing, so preserve the blanks here. - inner = doc.Concat{doc.Concat(f.blankLineDocs(f.token(close).BlankLinesBefore, doc.HardLineNoBreak)), inner} + // ownLineComments emits nothing, so preserve the blanks here. + inner = doc.Concat{doc.Concat(f.blankLineDocs(f.token(close).BlankLinesBefore, doc.HardLine)), inner} + } + + openComments := f.sameLineComments(open) + openDoc := append([]doc.Doc{doc.Text(" {")}, openComments...) + if len(openComments) > 0 { + openDoc = append(openDoc, doc.BreakParent) + } + + // The line before the close collapses when the body already ended with + // a line comment (which owns its line end); it stays hard otherwise, so + // a blank line before the close still renders. + lastFn := -1 + if len(v.Functions) > 0 { + lastFn = v.Functions[len(v.Functions)-1].TokEnd() + } + + preClose := doc.HardLine + if f.endsWithLineComment(open, close, lastFn) { + preClose = doc.Line } - openDoc := append([]doc.Doc{doc.Text(" {")}, f.openTriviaAt(open)...) body := doc.Concat{ doc.Concat(openDoc), doc.Indent(inner), - doc.HardLineNoBreak, + preClose, f.emitTokens(close, close, emitOpts{trailing: close != v.TokEnd()}), } out := []doc.Doc{ - f.emitTokens(v.TokStart(), open, emitOpts{skipText: []int{open}, breakSkip: true}), + f.emitTokens(v.TokStart(), open, emitOpts{skipText: []int{open}}), body, } - if v.Annotations != nil { - out = append(out, f.breakBeforeAnnotations(close)) - } - out = append(out, f.annotationsDoc(v.Annotations, v.Annotations != nil && v.Annotations.TokEnd() == v.TokEnd())) - out = append(out, f.afterAnnotations(v.Annotations, v.TokEnd())) return doc.Concat(out) } +// endsWithLineComment reports whether the emission before the closing token +// ends with a line comment, which owns its line end: the last function's +// same-line comments, the open brace's same-line comments (empty body), or +// the final comment in the gap before the close. The closing line must then +// collapse instead of leaving a blank. +func (f *formatter) endsWithLineComment(open, close, lastIdx int) bool { + if lastIdx >= 0 && f.sameLineEndsLine(lastIdx) { + return true + } + + if lastIdx < 0 && f.sameLineEndsLine(open) { + return true + } + + prev := f.prevReal(close - 1) + if c := close - 1; c > prev { + ct := f.token(c) + if ct.Line == f.token(prev).Line { + return lineComment(ct.Kind) + } + + return true // own-line comment: always ends with a hard line + } + + return false +} + // function formats a service method. The signature escalates via nested // groups: the whole signature folds when it fits, otherwise the throws // clause unfolds while the arguments stay flat, and the arguments unfold @@ -233,8 +230,8 @@ func (f *formatter) service(v *syntax.Service) doc.Doc { // trailing delimiter in throws never break the arguments, because the // throws clause is a sibling group, not an ancestor. func (f *formatter) function(v *syntax.Function) doc.Doc { - parts := append(f.leadingComments(v), f.functionBody(v)) - parts = append(parts, f.trailingComments(v, true)...) + parts := append(f.ownLineComments(v.TokStart()), f.functionBody(v)) + parts = append(parts, f.sameLineComments(v.TokEnd())...) return doc.Concat(parts) } @@ -242,10 +239,10 @@ func (f *formatter) function(v *syntax.Function) doc.Doc { func (f *formatter) functionBody(v *syntax.Function) doc.Doc { // The header (up to the args open paren) renders as a token run, so // comments between the header tokens are preserved by construction. - // The open paren's text is emitted by the args group; its trailing - // trivia belongs to openTrivia. + // The open paren's text is emitted by the args group; its same-line + // comments belong to that group too. open := f.scanKind(v.TokStart(), v.TokEnd(), syntax.TokenLParen) - header := f.emitTokens(v.TokStart(), open, emitOpts{skipText: []int{open}, breakSkip: true}) + header := f.emitTokens(v.TokStart(), open, emitOpts{skipText: []int{open}}) // Comments or blank lines in the arguments force the multiline layout: // the flat argument group would drop them. @@ -256,7 +253,7 @@ func (f *formatter) functionBody(v *syntax.Function) doc.Doc { // The argument group folds to "(a, b)" when it fits and unfolds to one // field per line otherwise, like the throws clause. - args := f.parenGroup(v.Args, open, parenClose(v.Args, open), false, argsMode) + args := f.parenGroup(v.Args, open, f.parenClose(v.Args, open), false, argsMode) if v.Throws == nil { return doc.Group(doc.Concat{ @@ -276,17 +273,20 @@ func (f *formatter) functionBody(v *syntax.Function) doc.Doc { // parenGroup renders "(fields)" as its own group, folding independently: // flat when it fits, one field per line otherwise. open and close are the -// paren token indices, whose trivia is preserved. forced requires the -// broken layout (comments, blank lines, or a trailing delimiter inside). +// paren token indices. forced requires the broken layout (comments, blank +// lines, or a trailing delimiter inside). func (f *formatter) parenGroup(fields []*syntax.Field, open, close int, forced bool, sepMode SeparatorMode) doc.Doc { broken := f.brokenParens(fields, open, close, sepMode) if forced { broken = doc.Concat{broken, doc.BreakParent} } + flat := append([]doc.Doc{doc.Text("(")}, f.sameLineComments(open)...) + flat = append(flat, f.flatFieldsJoin(fields, sepMode), doc.Text(")")) + return doc.Group(doc.IfBreak( broken, - doc.Concat{doc.Text("("), f.flatFieldsJoin(fields, sepMode), doc.Text(")")}, + doc.Concat(flat), )) } @@ -300,12 +300,12 @@ func (f *formatter) throwsGroup(v *syntax.Function) doc.Doc { // parenClose returns the close paren index matching the open paren at // open, given the field list it encloses. -func parenClose(fields []*syntax.Field, open int) int { +func (f *formatter) parenClose(fields []*syntax.Field, open int) int { if len(fields) == 0 { - return open + 1 + return f.nextReal(open + 1) } - return fields[len(fields)-1].TokEnd() + 1 + return f.nextReal(fields[len(fields)-1].TokEnd() + 1) } // sepForcesBreak reports whether the source's separators force the broken @@ -375,7 +375,7 @@ func (f *formatter) functionBrokenArgs(v *syntax.Function, header doc.Doc) doc.D parts := []doc.Doc{ header, - f.parenGroup(v.Args, open, parenClose(v.Args, open), true, f.opts.Separator.Get(ConstructArguments)), + f.parenGroup(v.Args, open, f.parenClose(v.Args, open), true, f.opts.Separator.Get(ConstructArguments)), } if v.Throws != nil { parts = append(parts, f.throwsGroup(v)) @@ -387,15 +387,15 @@ func (f *formatter) functionBrokenArgs(v *syntax.Function, header doc.Doc) doc.D } // functionTail renders the tokens of the function after its args and -// throws clauses: the trailing trivia of the close parens, the +// throws clauses: the same-line comments of the close parens, the // annotations, and any stray tokens lenient sources leave — everything the // structural layout does not emit itself. open is the args open paren. func (f *formatter) functionTail(v *syntax.Function, open int) doc.Doc { parts := make([]doc.Doc, 0, 8) - argsClose := parenClose(v.Args, open) + argsClose := f.parenClose(v.Args, open) if argsClose < v.TokEnd() { - parts = append(parts, f.tailAfter(argsClose)...) + parts = append(parts, f.sameLineComments(argsClose)...) } idx := argsClose + 1 @@ -403,7 +403,7 @@ func (f *formatter) functionTail(v *syntax.Function, open int) doc.Doc { if v.Throws != nil { throwsClose := v.Throws.TokEnd() if throwsClose < v.TokEnd() { - parts = append(parts, f.tailAfter(throwsClose)...) + parts = append(parts, f.sameLineComments(throwsClose)...) } idx = throwsClose + 1 @@ -411,7 +411,7 @@ func (f *formatter) functionTail(v *syntax.Function, open int) doc.Doc { if v.Annotations != nil { if idx < v.Annotations.TokStart() { - parts = append(parts, f.emitTokens(idx, v.Annotations.TokStart()-1, emitOpts{leading: true})) + parts = append(parts, f.emitTokens(idx, f.prevReal(v.Annotations.TokStart()-1), emitOpts{leading: true})) } parts = append(parts, f.annotationsDoc(v.Annotations, v.Annotations.TokEnd() == v.TokEnd())) @@ -419,34 +419,12 @@ func (f *formatter) functionTail(v *syntax.Function, open int) doc.Doc { } if idx <= v.TokEnd() { - // A line comment on the previous token (e.g. the annotations' - // close paren) would swallow the stray tokens. - if f.lineAfter(idx-1) || len(f.token(idx).Leading) > 0 { - parts = append(parts, doc.HardLine) - } - - parts = append(parts, f.emitTokens(idx, v.TokEnd(), emitOpts{leading: true})) + parts = append(parts, f.emitTokens(f.nextReal(idx), v.TokEnd(), emitOpts{leading: true})) } return doc.Concat(parts) } -// tailAfter renders the trailing trivia of a close paren followed by more -// tokens: the comments inline, with a hard line after line comments so -// nothing is swallowed. -func (f *formatter) tailAfter(idx int) []doc.Doc { - parts := []doc.Doc{} - for _, c := range f.token(idx).Trailing { - parts = append(parts, doc.Text(" "+trimComment(c.Text))) - } - - if f.lineAfter(idx) || len(f.token(idx+1).Leading) > 0 { - parts = append(parts, doc.HardLine) - } - - return parts -} - // brokenFields renders fields one per line, each with its trailing // separator per the sepMode option. Comments and blank lines inside // the list are preserved. @@ -455,19 +433,15 @@ func (f *formatter) brokenFields(fields []*syntax.Field, sepMode SeparatorMode) for i, field := range fields { if i > 0 { - parts = append(parts, doc.HardLineNoBreak) - parts = append(parts, f.blankLines(field, doc.HardLineNoBreak)...) + // The separator line collapses when the previous field ended + // with a line comment (which owns its line end). + parts = append(parts, doc.Line) + parts = append(parts, f.blankLines(field, doc.HardLine)...) } else { - parts = append(parts, f.blankLines(field, doc.HardLineNoBreak)...) + parts = append(parts, f.blankLines(field, doc.HardLine)...) } - content := doc.Concat{ - f.fieldContent(field, f.alignmentFor(fields, i, sepMode), true, sepMode), - trailingSep(field.Sep, sepMode), - } - fieldDoc := append(f.leadingComments(field), content) - fieldDoc = append(fieldDoc, f.trailingComments(field, sepEmits(field.Sep, sepMode))...) - parts = append(parts, doc.Concat(fieldDoc)) + parts = append(parts, f.fieldDoc(field, f.alignmentFor(fields, i, sepMode), 0, sepMode)) } return doc.Concat(parts) @@ -475,24 +449,19 @@ func (f *formatter) brokenFields(fields []*syntax.Field, sepMode SeparatorMode) // brokenParens renders "open, fields, close" one field per line, or just // "openclose" when there are no fields. open and close are the paren token -// indices; their trivia is preserved even with no fields. +// indices; their comments are preserved even with no fields. func (f *formatter) brokenParens(fields []*syntax.Field, open, close int, sepMode SeparatorMode) doc.Doc { if len(fields) == 0 { - closeDoc := f.emitTokens(close, close, emitOpts{leading: true}) - if f.lineAfter(open) || len(f.token(close).Leading) > 0 { - closeDoc = doc.Concat{doc.HardLine, closeDoc} - } - return doc.Concat{ doc.Text("("), - doc.Concat(f.openTriviaAt(open)), - closeDoc, + doc.Concat(f.sameLineComments(open)), + f.emitTokens(close, close, emitOpts{leading: true}), } } - inner := append([]doc.Doc{doc.HardLineNoBreak, f.brokenFields(fields, sepMode)}, f.closingTriviaAt(close)...) - parts := append([]doc.Doc{doc.Text("(")}, f.openTriviaAt(open)...) - parts = append(parts, doc.Indent(doc.Concat(inner)), doc.HardLineNoBreak, doc.Text(")")) + inner := append([]doc.Doc{doc.Line, f.brokenFields(fields, sepMode)}, f.ownLineComments(close)...) + parts := append([]doc.Doc{doc.Text("(")}, f.sameLineComments(open)...) + parts = append(parts, doc.Indent(doc.Concat(inner)), doc.Line, doc.Text(")")) return doc.Concat(parts) } @@ -505,12 +474,12 @@ func (f *formatter) fieldsForcedBroken(fields []*syntax.Field) bool { } // Comments on the opening or closing paren would be lost in the flat // layout. - if len(f.token(fields[0].TokStart()-1).Trailing) > 0 { + if f.hasSameLineComments(f.prevReal(fields[0].TokStart() - 1)) { return true } - close := f.token(fields[len(fields)-1].TokEnd() + 1) - if len(close.Leading) > 0 { + close := f.nextReal(fields[len(fields)-1].TokEnd() + 1) + if f.hasOwnLineComments(close) { return true } @@ -519,7 +488,7 @@ func (f *formatter) fieldsForcedBroken(fields []*syntax.Field) bool { return true } - if len(f.token(field.TokStart()).Leading) > 0 || len(f.token(field.TokEnd()).Trailing) > 0 { + if f.hasOwnLineComments(field.TokStart()) || f.hasSameLineComments(field.TokEnd()) { return true } } @@ -528,11 +497,11 @@ func (f *formatter) fieldsForcedBroken(fields []*syntax.Field) bool { } // blankLines returns count hard-line docs for the blank lines before a -// node's first token. When the node carries leading comments the blank -// lines belong to that run and leadingComments emits them; returning nil +// node's first token. When the node carries own-line comments the blank +// lines belong to that run and ownLineComments emits them; returning nil // here keeps them from being printed twice. func (f *formatter) blankLines(n syntax.Node, line doc.Doc) []doc.Doc { - if len(f.token(n.TokStart()).Leading) > 0 { + if f.hasOwnLineComments(n.TokStart()) { return nil } diff --git a/formatter/field.go b/formatter/field.go index c59d6ce..842a939 100644 --- a/formatter/field.go +++ b/formatter/field.go @@ -80,23 +80,51 @@ func (f *formatter) enumValueList(values []*syntax.EnumValue, bodyID int) doc.Do } // groupedWith reports whether the node joins the alignment group of the -// item before it: no blank line and no comment sits between them. A -// comment in the gap is a visual break, like whitespace. A comment -// trailing the previous item's separator only breaks the group when the -// separator is suppressed: the comment then moves to its own line and -// re-attaches, so the group must not depend on it. +// item before it: no blank line and no own-line comment sits between them. +// A comment renders on its own line when its line differs from the +// previous content's line — a visual break, like whitespace. Same-line +// comments stay on the item's line and do not break the group. func (f *formatter) groupedWith(prev, cur syntax.Node, sepMode SeparatorMode) bool { - // A comment trailing the previous item's separator only breaks the - // group when the separator is suppressed and the comment moves to its - // own line (re-attaching): the group must not depend on it. Comments - // that stay on the separator's line do not break the group. - prevTok := f.token(prev.TokEnd()) - sepBreak := isListSep(prevTok.Kind) && !sepEmits(prevTok.Kind, sepMode) && - (leadingLineComment(prevTok) || len(prevTok.Trailing) > 0 && f.lineAfter(prev.TokEnd()-1)) + prevEnd := prev.TokEnd() + sep := syntax.TokenKind(0) + if isListSep(f.token(prevEnd).Kind) { + // The previous item's separator: comments before it belong to + // the item's own span, so the scan starts at the content end. + sep = f.token(prevEnd).Kind + prevEnd = f.prevReal(prevEnd - 1) + } return f.blankBefore(cur) < 1 && - len(f.token(cur.TokStart()).Leading) == 0 && - !sepBreak + !f.commentBreaksGroup(prevEnd, cur.TokStart(), sep, sepMode) +} + +// commentBreaksGroup reports whether a comment between the real tokens at +// prevEnd and curStart renders on its own line — a visual break between +// the items. A comment renders inline when it shares the previous +// content's line, or when it follows the previous item's separator on the +// separator's line and the separator text is emitted; everything else +// starts its own line. +func (f *formatter) commentBreaksGroup(prevEnd, curStart int, sep syntax.TokenKind, sepMode SeparatorMode) bool { + line := f.token(prevEnd).Line + + for c := prevEnd + 1; c < curStart; c++ { + ct := f.token(c) + if !isComment(ct.Kind) { + continue + } + + if ct.Line == line { + continue // inline with the content + } + + if sep != 0 && sepEmits(sep, sepMode) && ct.Line == f.token(f.nextReal(prevEnd+1)).Line { + continue // inline after the emitted separator + } + + return true + } + + return false } // alignmentFor returns the column alignment for field i, or nil when @@ -298,10 +326,46 @@ func computeEnumAlign(values []*syntax.EnumValue) *columnAlign { return a } -// field assembles a struct-like body field: leading comments, content, and -// trailing comments. When the body breaks (referenced by bodyID), the -// content switches to its column-aligned form with the trailing separator. -func (f *formatter) field(v *syntax.Field, align *columnAlign, bodyID int, sepMode SeparatorMode) doc.Doc { +// nameOnlyComment reports whether a comment follows the name of a +// name-only value (no content tokens) on the same source line, directly +// or after the separator: with the separator text dropped it renders +// against the name pad. +func (f *formatter) nameOnlyComment(name int) bool { + for c := name + 1; c < len(f.toks); c++ { + ct := f.token(c) + if isComment(ct.Kind) { + return ct.Line == f.token(name).Line + } + + if ct.Line != f.token(name).Line { + return false + } + } + + return false +} + +// nodeTrailingInline reports whether the same-line comments after the +// node's last token (its separator, when present) render inline after it: +// the token's text was emitted, or — for a dropped separator — the +// comments share the previous content's line. Otherwise the comments start +// their own line and are emitted by the next item's leading comments. +func (f *formatter) nodeTrailingInline(end int, sep syntax.TokenKind, sepMode SeparatorMode) bool { + if sep == 0 || sepEmits(sep, sepMode) { + return true + } + + prev := f.prevReal(end - 1) + + return prev >= 0 && f.token(prev).Line == f.token(end).Line +} + +// fieldDoc assembles a field with its comments: own-line comments, the +// content, and the same-line comments at the item boundary. With a +// non-zero bodyID the content switches between the flat and column-aligned +// forms on the body group's break state; with bodyID zero the broken form +// renders directly (paren bodies are always broken). +func (f *formatter) fieldDoc(v *syntax.Field, align *columnAlign, bodyID int, sepMode SeparatorMode) doc.Doc { content := f.fieldContent(v, align, false, sepMode) if bodyID != 0 { content = doc.IfBreakFor( @@ -309,33 +373,41 @@ func (f *formatter) field(v *syntax.Field, align *columnAlign, bodyID int, sepMo content, bodyID, ) + } else { + content = doc.Concat{f.fieldContent(v, align, true, sepMode), trailingSep(v.Sep, sepMode)} } - parts := append(f.leadingComments(v), content) - parts = append(parts, f.trailingComments(v, sepEmits(v.Sep, sepMode))...) + parts := append(f.ownLineComments(v.TokStart()), content) + if f.nodeTrailingInline(v.TokEnd(), v.Sep, sepMode) { + parts = append(parts, f.sameLineComments(v.TokEnd())...) + } else { + parts = append(parts, f.suppressedSepComments(v.TokEnd())...) + } return doc.Concat(parts) } +// field assembles a struct-like body field, switching on the body group's +// break state. +func (f *formatter) field(v *syntax.Field, align *columnAlign, bodyID int, sepMode SeparatorMode) doc.Doc { + return f.fieldDoc(v, align, bodyID, sepMode) +} + // emitWithAnnotations renders a token run split at the node's annotations, // so they keep their foldable group instead of being inlined. func (f *formatter) emitWithAnnotations(start, end int, ann *syntax.Annotations, o emitOpts) doc.Doc { if ann == nil { return f.emitTokens(start, end, o) } - // The segment before the annotations owns its last token's trailing - // trivia (the annotations' close owns the node's trailing instead). + // The segment before the annotations owns its last token's same-line + // comments (the annotations' close owns the node's trailing instead). first := o first.trailing = true parts := []doc.Doc{f.emitTokens(start, ann.TokStart()-1, first)} - if f.lineAfter(ann.TokStart()-1) || len(f.token(ann.TokStart()).Leading) > 0 { - parts = append(parts, doc.HardLine) - } - parts = append(parts, f.annotationsDoc(ann, ann.TokEnd() == end)) if ann.TokEnd() < end { - parts = append(parts, f.emitTokens(ann.TokEnd()+1, end, o)) + parts = append(parts, f.emitTokens(f.nextReal(ann.TokEnd()+1), end, emitOpts{leading: true, skipText: o.skipText})) } return doc.Concat(parts) @@ -344,14 +416,13 @@ func (f *formatter) emitWithAnnotations(start, end int, ann *syntax.Annotations, // fieldContent renders the field as a token run. padded selects the // column-aligned form used when the enclosing body breaks; it has no // effect when align is nil. The separator token's text is suppressed (the -// caller emits it), but its trivia is preserved. +// caller emits it), but its comments are preserved. func (f *formatter) fieldContent(v *syntax.Field, align *columnAlign, padded bool, sepMode SeparatorMode) doc.Doc { padded = padded && align != nil - o := emitOpts{breakTrailing: true} + o := emitOpts{} if v.Sep != 0 { o.skipText = []int{v.TokEnd()} - o.breakSkip = sepEmits(v.Sep, sepMode) } if padded { @@ -371,14 +442,13 @@ func (f *formatter) fieldPads(v *syntax.Field, a *columnAlign) ([]padEntry, stri end-- } - for i := start + 1; i <= end; i++ { - if len(f.token(i).Leading) > 0 { - return nil, "" - } - } - - for i := start; i < end; i++ { - if len(f.token(i).Trailing) > 0 { + // A comment inside the field — a comment token in the span, or a + // same-line comment after any token before the last — makes the padded + // widths unknowable. The last content token's same-line comments render + // after the pads and are fine. + contentEnd := f.prevReal(end) + for i := start; i < contentEnd; i++ { + if isComment(f.token(i).Kind) || f.hasSameLineComments(i) { return nil, "" } } @@ -429,8 +499,12 @@ func (f *formatter) enumValue(v *syntax.EnumValue, align *columnAlign, bodyID in ) } - parts := append(f.leadingComments(v), content) - parts = append(parts, f.trailingComments(v, sepEmits(v.Sep, f.opts.Separator.Get(ConstructEnum)))...) + parts := append(f.ownLineComments(v.TokStart()), content) + if f.nodeTrailingInline(v.TokEnd(), v.Sep, f.opts.Separator.Get(ConstructEnum)) { + parts = append(parts, f.sameLineComments(v.TokEnd())...) + } else { + parts = append(parts, f.suppressedSepComments(v.TokEnd())...) + } return doc.Concat(parts) } @@ -438,14 +512,40 @@ func (f *formatter) enumValue(v *syntax.EnumValue, align *columnAlign, bodyID in func (f *formatter) enumValueContent(v *syntax.EnumValue, align *columnAlign, padded bool, sepMode SeparatorMode) doc.Doc { padded = padded && align != nil - o := emitOpts{breakTrailing: true} + o := emitOpts{} if v.Sep != 0 { o.skipText = []int{v.TokEnd()} - o.breakSkip = sepEmits(v.Sep, sepMode) } if padded && align.enumAssign && align.nameWidth > 0 { - o.pads = []padEntry{{v.Name.TokStart(), padRight("", align.nameWidth-len(v.Name.Text))}} + // A comment after the name (or anywhere before the value's end) + // makes the pad ambiguous; the value's own same-line comments + // render after the pads and are fine. + contentEnd := v.TokEnd() + if v.Sep != 0 { + contentEnd = f.prevReal(v.TokEnd() - 1) + } + + clean := true + + // A comment between the name and the value's end renders against + // the pad. + for i := v.TokStart() + 1; i <= contentEnd && clean; i++ { + if isComment(f.token(i).Kind) { + clean = false + } + } + + // A name-only value renders a same-line comment (directly or + // after its separator) against the pad when no separator text + // separates them. + if clean && contentEnd == v.TokStart() && !sepEmits(v.Sep, sepMode) && f.nameOnlyComment(v.TokStart()) { + clean = false + } + + if clean { + o.pads = []padEntry{{v.Name.TokStart(), padRight("", align.nameWidth-len(v.Name.Text))}} + } } return f.emitWithAnnotations(v.TokStart(), v.TokEnd(), v.Annotations, o) diff --git a/formatter/format.go b/formatter/format.go index b010962..9c1bca2 100644 --- a/formatter/format.go +++ b/formatter/format.go @@ -56,16 +56,22 @@ const ( ConstructEnum ConstructArguments ConstructThrows + ConstructList + ConstructMap ) -// PerConstruct holds one option value per construct. +// PerConstruct holds one option value per construct. The JSON tags make the +// per-construct option maps config-compatible ("structs", "arguments", ...), +// so the options layer and the CLI share this single source of truth. type PerConstruct[T any] struct { - Structs T - Unions T - Exceptions T - Enums T - Arguments T - Throws T + Structs T `json:"structs"` + Unions T `json:"unions"` + Exceptions T `json:"exceptions"` + Enums T `json:"enums"` + Arguments T `json:"arguments"` + Throws T `json:"throws"` + Lists T `json:"lists"` + Maps T `json:"maps"` } // Get returns the value for the construct. @@ -81,6 +87,10 @@ func (p PerConstruct[T]) Get(c Construct) T { return p.Arguments case ConstructThrows: return p.Throws + case ConstructList: + return p.Lists + case ConstructMap: + return p.Maps } return p.Structs @@ -99,6 +109,10 @@ func (p *PerConstruct[T]) Set(c Construct, v T) { p.Arguments = v case ConstructThrows: p.Throws = v + case ConstructList: + p.Lists = v + case ConstructMap: + p.Maps = v default: p.Structs = v } @@ -108,6 +122,7 @@ func (p *PerConstruct[T]) Set(c Construct, v T) { var AllConstructs = []Construct{ ConstructStruct, ConstructUnion, ConstructException, ConstructEnum, ConstructArguments, ConstructThrows, + ConstructList, ConstructMap, } // String returns the config key of the construct. @@ -123,6 +138,10 @@ func (c Construct) String() string { return "arguments" case ConstructThrows: return "throws" + case ConstructList: + return "lists" + case ConstructMap: + return "maps" } return "structs" @@ -267,14 +286,13 @@ func (f *formatter) token(i int) syntax.Token { // emitOpts controls token emission. type emitOpts struct { - leading bool // emit the first token's leading trivia - trailing bool // emit the last token's trailing trivia - breakTrailing bool // line-comment trailing forces groups to break - skipText []int // token indexes whose text and gap are suppressed - breakSkip bool // hard line before a skipped token whose text - // the caller emits (separators) - pads []padEntry // spaces inserted after a token, before its gap - prefix string // spaces emitted before the first token + leading bool // emit the own-line comments before the first token + trailing bool // emit the same-line comments after the last token + + skipText []int // token indexes whose text is suppressed + text string // text emitted in place of a skipped token's text + pads []padEntry // spaces inserted after a token, before its gap + prefix string // spaces emitted before the first token } // padEntry is one alignment pad at a token index. @@ -308,54 +326,90 @@ func padAt(pads []padEntry, idx int) string { return out } -// emitTokens renders the tokens in [start, end] with their trivia, joined -// with canonical spacing. The first token's leading and last token's -// trailing trivia belong to the caller's comment helpers unless the +// isComment reports whether the token kind is a comment trivia token. +func isComment(k syntax.TokenKind) bool { + return syntax.IsComment(k) +} + +// lineComment reports whether the token kind is a line comment or +// annotation, which consumes the rest of its source line: whatever follows +// always starts a fresh line. +func lineComment(k syntax.TokenKind) bool { + return k == syntax.TokenLineComment || k == syntax.TokenAnnotation +} + +// prevReal returns the index of the previous real (non-comment) token +// strictly before idx, or -1. +func (f *formatter) prevReal(idx int) int { + for idx >= 0 && isComment(f.token(idx).Kind) { + idx-- + } + + return idx +} + +// nextReal returns the index of the next real (non-comment) token at or +// after idx. +func (f *formatter) nextReal(idx int) int { + for idx < len(f.toks) && isComment(f.token(idx).Kind) { + idx++ + } + + return idx +} + +// emitTokens renders the tokens in [start, end] with the comments +// interleaved between them, joined with canonical spacing. Comments render +// by one uniform rule: a comment on the same source line as the previous +// real token renders inline after it (a line comment owns its line end and +// emits a hard line), a comment on its own line renders on its own line +// and always ends with a hard line. The first token's own-line comments +// and the last token's same-line comments belong to the caller unless the // corresponding flag is set. skipText suppresses separator tokens that the -// structural layout emits itself; pads widen alignment columns. +// structural layout emits itself (text replaces the suppressed text when +// set); pads widen alignment columns. func (f *formatter) emitTokens(start, end int, o emitOpts) doc.Doc { parts := make([]doc.Doc, 0, 8) if o.prefix != "" { parts = append(parts, doc.Text(o.prefix)) } + if o.leading { + parts = append(parts, f.ownLineComments(start)...) + } + + prev := start + first := true + for i := start; i <= end; i++ { - tok := f.token(i) + if isComment(f.token(i).Kind) { + continue + } skipped := containsInt(o.skipText, i) - if i > start { - if skipped { - // Leading trivia always forces a hard line; a trailing - // line comment only when the caller emits the skipped - // token's text after it (separators), which would - // otherwise be swallowed. - if len(tok.Leading) > 0 || o.breakSkip && f.lineAfter(i-1) { - parts = append(parts, doc.HardLine) - } - } else { - parts = append(parts, f.tokenGap(f.token(i-1), tok)) - } - } - if i > start || o.leading { - for j, c := range tok.Leading { - parts = append(parts, doc.Text(trimComment(c.Text))) - // The last comment's line end comes from the caller's - // structure for suppressed tokens, unless the caller - // emits the token's text after it. - if j < len(tok.Leading)-1 || !skipped || o.breakSkip { - parts = append(parts, doc.HardLine) - } + if !first { + // Comments between the previous real token and this one + // render in place; the canonical gap follows only when the + // run left the line open. A skipped token whose text the + // caller emits itself (braces, field separators) gets no + // canonical gap — the caller's own spacing provides it. + comments, lineEnded := f.commentsRun(prev, i) + parts = append(parts, comments...) + if !lineEnded && (!skipped || o.text != "") { + parts = append(parts, f.tokenGap(prev, i)) } } if !skipped { - text := tok.Text - if tok.Kind == syntax.TokenAsync { + text := f.token(i).Text + if f.token(i).Kind == syntax.TokenAsync { text = "oneway" } parts = append(parts, doc.Text(text)) + } else if o.text != "" { + parts = append(parts, doc.Text(o.text)) } if !skipped && o.pads != nil { @@ -364,43 +418,159 @@ func (f *formatter) emitTokens(start, end int, o emitOpts) doc.Doc { } } - if i < end || o.trailing { - for _, c := range tok.Trailing { - parts = append(parts, doc.Text(" "+trimComment(c.Text))) - if o.breakTrailing && (c.Kind == syntax.TriviaLineComment || c.Kind == syntax.TriviaAnnotation) { - // A line comment must end its line: force the - // enclosing groups to break so nothing follows it. - parts = append(parts, doc.BreakParent) - } + prev = i + first = false + } + + if o.trailing { + // Same-line comments after the last real token. When the token's + // text is suppressed (the separator mode drops it), its same-line + // comments render inline only when they also share the previous + // content's line; otherwise they start their own line, so the + // output round-trips — the next emission would skip them as + // same-line with the suppressed token. + if containsInt(o.skipText, prev) && o.text == "" { + prevTok := f.prevReal(prev - 1) + if prevTok >= 0 && f.token(prevTok).Line == f.token(prev).Line { + parts = append(parts, f.sameLineComments(prev)...) + } else { + parts = append(parts, f.suppressedSepComments(prev)...) } + } else { + parts = append(parts, f.sameLineComments(prev)...) } } return doc.Concat(parts) } -// tokenGap returns the doc between two adjacent tokens: a line break when -// the source separated them with a line comment (which would swallow the -// next token on the same line), a canonical space otherwise. -func (f *formatter) tokenGap(prev, cur syntax.Token) doc.Doc { - for _, c := range prev.Trailing { - if c.Kind == syntax.TriviaLineComment || c.Kind == syntax.TriviaAnnotation { - return doc.HardLine +// suppressedSepComments renders the same-line comments of a suppressed +// separator (the mode drops its text) that cannot share the previous +// content's line: each starts its own line, so the output round-trips. +func (f *formatter) suppressedSepComments(idx int) []doc.Doc { + parts := make([]doc.Doc, 0, 4) + + for c := idx + 1; c < len(f.toks); c++ { + ct := f.token(c) + if !isComment(ct.Kind) || ct.Line != f.token(idx).Line { + break + } + + parts = append(parts, doc.SoftLine) + parts = append(parts, f.blankLineDocs(ct.BlankLinesBefore, doc.HardLine)...) + parts = append(parts, doc.Text(trimComment(ct.Text)), doc.HardLine) + } + + return parts +} + +// commentsRun renders the comments strictly between the real tokens at +// prev and cur (all stream entries in between are comments): a comment on +// the previous token's line renders inline after it, a comment on its own +// line renders on its own line and owns its line end. It reports whether +// the run ended the line, in which case the caller must not emit the +// canonical gap. +func (f *formatter) commentsRun(prev, cur int) ([]doc.Doc, bool) { + parts := make([]doc.Doc, 0, 4) + lineEnded := false + + for c := prev + 1; c < cur; c++ { + ct := f.token(c) + if ct.Line == f.token(prev).Line { + // Same-line: inline after the previous token. A line comment + // owns its line end. + parts = append(parts, doc.Text(" "), doc.Text(trimComment(ct.Text))) + if lineComment(ct.Kind) { + parts = append(parts, doc.HardLine) + lineEnded = true + } else { + lineEnded = false + } + + continue + } + + // Own-line: the comment starts its own line. The soft line + // collapses when the output already ended the line (the comment + // before it owns its line end) and provides the break otherwise. + parts = append(parts, doc.SoftLine) + parts = append(parts, f.blankLineDocs(ct.BlankLinesBefore, doc.HardLine)...) + parts = append(parts, doc.Text(trimComment(ct.Text)), doc.HardLine) + lineEnded = true + } + + return parts, lineEnded +} + +// ownLineComments renders the own-line comments in the gap before the real +// token at idx: the comments between the previous real token and idx that +// start their own source line, each preceded by its blank lines and +// followed by a hard line, plus the blank lines between the last comment +// and idx itself. Same-line comments in the gap belong to the previous +// token's trailing and are emitted by the caller's trailing phase or the +// next commentsRun. +func (f *formatter) ownLineComments(idx int) []doc.Doc { + prev := f.prevReal(idx - 1) + + parts := make([]doc.Doc, 0, 4) + first := true + + for c := prev + 1; c < idx; c++ { + ct := f.token(c) + if prev >= 0 && ct.Line == f.token(prev).Line { + continue } + + if prev >= 0 { + // The soft line collapses when the output already ended the + // line and provides the break otherwise. + parts = append(parts, doc.SoftLine) + } else if first && ct.BlankLinesBefore > 0 { + // At file start there is no separator line before the first + // comment: N blank lines round-trip as N+1 newlines. + parts = append(parts, doc.HardLine) + } + + first = false + + parts = append(parts, f.blankLineDocs(ct.BlankLinesBefore, doc.HardLine)...) + parts = append(parts, doc.Text(trimComment(ct.Text)), doc.HardLine) } - if len(cur.Leading) > 0 { - return doc.HardLine + if len(parts) > 0 && f.token(idx).BlankLinesBefore > 0 { + parts = append(parts, f.blankLineDocs(f.token(idx).BlankLinesBefore, doc.HardLine)...) } - return doc.Text(rawTokenGap(prev, cur)) + return parts } -// lineAfter reports whether the token ends its line with a line comment or -// annotation, which forces the next doc onto a new line. -func (f *formatter) lineAfter(i int) bool { - for _, c := range f.token(i).Trailing { - if c.Kind == syntax.TriviaLineComment || c.Kind == syntax.TriviaAnnotation { +// sameLineComments renders the same-line comments after the real token at +// idx: comments sharing its source line, each preceded by a space and — +// for line comments — ending with a hard line. +func (f *formatter) sameLineComments(idx int) []doc.Doc { + parts := make([]doc.Doc, 0, 4) + + for c := idx + 1; c < len(f.toks); c++ { + ct := f.token(c) + if !isComment(ct.Kind) || ct.Line != f.token(idx).Line { + break + } + + parts = append(parts, doc.Text(" "), doc.Text(trimComment(ct.Text))) + if lineComment(ct.Kind) { + parts = append(parts, doc.HardLine) + } + } + + return parts +} + +// hasOwnLineComments reports whether the gap before the real token at idx +// contains an own-line comment. +func (f *formatter) hasOwnLineComments(idx int) bool { + prev := f.prevReal(idx - 1) + for c := prev + 1; c < idx; c++ { + if prev < 0 || f.token(c).Line != f.token(prev).Line { return true } } @@ -408,30 +578,63 @@ func (f *formatter) lineAfter(i int) bool { return false } -// foldBreak is the foldable gap after an opening token or separating -// comma: a line in the broken layout, a space (or nothing) flat. A -// trailing line comment forces a hard line. -func (f *formatter) foldBreak(i int, flat string) doc.Doc { - if f.lineAfter(i) { - return doc.HardLine +// hasSameLineComments reports whether the real token at idx has same-line +// comments after it. +func (f *formatter) hasSameLineComments(idx int) bool { + c := idx + 1 + if c >= len(f.toks) { + return false } - return doc.IfBreak(doc.Line, doc.Text(flat)) + ct := f.token(c) + if !isComment(ct.Kind) || ct.Line != f.token(idx).Line { + return false + } + + return true } -// commaSep renders a separating comma with its trivia: a hard line when -// the previous item ends its line with a comment (which would swallow the -// comma), then the comma and the foldable gap after it. -func (f *formatter) commaSep(comma int) []doc.Doc { - parts := []doc.Doc{} - if f.lineAfter(comma-1) || len(f.token(comma).Leading) > 0 { - parts = append(parts, doc.HardLine) +// tokenGap returns the canonical gap between two adjacent real tokens: +// nothing when the previous token's same-line comments already ended the +// line, the canonical spacing otherwise. +func (f *formatter) tokenGap(prev, cur int) doc.Doc { + if f.sameLineEndsLine(prev) { + return doc.Concat{} } - parts = append(parts, f.emitTokens(comma, comma, emitOpts{leading: true, trailing: true})) - parts = append(parts, f.foldBreak(comma, " ")) + return doc.Text(rawTokenGap(f.token(prev), f.token(cur))) +} - return parts +// sameLineEndsLine reports whether the token's same-line comments end with +// a line comment, which owns its line end. +func (f *formatter) sameLineEndsLine(idx int) bool { + for c := idx + 1; c < len(f.toks); c++ { + ct := f.token(c) + if !isComment(ct.Kind) || ct.Line != f.token(idx).Line { + break + } + + if lineComment(ct.Kind) { + return true + } + } + + return false +} + +// foldBreak is the foldable gap after an opening token or separating +// comma: a line in the broken layout, a space (or nothing) flat. +func (f *formatter) foldBreak(i int, flat string) doc.Doc { + return doc.IfBreak(doc.Line, doc.Text(flat)) +} + +// commaSep renders a separating comma with its trivia, then the foldable +// gap after it. +func (f *formatter) commaSep(comma int) []doc.Doc { + return []doc.Doc{ + f.emitTokens(comma, comma, emitOpts{leading: true, trailing: true}), + f.foldBreak(comma, " "), + } } // rawTokenGap returns the canonical text between two adjacent tokens: @@ -468,81 +671,11 @@ func (f *formatter) blankBefore(n syntax.Node) int { return f.token(n.TokStart()).BlankLinesBefore } -// leadingComments returns the comments attached before the node's first -// token, each ending with a hard line, with the blank lines from the source -// gap distributed exactly as written: before the run, between comments, and -// between the last comment and the node itself. blankLines deliberately -// emits nothing when leading comments exist. -func (f *formatter) leadingComments(n syntax.Node) []doc.Doc { - tok := f.token(n.TokStart()) - if len(tok.Leading) == 0 { - return nil - } - - parts := make([]doc.Doc, 0, 8) - - prevBlank := 0 - for _, c := range tok.Leading { - parts = append(parts, f.blankLineDocs(c.BlankLinesBefore-prevBlank, doc.HardLine)...) - prevBlank = c.BlankLinesBefore - parts = append(parts, doc.Text(trimComment(c.Text)), doc.HardLine) - } - - parts = append(parts, f.blankLineDocs(tok.BlankLinesBefore-prevBlank, doc.HardLine)...) - - return parts -} - -// trailingComments returns the comments attached after the node's last token -// on the same line. Block comments render as line-suffix docs; line comments -// and annotations render inline with a break parent, since a line suffix -// after them would merge into the comment's text. When the content before -// the separator already ends with a line comment, these comments cannot -// share the line and get their own lines instead. -func (f *formatter) trailingComments(n syntax.Node, sepEmitted bool) []doc.Doc { - parts := make([]doc.Doc, 0, 8) - // Comments attached to a separator share the separator's own line, - // unless the separator is not emitted (the mode drops it): then a line - // comment before it would swallow them, and they need their own lines. - last := f.token(n.TokEnd()) - - ownLine := !sepEmitted && (last.Kind == syntax.TokenComma || last.Kind == syntax.TokenSemicolon) && - (f.lineAfter(n.TokEnd()-1) || leadingLineComment(last)) - for _, c := range last.Trailing { - line := c.Kind == syntax.TriviaLineComment || c.Kind == syntax.TriviaAnnotation - if ownLine { - parts = append(parts, doc.HardLine, doc.Text(trimComment(c.Text)), doc.BreakParent) - - continue - } - - if line { - parts = append(parts, doc.Text(" "+trimComment(c.Text)), doc.BreakParent) - } else { - parts = append(parts, doc.LineSuffix(doc.Text(" "+trimComment(c.Text))), doc.BreakParent) - } - } - - return parts -} - -// leadingLineComment reports whether the token's leading trivia contains a -// line comment or annotation, which ends the previous line. -func leadingLineComment(tok syntax.Token) bool { - for _, c := range tok.Leading { - if c.Kind == syntax.TriviaLineComment || c.Kind == syntax.TriviaAnnotation { - return true - } - } - - return false -} - -// node assembles a top-level node: its leading comments, its formatted -// body, and its trailing comments. +// node assembles a top-level node: its own-line comments, its formatted +// body, and its same-line comments. func (f *formatter) node(n syntax.Node) doc.Doc { - parts := append(f.leadingComments(n), f.nodeBody(n)) - parts = append(parts, f.trailingComments(n, true)...) + parts := append(f.ownLineComments(n.TokStart()), f.nodeBody(n)) + parts = append(parts, f.sameLineComments(n.TokEnd())...) return doc.Concat(parts) } @@ -571,49 +704,29 @@ func (f *formatter) nodeBody(n syntax.Node) doc.Doc { } } -// document assembles the whole file: top-level nodes separated by hard -// lines, blank lines preserved, and trailing comments. +// document assembles the whole file: top-level nodes separated by +// collapsible lines, blank lines preserved, and trailing comments. func (f *formatter) document() doc.Doc { parts := make([]doc.Doc, 0, 8) for i, n := range f.doc.Nodes { if i > 0 { - parts = append(parts, doc.HardLine) + // The separator line collapses when the previous node ended + // with a line comment (which owns its line end). + parts = append(parts, doc.Line) parts = append(parts, f.blankLines(n, doc.HardLine)...) - } else if lead := f.token(n.TokStart()).Leading; len(lead) > 0 && lead[0].BlankLinesBefore > 0 { - // At file start the leading comments carry their blanks without - // a separator line, so N blanks would round-trip as N-1. The - // extra line keeps the count canonical: N blanks are N+1 - // newlines. - parts = append(parts, doc.HardLine) } parts = append(parts, f.node(n)) } - // Comments at the end of the file attach to the EOF token. Like the - // first node's leading comments, a comment run at file start (no nodes) - // needs a separator line before its blanks to round-trip the count. - eof := f.toks[len(f.toks)-1] - if len(eof.Leading) > 0 { - if len(f.doc.Nodes) > 0 || eof.Leading[0].BlankLinesBefore > 0 { - parts = append(parts, doc.HardLine) - } - - prevBlank := 0 - for i, c := range eof.Leading { - parts = append(parts, f.blankLineDocs(c.BlankLinesBefore-prevBlank, doc.HardLine)...) - prevBlank = c.BlankLinesBefore - - parts = append(parts, doc.Text(trimComment(c.Text))) - if i < len(eof.Leading)-1 { - parts = append(parts, doc.HardLine) - } - } - } + // Comments at the end of the file precede the EOF token. + parts = append(parts, f.ownLineComments(len(f.toks)-1)...) if !f.opts.NoTrailingNewline { - parts = append(parts, doc.HardLine) + // The final newline collapses when the file ends with a line + // comment (which owns its line end). + parts = append(parts, doc.Line) } return doc.Concat(parts) @@ -641,10 +754,6 @@ func (f *formatter) namespace(v *syntax.Namespace) doc.Doc { } parts := []doc.Doc{f.emitTokens(v.TokStart(), end, o)} - if v.Annotations != nil { - parts = append(parts, f.breakBeforeAnnotations(end)) - } - parts = append(parts, f.annotationsDoc(v.Annotations, v.Annotations != nil && v.Annotations.TokEnd() == v.TokEnd())) parts = append(parts, f.afterAnnotations(v.Annotations, v.TokEnd())) @@ -663,10 +772,6 @@ func (f *formatter) typedef(v *syntax.Typedef) doc.Doc { } parts := []doc.Doc{f.emitTokens(v.TokStart(), end, o)} - if v.Annotations != nil { - parts = append(parts, f.breakBeforeAnnotations(end)) - } - parts = append(parts, f.annotationsDoc(v.Annotations, v.Annotations != nil && v.Annotations.TokEnd() == v.TokEnd())) parts = append(parts, f.afterAnnotations(v.Annotations, v.TokEnd())) @@ -679,23 +784,22 @@ func (f *formatter) constant(v *syntax.Const) doc.Doc { return f.emitTokens(v.TokStart(), v.TokEnd(), emitOpts{}) } - eq := value.TokStart() - 1 - gap := f.tokenGap(f.token(eq), f.token(value.TokStart())) + eq := f.prevReal(value.TokStart() - 1) parts := []doc.Doc{ f.emitTokens(v.TokStart(), eq, emitOpts{trailing: true}), - gap, - f.constValue(value, value.TokEnd() == v.TokEnd()), + f.tokenGap(eq, value.TokStart()), } + // Own-line comments before the value render at the value boundary, + // outside the value's own group. + parts = append(parts, f.ownLineComments(value.TokStart())...) + parts = append(parts, f.constValue(value, value.TokEnd() == v.TokEnd())) if value.TokEnd() < v.TokEnd() { - // Stray tokens after the value (lenient sources): preserve them - // and their trivia, with a line break after the value's close. - stray := f.emitTokens(value.TokEnd()+1, v.TokEnd(), emitOpts{leading: true}) - if f.lineAfter(value.TokEnd()) || len(f.token(value.TokEnd()+1).Leading) > 0 { - stray = doc.Concat{doc.HardLine, stray} - } - - parts = append(parts, stray) + // Same-line comments after the value render at the value + // boundary, outside the value's own group, before the stray + // tokens (the const's trailing separator). + parts = append(parts, f.sameLineComments(value.TokEnd())...) + parts = append(parts, f.emitTokens(f.nextReal(value.TokEnd()+1), v.TokEnd(), emitOpts{leading: true})) } return doc.Concat(parts) @@ -703,39 +807,20 @@ func (f *formatter) constant(v *syntax.Const) doc.Doc { // --- annotations ----------------------------------------------------------- -// breakBeforeAnnotations returns a hard line when the token at idx ends -// its line with a comment, or the annotations' first token carries leading -// trivia, so neither gets swallowed by the other. -func (f *formatter) breakBeforeAnnotations(idx int) doc.Doc { - if f.lineAfter(idx) || len(f.token(idx+1).Leading) > 0 { - return doc.HardLine - } - - return doc.Concat{} -} - // afterAnnotations renders any tokens between the annotations and the // node's end — stray separators lenient sources may leave — preserving -// their leading trivia and forcing a line break after the annotations' -// close when it ends its line with a comment. +// their comments. func (f *formatter) afterAnnotations(a *syntax.Annotations, end int) doc.Doc { if a == nil || a.TokEnd() >= end { return doc.Concat{} } - parts := []doc.Doc{} - if f.lineAfter(a.TokEnd()) || len(f.token(a.TokEnd()+1).Leading) > 0 { - parts = append(parts, doc.HardLine) - } - - parts = append(parts, f.emitTokens(a.TokEnd()+1, end, emitOpts{leading: true})) - - return doc.Concat(parts) + return f.emitTokens(f.nextReal(a.TokEnd()+1), end, emitOpts{leading: true}) } // annotationsDoc returns an annotation group, or an empty doc when absent. // The group folds when it does not fit; the items and their separating -// commas render as token runs, so trivia inside the parens is preserved. +// commas render as token runs, so comments inside the parens are preserved. func (f *formatter) annotationsDoc(a *syntax.Annotations, isLast bool) doc.Doc { if a == nil { return doc.Concat{} @@ -743,16 +828,18 @@ func (f *formatter) annotationsDoc(a *syntax.Annotations, isLast bool) doc.Doc { open, close := a.TokStart(), a.TokEnd() if len(a.Items) == 0 { - closeDoc := f.emitTokens(close, close, emitOpts{leading: true, trailing: !isLast}) - if f.lineAfter(open) || len(f.token(close).Leading) > 0 { - closeDoc = doc.Concat{doc.HardLine, closeDoc} - } - - return doc.Concat{ + out := doc.Concat{ doc.Text(" "), f.emitTokens(open, open, emitOpts{leading: true, trailing: true}), - closeDoc, + f.emitTokens(close, close, emitOpts{leading: true}), + } + if !isLast { + // Same-line comments after the close render at the group + // boundary, outside it. + out = doc.Concat{out, doc.Concat(f.sameLineComments(close))} } + + return out } all := emitOpts{leading: true, trailing: true} @@ -790,10 +877,17 @@ func (f *formatter) annotationsDoc(a *syntax.Annotations, isLast bool) doc.Doc { f.emitTokens(open, open, all), doc.Indent(doc.Concat{f.foldBreak(open, ""), doc.Concat(middle)}), f.foldBreak(last, ""), - f.emitTokens(close, close, emitOpts{leading: true, trailing: !isLast}), + f.emitTokens(close, close, emitOpts{leading: true}), }) - return doc.Concat{doc.Text(" "), group} + out := doc.Concat{doc.Text(" "), group} + if !isLast { + // Same-line comments after the close render at the group boundary, + // outside the group, so the group folds independently. + out = doc.Concat{out, doc.Concat(f.sameLineComments(close))} + } + + return out } // trimComment returns the comment text without trailing whitespace, which diff --git a/formatter/format_fuzz_test.go b/formatter/format_fuzz_test.go index 45fa957..ef9036d 100644 --- a/formatter/format_fuzz_test.go +++ b/formatter/format_fuzz_test.go @@ -164,13 +164,11 @@ func commentTexts(src string) []string { var texts []string for _, tok := range toks { - for _, tr := range tok.Leading { - texts = append(texts, strings.TrimRight(tr.Text, " \t")) + if !syntax.IsComment(tok.Kind) { + continue } - for _, tr := range tok.Trailing { - texts = append(texts, strings.TrimRight(tr.Text, " \t")) - } + texts = append(texts, strings.TrimRight(tok.Text, " \t")) } return texts diff --git a/formatter/format_test.go b/formatter/format_test.go index 38cc1e6..7a91ce1 100644 --- a/formatter/format_test.go +++ b/formatter/format_test.go @@ -1314,3 +1314,96 @@ func TestFormatPreserveSeparators(t *testing.T) { }) } } + +// TestFormatConstsOptions exercises the per-construct separator and break +// options for list and map constants. +func TestFormatConstsOptions(t *testing.T) { + opts := func(mut func(*Options)) Options { + o := testOpts(80) + mut(&o) + + return o + } + + tests := []struct { + name string + src string + opts Options + want string + }{ + { + name: "lists forced comma with break", + src: "const list a = [1, 2]", + opts: opts(func(o *Options) { + o.Separator.Set(ConstructList, SeparatorComma) + o.Break.Set(ConstructList, true) + }), + want: "const list a = [\n 1,\n 2,\n]\n", + }, + { + name: "lists forced semicolon flat", + src: "const list a = [1, 2]", + opts: opts(func(o *Options) { + o.Separator.Set(ConstructList, SeparatorSemicolon) + }), + want: "const list a = [1; 2; ]\n", + }, + { + name: "lists none drops separators", + src: "const list a = [1, 2]", + opts: opts(func(o *Options) { + o.Separator.Set(ConstructList, SeparatorNone) + }), + want: "const list a = [1 2]\n", + }, + { + name: "maps forced comma with break", + src: "const map m = {\"a\": 1, \"b\": 2}", + opts: opts(func(o *Options) { + o.Separator.Set(ConstructMap, SeparatorComma) + o.Break.Set(ConstructMap, true) + }), + want: "const map m = {\n \"a\": 1,\n \"b\": 2,\n}\n", + }, + { + name: "trailing separator never leaves a blank before the close", + src: "const list a = [\n 1,\n 2,\n]", + opts: opts(func(o *Options) { + o.Separator.Set(ConstructList, SeparatorComma) + o.Break.Set(ConstructList, true) + }), + want: "const list a = [\n 1,\n 2,\n]\n", + }, + { + name: "line comment after trailing separator owns its line end", + src: "const list a = [1; #0\n]", + opts: opts(func(o *Options) { + o.Separator.Set(ConstructList, SeparatorPreserve) + o.Break.Set(ConstructList, true) + }), + want: "const list a = [\n 1; #0\n]\n", + }, + { + name: "suppressed separator keeps the comment inline", + src: "const list a = [1, // c\n2]", + opts: opts(func(o *Options) { + o.Separator.Set(ConstructList, SeparatorNone) + }), + want: "const list a = [\n 1 // c\n 2\n]\n", + }, + { + name: "nested containers stay flat with a comment at the item boundary", + src: "const list a = [[0]#\n]", + opts: opts(func(o *Options) { + o.Separator.Set(ConstructList, SeparatorNone) + }), + want: "const list a = [\n [0] #\n]\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runCase(t, tt.src, tt.opts, tt.want) + }) + } +} diff --git a/formatter/testdata/fuzz/FuzzFormat/04948906118ce669 b/formatter/testdata/fuzz/FuzzFormat/04948906118ce669 new file mode 100644 index 0000000..afc0626 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/04948906118ce669 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("enum A{A0=02\n#0\n,A10=00}#0010") diff --git a/formatter/testdata/fuzz/FuzzFormat/082e5b151923cb60 b/formatter/testdata/fuzz/FuzzFormat/082e5b151923cb60 new file mode 100644 index 0000000..94c4f9c --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/082e5b151923cb60 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const listA=[0#\n,#\n7]") diff --git a/formatter/testdata/fuzz/FuzzFormat/1afcaa1f5ba1648d b/formatter/testdata/fuzz/FuzzFormat/1afcaa1f5ba1648d new file mode 100644 index 0000000..d67da9e --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/1afcaa1f5ba1648d @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const X0 A0=[0, #0\n]") diff --git a/formatter/testdata/fuzz/FuzzFormat/2e2cd4573f6256d2 b/formatter/testdata/fuzz/FuzzFormat/2e2cd4573f6256d2 new file mode 100644 index 0000000..cac28c0 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/2e2cd4573f6256d2 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const listA=[0#0\n]") diff --git a/formatter/testdata/fuzz/FuzzFormat/33d2264098cb5887 b/formatter/testdata/fuzz/FuzzFormat/33d2264098cb5887 new file mode 100644 index 0000000..3b320cb --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/33d2264098cb5887 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("enum A02{A A #\nA0=0}") diff --git a/formatter/testdata/fuzz/FuzzFormat/33f351692615c4b5 b/formatter/testdata/fuzz/FuzzFormat/33f351692615c4b5 new file mode 100644 index 0000000..d0eac97 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/33f351692615c4b5 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const listA=[0#\n,#\n0]") diff --git a/formatter/testdata/fuzz/FuzzFormat/5ca7b7c46d86669a b/formatter/testdata/fuzz/FuzzFormat/5ca7b7c46d86669a new file mode 100644 index 0000000..321b96b --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/5ca7b7c46d86669a @@ -0,0 +1,2 @@ +go test fuzz v1 +string("service A{A A(A A()\n#\n,)}") diff --git a/formatter/testdata/fuzz/FuzzFormat/63964b8f79b0f0af b/formatter/testdata/fuzz/FuzzFormat/63964b8f79b0f0af new file mode 100644 index 0000000..38bb760 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/63964b8f79b0f0af @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const A A={0:0,0:0}") diff --git a/formatter/testdata/fuzz/FuzzFormat/6a94fa347d81695d b/formatter/testdata/fuzz/FuzzFormat/6a94fa347d81695d new file mode 100644 index 0000000..37edc7a --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/6a94fa347d81695d @@ -0,0 +1,2 @@ +go test fuzz v1 +string("#\n\n#") diff --git a/formatter/testdata/fuzz/FuzzFormat/892461e84151ab69 b/formatter/testdata/fuzz/FuzzFormat/892461e84151ab69 new file mode 100644 index 0000000..6c6e988 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/892461e84151ab69 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const A A=[0#\n,]") diff --git a/formatter/testdata/fuzz/FuzzFormat/8eb831fbb94050a2 b/formatter/testdata/fuzz/FuzzFormat/8eb831fbb94050a2 new file mode 100644 index 0000000..e342846 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/8eb831fbb94050a2 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("enum A{A00=000A=000}#") diff --git a/formatter/testdata/fuzz/FuzzFormat/a11ff1828859b41a b/formatter/testdata/fuzz/FuzzFormat/a11ff1828859b41a new file mode 100644 index 0000000..0497440 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/a11ff1828859b41a @@ -0,0 +1,2 @@ +go test fuzz v1 +string("service X{A A(A A#\n,X000 A20)}") diff --git a/formatter/testdata/fuzz/FuzzFormat/a31d8aa66e9cb835 b/formatter/testdata/fuzz/FuzzFormat/a31d8aa66e9cb835 new file mode 100644 index 0000000..29a0fbf --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/a31d8aa66e9cb835 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const A A0= [0] #02") diff --git a/formatter/testdata/fuzz/FuzzFormat/a5fefb71b3af4cca b/formatter/testdata/fuzz/FuzzFormat/a5fefb71b3af4cca new file mode 100644 index 0000000..d453989 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/a5fefb71b3af4cca @@ -0,0 +1,2 @@ +go test fuzz v1 +string("enum A{A\n=01#0\nA0=00}") diff --git a/formatter/testdata/fuzz/FuzzFormat/b2efa571bc1b072c b/formatter/testdata/fuzz/FuzzFormat/b2efa571bc1b072c new file mode 100644 index 0000000..d3fd90a --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/b2efa571bc1b072c @@ -0,0 +1,2 @@ +go test fuzz v1 +string("enum A1{A#\n,A0=0}") diff --git a/formatter/testdata/fuzz/FuzzFormat/bc34ddc23b78af85 b/formatter/testdata/fuzz/FuzzFormat/bc34ddc23b78af85 new file mode 100644 index 0000000..700b457 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/bc34ddc23b78af85 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const A A=[0#\n,00070]") diff --git a/formatter/testdata/fuzz/FuzzFormat/c2ae9c489afb18bd b/formatter/testdata/fuzz/FuzzFormat/c2ae9c489afb18bd new file mode 100644 index 0000000..ddccb7d --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/c2ae9c489afb18bd @@ -0,0 +1,2 @@ +go test fuzz v1 +string("service A{A A(A A(A)#\n,)}") diff --git a/formatter/testdata/fuzz/FuzzFormat/c2c6fe67031e7e51 b/formatter/testdata/fuzz/FuzzFormat/c2c6fe67031e7e51 new file mode 100644 index 0000000..8e02da4 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/c2c6fe67031e7e51 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("enum A02{A11 A,#\nA0=0}") diff --git a/formatter/testdata/fuzz/FuzzFormat/c54228ba820055dc b/formatter/testdata/fuzz/FuzzFormat/c54228ba820055dc new file mode 100644 index 0000000..222f00e --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/c54228ba820055dc @@ -0,0 +1,2 @@ +go test fuzz v1 +string("service X{A A(A A#\n, #\n)}") diff --git a/formatter/testdata/fuzz/FuzzFormat/c87d6b6ae9b22e44 b/formatter/testdata/fuzz/FuzzFormat/c87d6b6ae9b22e44 new file mode 100644 index 0000000..dbed21d --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/c87d6b6ae9b22e44 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const A A=[[0];#\n]") diff --git a/formatter/testdata/fuzz/FuzzFormat/cf04434f5850d26c b/formatter/testdata/fuzz/FuzzFormat/cf04434f5850d26c new file mode 100644 index 0000000..57df437 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/cf04434f5850d26c @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const A A={0:0,}") diff --git a/formatter/testdata/fuzz/FuzzFormat/d75a409f77d8f20c b/formatter/testdata/fuzz/FuzzFormat/d75a409f77d8f20c new file mode 100644 index 0000000..03e2939 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/d75a409f77d8f20c @@ -0,0 +1,2 @@ +go test fuzz v1 +string("enum A{A=00\n,#\nA0=0}") diff --git a/formatter/testdata/fuzz/FuzzFormat/d8e6b3568a917c3a b/formatter/testdata/fuzz/FuzzFormat/d8e6b3568a917c3a new file mode 100644 index 0000000..8a1e8f6 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/d8e6b3568a917c3a @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const A A=[0\r#\n,]") diff --git a/formatter/testdata/fuzz/FuzzFormat/dbd723d2d105484d b/formatter/testdata/fuzz/FuzzFormat/dbd723d2d105484d new file mode 100644 index 0000000..7cc1d5e --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/dbd723d2d105484d @@ -0,0 +1,2 @@ +go test fuzz v1 +string("enum A{A=0\n,#01\nA0=0}") diff --git a/formatter/testdata/fuzz/FuzzFormat/dc417af495bda113 b/formatter/testdata/fuzz/FuzzFormat/dc417af495bda113 new file mode 100644 index 0000000..2f2a1c7 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/dc417af495bda113 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const listA=[00\n,#\n[0002]0070]") diff --git a/formatter/testdata/fuzz/FuzzFormat/edaef5a83157ec94 b/formatter/testdata/fuzz/FuzzFormat/edaef5a83157ec94 new file mode 100644 index 0000000..d5d0bf7 --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/edaef5a83157ec94 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const listA=[00\n,#\n0]") diff --git a/formatter/testdata/fuzz/FuzzFormat/f8332e99af6f5f24 b/formatter/testdata/fuzz/FuzzFormat/f8332e99af6f5f24 new file mode 100644 index 0000000..a427a9c --- /dev/null +++ b/formatter/testdata/fuzz/FuzzFormat/f8332e99af6f5f24 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("const A0 A=[000#\n,]") diff --git a/formatter/value.go b/formatter/value.go index e3922d9..4551f54 100644 --- a/formatter/value.go +++ b/formatter/value.go @@ -28,40 +28,70 @@ func (f *formatter) constValue(v *syntax.ConstValue, isLast bool) doc.Doc { case syntax.ValueMap: return f.constMap(v, isLast) default: - o := emitOpts{leading: true} - if !isLast { - o.trailing = true - } - - return f.emitTokens(v.TokStart(), v.TokEnd(), o) + // Comments before and after the value belong to the enclosing + // structure (constant's or constItems' boundaries). + return f.emitTokens(v.TokStart(), v.TokEnd(), emitOpts{}) } } -// constList formats "[ items ]" as a foldable group. +// constList formats "[ items ]" as a foldable group honoring the lists +// separator and break options. func (f *formatter) constList(v *syntax.ConstValue, isLast bool) doc.Doc { - open, close := v.TokStart(), v.TokEnd() + items := make([]constItem, len(v.List)) + for i, item := range v.List { + items[i] = constItem{start: item.TokStart(), end: item.TokEnd(), doc: f.constValue(item, false)} + } - all := emitOpts{leading: true, trailing: true} - if len(v.List) == 0 { - closeDoc := f.emitTokens(close, close, emitOpts{leading: true, trailing: !isLast}) - if f.lineAfter(open) || len(f.token(close).Leading) > 0 { - closeDoc = doc.Concat{doc.HardLine, closeDoc} + return f.constItems(items, v.TokStart(), v.TokEnd(), ConstructList, isLast) +} + +// constMap formats "{ key: value, ... }" as a foldable group honoring the +// maps separator and break options. +func (f *formatter) constMap(v *syntax.ConstValue, isLast bool) doc.Doc { + items := make([]constItem, len(v.Map)) + for i, entry := range v.Map { + items[i] = constItem{ + start: entry.Key.TokStart(), + end: entry.Value.TokEnd(), + doc: f.emitTokens(entry.Key.TokStart(), entry.Value.TokEnd(), emitOpts{}), } + } + + return f.constItems(items, v.TokStart(), v.TokEnd(), ConstructMap, isLast) +} + +// constItem is one list/map entry: its formatted doc and the token span of +// its last token (the separator, if any, follows it). +type constItem struct { + doc doc.Doc + start int + end int +} + +// constItems is the shared list/map body: a foldable group with one entry +// per line when broken, honoring the construct's separator and break +// options. The separator between entries and the trailing separator follow +// the mode; the closing bracket gets exactly one break, so a trailing +// separator never leaves a blank line before it. +func (f *formatter) constItems(items []constItem, open, close int, c Construct, isLast bool) doc.Doc { + sepMode := f.opts.Separator.Get(c) + openOpts := emitOpts{trailing: true} + if len(items) == 0 { return doc.Concat{ - f.emitTokens(open, open, all), - closeDoc, + f.emitTokens(open, open, openOpts), + f.emitTokens(close, close, emitOpts{leading: true}), } } - middle := make([]doc.Doc, 0, len(v.List)*2) - last := open + middle := make([]doc.Doc, 0, len(items)*2) - for i, item := range v.List { + for i, item := range items { if i > 0 { - prevEnd := v.List[i-1].TokEnd() - if isListSep(f.token(prevEnd + 1).Kind) { - middle = append(middle, f.commaSep(prevEnd+1)...) + prevEnd := items[i-1].end + sepIdx := f.nextReal(prevEnd + 1) + if isListSep(f.token(sepIdx).Kind) { + middle = append(middle, f.itemSep(sepIdx, sepMode)...) } else { // Lenient sources may omit separators; keep the items // apart so their tokens cannot merge. @@ -69,77 +99,160 @@ func (f *formatter) constList(v *syntax.ConstValue, isLast bool) doc.Doc { } } - middle = append(middle, f.constValue(item, false)) - last = item.TokEnd() - } - // A trailing comma after the last item (which may carry comments) is - // not between two items, so it is emitted here. - if isListSep(f.token(last + 1).Kind) { - middle = append(middle, f.commaSep(last+1)...) - last++ + // Comments before and after the item render at the item boundary, + // outside the item's own group, so a nested container stays flat. + middle = append(middle, f.ownLineComments(item.start)...) + middle = append(middle, item.doc) + middle = append(middle, f.sameLineComments(item.end)...) } + // The trailing separator: the mode's text (or nothing), with the + // source separator token's comments preserved. Its gap is flat: the + // close break below is the only line before the closing bracket. + trailing := f.trailingItemSep(items[len(items)-1].end, sepMode) + closeOpts := emitOpts{leading: true} - if !isLast { - closeOpts.trailing = true - } - return doc.Group(doc.Concat{ - f.emitTokens(open, open, all), + // The single break before the close. A line comment owns its line end + // (HardLine); the printer collapses a following structural line, so + // the SoftLine never leaves a blank line. + closeBreak := doc.IfBreak(doc.SoftLine, doc.Text("")) + + inner := doc.Concat{ + f.emitTokens(open, open, openOpts), doc.Indent(doc.Concat{f.foldBreak(open, ""), doc.Concat(middle)}), - f.foldBreak(last, ""), + trailing, + closeBreak, f.emitTokens(close, close, closeOpts), - }) + } + if f.opts.Break.Get(c) || sepForcesBreakList(f.sepsOf(items), sepMode) { + // BreakParent inside the group forces it to the broken layout. + inner = doc.Concat{doc.BreakParent, inner} + } + + return doc.Group(inner) } -// constMap formats "{ key: value, ... }" as a foldable group. -func (f *formatter) constMap(v *syntax.ConstValue, isLast bool) doc.Doc { - open, close := v.TokStart(), v.TokEnd() +// sepForcesBreakList reports whether a preserved separator mix forces the +// broken layout: items separated and unseparated inconsistently look broken +// on a flat line. Unlike fields, list separators sit between items only, so +// a single separator never forces a break. +func sepForcesBreakList(seps []syntax.TokenKind, mode SeparatorMode) bool { + if mode != SeparatorPreserve || len(seps) < 2 { + return false + } - all := emitOpts{leading: true, trailing: true} - if len(v.Map) == 0 { - closeDoc := f.emitTokens(close, close, emitOpts{leading: true, trailing: !isLast}) - if f.lineAfter(open) || len(f.token(close).Leading) > 0 { - closeDoc = doc.Concat{doc.HardLine, closeDoc} + want := seps[0] != 0 + for _, sep := range seps[1:] { + if (sep != 0) != want { + return true } + } - return doc.Concat{ - f.emitTokens(open, open, all), - closeDoc, - } + return false +} + +// itemSep renders the separator between two list/map items: the source +// separator token (with comments) when the mode preserves its text, the +// forced text with the source token's comments otherwise, and the foldable +// gap after it that lines up the next item. A line comment owns its line +// end (HardLine), so the separator lands on the next line by construction. +func (f *formatter) itemSep(sep int, mode SeparatorMode) []doc.Doc { + text := f.token(sep).Text + switch mode { + case SeparatorComma: + text = "," + case SeparatorSemicolon: + text = ";" + case SeparatorNone: + text = "" } - middle := make([]doc.Doc, 0, len(v.Map)*2) - last := open + if text == f.token(sep).Text { + return []doc.Doc{f.emitTokens(sep, sep, emitOpts{leading: true, trailing: true}), f.foldBreak(sep, " ")} + } - for i, entry := range v.Map { - if i > 0 { - prevEnd := v.Map[i-1].Value.TokEnd() - if isListSep(f.token(prevEnd + 1).Kind) { - middle = append(middle, f.commaSep(prevEnd+1)...) - } else { - middle = append(middle, f.foldBreak(prevEnd, " ")) - } + // Forced separator differing from the source: the forced text replaces + // the suppressed text inside the run, so the source token's comments + // stay ordered around it — own-line comments before it, same-line + // comments after. + return []doc.Doc{ + f.emitTokens(sep, sep, emitOpts{leading: true, trailing: true, skipText: []int{sep}, text: text}), + f.foldBreak(sep, " "), + } +} + +// trailingItemSep is the trailing separator of a list/map: the source +// separator token (with comments) when present and the mode preserves its +// text, the forced text otherwise, nothing under SeparatorNone. Its gap is +// a flat space — the close break provides the line before the bracket. A +// line comment owns its line end, so the separator lands on the next line +// by construction. +func (f *formatter) trailingItemSep(last int, mode SeparatorMode) doc.Doc { + sep := f.nextReal(last + 1) + hasSep := isListSep(f.token(sep).Kind) + + text := "" + if hasSep { + text = f.token(sep).Text + } + switch mode { + case SeparatorComma: + text = "," + case SeparatorSemicolon: + text = ";" + case SeparatorNone: + text = "" + } + + if !hasSep && text == "" { + return doc.Concat{} + } + + if hasSep && text == f.token(sep).Text { + // Preserve the source separator with its comments; the flat gap + // before the closing bracket. A trailing line comment owns its + // line end instead. + sepDoc := f.emitTokens(sep, sep, emitOpts{leading: true, trailing: true}) + if !f.sameLineEndsLine(sep) { + sepDoc = doc.Concat{sepDoc, doc.Text(" ")} } - middle = append(middle, f.emitTokens(entry.Key.TokStart(), entry.Value.TokEnd(), all)) - last = entry.Value.TokEnd() + return sepDoc } - if isListSep(f.token(last + 1).Kind) { - middle = append(middle, f.commaSep(last+1)...) - last++ + if !hasSep { + // No source separator: the forced text and the flat gap before + // the closing bracket. + return doc.Concat{doc.Text(text), doc.Text(" ")} } - closeOpts := emitOpts{leading: true} - if !isLast { - closeOpts.trailing = true + // Forced text (or dropping the source separator under SeparatorNone): + // the forced text replaces the suppressed text inside the run, so the + // source token's comments stay ordered around it. The flat gap belongs + // to the forced text; a dropped separator leaves no gap, so the output + // is stable across a reparse. + sepDoc := f.emitTokens(sep, sep, emitOpts{leading: true, trailing: true, skipText: []int{sep}, text: text}) + if text != "" && !f.sameLineEndsLine(sep) { + sepDoc = doc.Concat{sepDoc, doc.Text(" ")} } - return doc.Group(doc.Concat{ - f.emitTokens(open, open, all), - doc.Indent(doc.Concat{f.foldBreak(open, ""), doc.Concat(middle)}), - f.foldBreak(last, ""), - f.emitTokens(close, close, closeOpts), - }) + return sepDoc +} + +// sepsOf returns the separator kinds between the items, in order (0 when +// an item boundary has no separator token). +func (f *formatter) sepsOf(items []constItem) []syntax.TokenKind { + seps := make([]syntax.TokenKind, 0, len(items)-1) + + for i := 1; i < len(items); i++ { + prevEnd := items[i-1].end + if isListSep(f.token(prevEnd + 1).Kind) { + seps = append(seps, f.token(prevEnd+1).Kind) + } else { + seps = append(seps, 0) + } + } + + return seps } diff --git a/main.go b/main.go index 775826b..b748bb5 100644 --- a/main.go +++ b/main.go @@ -103,6 +103,8 @@ var constructFlags = []struct { {"enum", formatter.ConstructEnum}, {"argument", formatter.ConstructArguments}, {"throws", formatter.ConstructThrows}, + {"list", formatter.ConstructList}, + {"map", formatter.ConstructMap}, } // formatFlags are the flags of the format subcommand. diff --git a/options/options.go b/options/options.go index 772a809..759819a 100644 --- a/options/options.go +++ b/options/options.go @@ -25,98 +25,14 @@ import ( const ConfigFileName = "thriftls.json" // Separators configures trailing separators per construct. A nil value is -// unset. -type Separators struct { - Structs *string `json:"structs"` - Unions *string `json:"unions"` - Exceptions *string `json:"exceptions"` - Enums *string `json:"enums"` - Arguments *string `json:"arguments"` - Throws *string `json:"throws"` -} - -// Get returns the value for the construct. -func (s Separators) Get(c formatter.Construct) *string { - switch c { - case formatter.ConstructUnion: - return s.Unions - case formatter.ConstructException: - return s.Exceptions - case formatter.ConstructEnum: - return s.Enums - case formatter.ConstructArguments: - return s.Arguments - case formatter.ConstructThrows: - return s.Throws - } - - return s.Structs -} - -// Set assigns the value for the construct. -func (s *Separators) Set(c formatter.Construct, v *string) { - switch c { - case formatter.ConstructUnion: - s.Unions = v - case formatter.ConstructException: - s.Exceptions = v - case formatter.ConstructEnum: - s.Enums = v - case formatter.ConstructArguments: - s.Arguments = v - case formatter.ConstructThrows: - s.Throws = v - default: - s.Structs = v - } -} +// unset. It is an alias of the formatter's per-construct container, so +// adding a construct adds the config key, CLI flags, and validation in one +// place. +type Separators = formatter.PerConstruct[*string] // Break configures layouts that are forced multiline per construct. A nil // value is unset. -type Break struct { - Structs *bool `json:"structs"` - Unions *bool `json:"unions"` - Exceptions *bool `json:"exceptions"` - Enums *bool `json:"enums"` - Arguments *bool `json:"arguments"` - Throws *bool `json:"throws"` -} - -// Get returns the value for the construct. -func (b Break) Get(c formatter.Construct) *bool { - switch c { - case formatter.ConstructUnion: - return b.Unions - case formatter.ConstructException: - return b.Exceptions - case formatter.ConstructEnum: - return b.Enums - case formatter.ConstructArguments: - return b.Arguments - case formatter.ConstructThrows: - return b.Throws - } - - return b.Structs -} - -// Set assigns the value for the construct. -func (b *Break) Set(c formatter.Construct, v *bool) { - switch c { - case formatter.ConstructUnion: - b.Unions = v - case formatter.ConstructException: - b.Exceptions = v - case formatter.ConstructEnum: - b.Enums = v - case formatter.ConstructArguments: - b.Arguments = v - case formatter.ConstructThrows: - b.Throws = v - default: - b.Structs = v - } -} +type Break = formatter.PerConstruct[*bool] // Patch is a partial set of options; nil fields are unset. type Patch struct { diff --git a/options/options_test.go b/options/options_test.go index 8426198..411d6ef 100644 --- a/options/options_test.go +++ b/options/options_test.go @@ -271,7 +271,11 @@ func TestPatchSeparatorModes(t *testing.T) { } for _, tt := range tests { t.Run(tt.value, func(t *testing.T) { - p := Patch{Separators: &Separators{Structs: &tt.value, Unions: &tt.value, Exceptions: &tt.value, Enums: &tt.value, Arguments: &tt.value, Throws: &tt.value}} + p := Patch{Separators: &Separators{ + Structs: &tt.value, Unions: &tt.value, Exceptions: &tt.value, + Enums: &tt.value, Arguments: &tt.value, Throws: &tt.value, + Lists: &tt.value, Maps: &tt.value, + }} o, err := p.Formatter() if err != nil { diff --git a/syntax/dump.go b/syntax/dump.go index 759eec8..c99a580 100644 --- a/syntax/dump.go +++ b/syntax/dump.go @@ -5,8 +5,8 @@ import ( "strings" ) -// Dump renders a parsed document as a debug tree: every token with its -// kind, position, blank-line count, and attached trivia, followed by the +// Dump renders a parsed document as a debug tree: every token (comments +// included) with its kind, position, and blank-line count, followed by the // node spans. Deterministic and stable for a given input, so dumps can be // diffed across versions. func Dump(d *Document) string { @@ -14,14 +14,6 @@ func Dump(d *Document) string { for i, tok := range d.Tokens { fmt.Fprintf(&b, "tok %3d %-16s line=%-3d col=%-3d blb=%d %q\n", i, tok.Kind, tok.Line, tok.Col, tok.BlankLinesBefore, tok.Text) - - for _, tr := range tok.Leading { - fmt.Fprintf(&b, " leading %-18s %q\n", tr.Kind, tr.Text) - } - - for _, tr := range tok.Trailing { - fmt.Fprintf(&b, " trailing %-18s %q\n", tr.Kind, tr.Text) - } } for i, n := range d.Nodes { diff --git a/syntax/lexer.go b/syntax/lexer.go index 605af0a..014e9f7 100644 --- a/syntax/lexer.go +++ b/syntax/lexer.go @@ -4,10 +4,10 @@ // Grammar reference: the Apache Thrift compiler // (compiler/cpp/src/thrift/thriftl.ll and thrifty.yy). // -// Losslessness: comments are preserved as trivia attached to tokens in +// Losslessness: comments are first-class tokens in the token stream, in // source order. Whitespace itself is not preserved; instead each token -// records how many blank lines preceded it, which is the only layout -// information a formatter is allowed to act on. +// (comments included) records how many blank lines preceded it, which is +// the only layout information a formatter is allowed to act on. package syntax import ( @@ -78,6 +78,13 @@ const ( TokenEqual TokenStar TokenAmp + + // Comment trivia. A line comment or annotation consumes the rest of + // its source line, so whatever follows always starts a fresh line. + TokenLineComment + TokenBlockComment + TokenDocComment + TokenAnnotation ) var keywordKinds = map[string]TokenKind{ @@ -141,6 +148,8 @@ var tokenKindNames = map[TokenKind]string{ TokenLBracket: "[", TokenRBracket: "]", TokenLt: "<", TokenGt: ">", TokenComma: ",", TokenSemicolon: ";", TokenColon: ":", TokenEqual: "=", TokenStar: "*", TokenAmp: "&", + TokenLineComment: "line comment", TokenBlockComment: "block comment", + TokenDocComment: "doc comment", TokenAnnotation: "annotation", } func (k TokenKind) String() string { @@ -155,47 +164,7 @@ func (k TokenKind) String() string { return fmt.Sprintf("TokenKind(%d)", uint8(k)) } -// TriviaKind identifies the kind of a comment trivia. -type TriviaKind uint8 - -const ( - TriviaLineComment TriviaKind = iota // // or # - TriviaBlockComment // /* */ - TriviaDocComment // /** */ - TriviaAnnotation // @name{...} to end of line -) - -func (k TriviaKind) String() string { - switch k { - case TriviaLineComment: - return "line comment" - case TriviaBlockComment: - return "block comment" - case TriviaDocComment: - return "doc comment" - case TriviaAnnotation: - return "annotation" - } - - return fmt.Sprintf("TriviaKind(%d)", uint8(k)) -} - -// Trivia is a comment preserved in source order. It is attached to a Token -// either as leading (before the token) or trailing (after the previous token -// on the same line). -type Trivia struct { - Kind TriviaKind - Text string // exact source text, including comment delimiters - Offset int // byte offset of the first character - Line int // 1-based line of the first character - Col int // 1-based rune column of the first character - - // BlankLinesBefore is the number of empty lines between the previous - // token (or trivia) and this one, within the enclosing gap. - BlankLinesBefore int -} - -// Token is a single lexical token with its attached comment trivia. +// Token is a single lexical token, comments included. type Token struct { Kind TokenKind Text string // exact source text @@ -204,17 +173,23 @@ type Token struct { Line int // 1-based line of the first character Col int // 1-based rune column of the first character - // Leading holds comments that appear between the previous token and - // this one, on their own lines. - Leading []Trivia - // Trailing holds comments that appear after this token on the same - // line. - Trailing []Trivia // BlankLinesBefore is the number of empty lines between the previous - // token and this one (0 means no blank line). + // stream entry (token or comment) and this one. BlankLinesBefore int } +// IsComment reports whether the token kind is a comment trivia token. The +// parser skips comment tokens when matching the grammar, so they only +// appear between real tokens in the stream. +func IsComment(k TokenKind) bool { + switch k { + case TokenLineComment, TokenBlockComment, TokenDocComment, TokenAnnotation: + return true + } + + return false +} + // Severity classifies a syntax error or warning. type Severity uint8 @@ -266,63 +241,57 @@ func (l *lexer) run() ([]Token, []Error) { tokens := make([]Token, 0, len(l.src)/6+8) for { - prevLine := -1 - if n := len(tokens); n > 0 { - prevLine = tokens[n-1].Line - } - - leading, trailing, blankLines := l.scanTrivia(prevLine) + blankLines, comments := l.scanTrivia() + tokens = append(tokens, comments...) tok := l.scanToken() - tok.Leading = leading tok.BlankLinesBefore = blankLines tokens = append(tokens, tok) - if len(tokens) > 1 { - tokens[len(tokens)-2].Trailing = trailing - } - if tok.Kind == TokenEOF { return tokens, l.errs } } } -// scanTrivia consumes whitespace and comments between the previous token and -// the next one. Comments starting on the previous token's line become its -// trailing trivia; everything else becomes leading trivia of the next token. -// The returned blankLines count empty lines in the gap. -func (l *lexer) scanTrivia(prevLine int) (leading, trailing []Trivia, blankLines int) { +// scanTrivia consumes whitespace and comments between the previous stream +// entry and the next real token. Comments are returned as tokens in source +// order with their own blank-line counts; the returned blankLines count the +// empty lines immediately before the next real token. +func (l *lexer) scanTrivia() (blankLines int, comments []Token) { for l.off < len(l.src) { switch c := l.src[l.off]; { case isWhitespace(c): blankLines += l.scanWhitespace() case c == '/' && l.peekByte(1) == '/': - leading, trailing = l.appendComment(leading, trailing, prevLine, blankLines, l.scanLineComment()) + t := l.scanLineComment() + t.BlankLinesBefore = blankLines + blankLines = 0 + comments = append(comments, t) case c == '/' && l.peekByte(1) == '*': - leading, trailing = l.appendComment(leading, trailing, prevLine, blankLines, l.scanBlockComment()) + t := l.scanBlockComment() + t.BlankLinesBefore = blankLines + blankLines = 0 + comments = append(comments, t) case c == '#': - leading, trailing = l.appendComment(leading, trailing, prevLine, blankLines, l.scanLineComment()) + t := l.scanLineComment() + t.BlankLinesBefore = blankLines + blankLines = 0 + comments = append(comments, t) case c == '@': // Java-style annotations (@name{...}) are preserved as trivia, // like comments, so they round-trip without being part of the // grammar. - leading, trailing = l.appendComment(leading, trailing, prevLine, blankLines, l.scanLineAnnotation()) + t := l.scanLineAnnotation() + t.BlankLinesBefore = blankLines + blankLines = 0 + comments = append(comments, t) default: - return leading, trailing, blankLines + return blankLines, comments } } - return leading, trailing, blankLines -} - -func (l *lexer) appendComment(leading, trailing []Trivia, prevLine, blankLines int, t Trivia) ([]Trivia, []Trivia) { - t.BlankLinesBefore = blankLines - if t.Line == prevLine { - return leading, append(trailing, t) - } - - return append(leading, t), trailing + return blankLines, comments } // scanWhitespace consumes a whitespace run and returns the number of empty @@ -362,31 +331,31 @@ func (l *lexer) scanWhitespace() int { return 0 } -func (l *lexer) scanLineComment() Trivia { +func (l *lexer) scanLineComment() Token { start := l.pos() for l.off < len(l.src) && l.src[l.off] != '\n' && l.src[l.off] != '\r' { l.advanceRune() } - return l.finishTrivia(TriviaLineComment, start) + return l.finishTrivia(TokenLineComment, start) } // scanLineAnnotation scans an @annotation line: from '@' to the end of the // line, verbatim. Like line comments, the newline itself is left for the // whitespace scanner. -func (l *lexer) scanLineAnnotation() Trivia { +func (l *lexer) scanLineAnnotation() Token { start := l.pos() for l.off < len(l.src) && l.src[l.off] != '\n' && l.src[l.off] != '\r' { l.advanceRune() } - return l.finishTrivia(TriviaAnnotation, start) + return l.finishTrivia(TokenAnnotation, start) } // scanBlockComment scans a /* */ or /** */ comment. /** ... */ yields a doc -// comment trivia; everything else a block comment trivia. An unterminated +// comment token; everything else a block comment token. An unterminated // comment consumes the rest of the input and reports an error. -func (l *lexer) scanBlockComment() Trivia { +func (l *lexer) scanBlockComment() Token { start := l.pos() doc := l.peekByte(2) == '*' l.advanceByte() // / @@ -403,9 +372,9 @@ func (l *lexer) scanBlockComment() Trivia { l.advanceByte() l.advanceByte() - kind := TriviaBlockComment + kind := TokenBlockComment if doc { - kind = TriviaDocComment + kind = TokenDocComment } return l.finishTrivia(kind, start) @@ -414,21 +383,21 @@ func (l *lexer) scanBlockComment() Trivia { if doc && l.off == start.offset+3 && l.src[l.off] == '/' { l.advanceByte() - return l.finishTrivia(TriviaDocComment, start) + return l.finishTrivia(TokenDocComment, start) } l.advanceRune() } - return l.finishTrivia(TriviaBlockComment, start) + return l.finishTrivia(TokenBlockComment, start) } func (l *lexer) pos() srcPos { return srcPos{l.off, l.line, l.col} } -func (l *lexer) finishTrivia(kind TriviaKind, start srcPos) Trivia { - return Trivia{Kind: kind, Text: l.src[start.offset:l.off], Offset: start.offset, Line: start.line, Col: start.col} +func (l *lexer) finishTrivia(kind TokenKind, start srcPos) Token { + return Token{Kind: kind, Text: l.src[start.offset:l.off], Offset: start.offset, Line: start.line, Col: start.col} } // scanToken scans the next real token, skipping over invalid characters with diff --git a/syntax/lexer_fuzz_test.go b/syntax/lexer_fuzz_test.go index 006df41..fc12728 100644 --- a/syntax/lexer_fuzz_test.go +++ b/syntax/lexer_fuzz_test.go @@ -11,9 +11,9 @@ import ( // - lexing always terminates with an EOF token (no infinite loops) // - lexing never panics, even on truncated, binary, or invalid UTF-8 input // - lexing is deterministic -// - token and trivia texts are exact slices of the source +// - stream entries (tokens and comments) are exact, non-overlapping +// slices of the source, in source order // - reported line/col positions match the source bytes -// - trailing trivia always starts on its token's line func FuzzLex(f *testing.F) { for _, seed := range [][]byte{ []byte(""), @@ -64,7 +64,8 @@ func FuzzLex(f *testing.F) { checkPos(t, srcStr, err.Offset, err.Line, err.Col, "error") } - // Token and trivia invariants. + // Stream invariants: every entry is an exact slice of the source, + // and entries are in source order, non-overlapping. prevEnd := 0 for i, tok := range toks { @@ -72,8 +73,8 @@ func FuzzLex(f *testing.F) { t.Fatalf("token %d has invalid kind", i) } - if tok.Offset < 0 || tok.Offset+len(tok.Text) > len(src) { - t.Fatalf("token %d (%s) spans outside the source", i, tok.Kind) + if tok.Offset < prevEnd || tok.Offset+len(tok.Text) > len(src) { + t.Fatalf("token %d (%s) outside the source or overlapping the previous entry", i, tok.Kind) } if got := src[tok.Offset : tok.Offset+len(tok.Text)]; string(got) != tok.Text { @@ -86,34 +87,7 @@ func FuzzLex(f *testing.F) { t.Fatalf("token %d has negative BlankLinesBefore", i) } - // Leading trivia lies between the previous token and this one. - for _, tr := range tok.Leading { - if tr.Offset < prevEnd || tr.Offset+len(tr.Text) > tok.Offset || tr.Offset+len(tr.Text) > len(src) { - t.Fatalf("leading trivia %q of token %d outside the gap [%d, %d)", tr.Text, i, prevEnd, tok.Offset) - } - - checkPos(t, srcStr, tr.Offset, tr.Line, tr.Col, "leading trivia") - } - - for _, tr := range tok.Trailing { - if tr.Offset < tok.Offset || tr.Offset+len(tr.Text) > len(src) { - t.Fatalf("trailing trivia %q of token %d outside the source", tr.Text, i) - } - - checkPos(t, srcStr, tr.Offset, tr.Line, tr.Col, "trailing trivia") - - if tr.Line != tok.Line { - t.Fatalf("trailing trivia %q of token %d starts on line %d, token is on line %d", - tr.Text, i, tr.Line, tok.Line) - } - } - - if tok.Kind != TokenEOF { - prevEnd = tok.Offset + len(tok.Text) - for _, tr := range tok.Trailing { - prevEnd += len(tr.Text) - } - } + prevEnd = tok.Offset + len(tok.Text) } }) } diff --git a/syntax/lexer_test.go b/syntax/lexer_test.go index a26da00..267d4d4 100644 --- a/syntax/lexer_test.go +++ b/syntax/lexer_test.go @@ -206,7 +206,7 @@ func TestLexPositions(t *testing.T) { { "columns count runes", "// é\nstruct S {}", - []posSpec{{2, 1, 6}, {2, 8, 13}, {2, 10, 15}, {2, 11, 16}, {2, 12, 17}}, + []posSpec{{1, 1, 0}, {2, 1, 6}, {2, 8, 13}, {2, 10, 15}, {2, 11, 16}, {2, 12, 17}}, }, { "crlf line endings", @@ -245,8 +245,9 @@ func TestLexPositions(t *testing.T) { type triviaCheck struct { idx int - leading []string // comment texts, in order - trailing []string + text string // expected text of the stream entry + kind TokenKind + sameLine bool // comments only: shares the previous token's line blankLinesBefore int } @@ -257,122 +258,123 @@ func TestLexTrivia(t *testing.T) { checks []triviaCheck }{ { - "same-line comment is trailing", + "same-line comment is a stream token on the token's line", "i32 x // c\ni32 y", []triviaCheck{ - {idx: 1, trailing: []string{"// c"}}, + {idx: 2, text: "// c", kind: TokenLineComment, sameLine: true}, }, }, { - "own-line comment is leading", + "own-line comment is a stream token on its own line", "// c\ni32 x", []triviaCheck{ - {idx: 0, leading: []string{"// c"}}, + {idx: 0, text: "// c", kind: TokenLineComment, sameLine: false}, }, }, { - "comment between tokens on own line is leading of next", + "comment between tokens on own line", "i32 x\n// c\ni32 y", []triviaCheck{ - {idx: 2, leading: []string{"// c"}}, + {idx: 2, text: "// c", kind: TokenLineComment, sameLine: false}, }, }, { - "block comment on token line is trailing", + "block comment on token line", "i32 x /* c */ i32 y", []triviaCheck{ - {idx: 1, trailing: []string{"/* c */"}}, + {idx: 2, text: "/* c */", kind: TokenBlockComment, sameLine: true}, }, }, { "hash comment is a line comment", "i32 x # c\ni32 y", []triviaCheck{ - {idx: 1, trailing: []string{"# c"}}, + {idx: 2, text: "# c", kind: TokenLineComment, sameLine: true}, }, }, { - "multiple trailing comments keep order", + "multiple same-line comments keep order", "i32 x /* a */ // b\ni32 y", []triviaCheck{ - {idx: 1, trailing: []string{"/* a */", "// b"}}, + {idx: 2, text: "/* a */", kind: TokenBlockComment, sameLine: true}, + {idx: 3, text: "// b", kind: TokenLineComment, sameLine: true}, }, }, { - "annotation is leading trivia of the next declaration", + "annotation is a stream token before the declaration", "@naming.PreviouslyKnownAs{'namespace_': 'x'}\nservice Foo {}", []triviaCheck{ - {idx: 0, leading: []string{"@naming.PreviouslyKnownAs{'namespace_': 'x'}"}}, + {idx: 0, text: "@naming.PreviouslyKnownAs{'namespace_': 'x'}", kind: TokenAnnotation, sameLine: false}, }, }, { - "annotation inside a struct body attaches to the closing brace", + "annotation inside a struct body precedes the closing brace", "struct S {\n 1: string x\n @weird\n}", []triviaCheck{ - {idx: 7, leading: []string{"@weird"}}, + {idx: 7, text: "@weird", kind: TokenAnnotation, sameLine: false}, }, }, { "doc comment kind", "/** doc */\nstruct S {}", []triviaCheck{ - {idx: 0, leading: []string{"/** doc */"}}, + {idx: 0, text: "/** doc */", kind: TokenDocComment, sameLine: false}, }, }, { "empty doc comment", "/**/\nstruct S {}", []triviaCheck{ - {idx: 0, leading: []string{"/**/"}}, + {idx: 0, text: "/**/", kind: TokenDocComment, sameLine: false}, }, }, { - "comments at end of file attach to eof", + "comments at end of file precede eof", "i32 x\n// tail", []triviaCheck{ - {idx: 2, leading: []string{"// tail"}}, + {idx: 2, text: "// tail", kind: TokenLineComment, sameLine: false}, }, }, { - "trailing comment at end of file attaches to token", + "trailing comment at end of file", "i32 x // tail", []triviaCheck{ - {idx: 1, trailing: []string{"// tail"}}, + {idx: 2, text: "// tail", kind: TokenLineComment, sameLine: true}, }, }, { "blank lines counted", "i32 x\n\n\ni32 y", []triviaCheck{ - {idx: 2, blankLinesBefore: 2}, + {idx: 2, text: "i32", kind: TokenI32, blankLinesBefore: 2}, }, }, { "blank line before comment counts", "i32 x\n\n// c\ni32 y", []triviaCheck{ - {idx: 2, leading: []string{"// c"}, blankLinesBefore: 1}, + {idx: 2, text: "// c", kind: TokenLineComment, blankLinesBefore: 1}, }, }, { - "blank line after trailing comment counts", + "blank line after same-line comment counts", "i32 x // c\n\ni32 y", []triviaCheck{ - {idx: 2, blankLinesBefore: 1}, + {idx: 3, text: "i32", kind: TokenI32, blankLinesBefore: 1}, }, }, { "no blank lines", "i32 x\ni32 y", []triviaCheck{ - {idx: 2, blankLinesBefore: 0}, + {idx: 3, text: "y", kind: TokenIdentifier, blankLinesBefore: 0}, }, }, { "multiline block comment spanning tokens", "i32 x /*\n c\n*/ i32 y", []triviaCheck{ - {idx: 1, trailing: []string{"/*\n c\n*/"}}, + {idx: 2, text: "/*\n c\n*/", kind: TokenBlockComment, sameLine: true}, }, }, } @@ -390,12 +392,24 @@ func TestLexTrivia(t *testing.T) { } tok := toks[check.idx] - if got := triviaTexts(tok.Leading); !reflect.DeepEqual(got, check.leading) { - t.Errorf("token %d leading: got %v, want %v", check.idx, got, check.leading) + if tok.Text != check.text { + t.Errorf("token %d text: got %q, want %q", check.idx, tok.Text, check.text) } - if got := triviaTexts(tok.Trailing); !reflect.DeepEqual(got, check.trailing) { - t.Errorf("token %d trailing: got %v, want %v", check.idx, got, check.trailing) + if tok.Kind != check.kind { + t.Errorf("token %d kind: got %v, want %v", check.idx, tok.Kind, check.kind) + } + + if IsComment(tok.Kind) && check.sameLine { + prev := check.idx - 1 + for prev >= 0 && IsComment(toks[prev].Kind) { + prev-- + } + + if prev < 0 || toks[prev].Line != tok.Line { + t.Errorf("token %d %q: want same line as token %d, got lines %d and %d", + check.idx, tok.Text, prev, tok.Line, lineOf(toks, prev)) + } } if tok.BlankLinesBefore != check.blankLinesBefore { @@ -406,30 +420,25 @@ func TestLexTrivia(t *testing.T) { } } -func triviaTexts(trivia []Trivia) []string { - if len(trivia) == 0 { - return nil - } - - texts := make([]string, 0, len(trivia)) - for _, t := range trivia { - texts = append(texts, t.Text) +func lineOf(toks []Token, idx int) int { + if idx < 0 { + return -1 } - return texts + return toks[idx].Line } func TestLexTriviaKinds(t *testing.T) { tests := []struct { name string src string - want []TriviaKind + want []TokenKind }{ - {"line", "// a\n# b\n", []TriviaKind{TriviaLineComment, TriviaLineComment}}, - {"block and doc", "/** d */\n/* b */\n", []TriviaKind{TriviaDocComment, TriviaBlockComment}}, - {"silly comment is doc", "/***/\n", []TriviaKind{TriviaDocComment}}, - {"annotation", "@naming.PreviouslyKnownAs{'x': 'y'}\n", []TriviaKind{TriviaAnnotation}}, - {"annotation with comment", "@deprecation.Deprecated{}\n// note\n", []TriviaKind{TriviaAnnotation, TriviaLineComment}}, + {"line", "// a\n# b\n", []TokenKind{TokenLineComment, TokenLineComment}}, + {"block and doc", "/** d */\n/* b */\n", []TokenKind{TokenDocComment, TokenBlockComment}}, + {"silly comment is doc", "/***/\n", []TokenKind{TokenDocComment}}, + {"annotation", "@naming.PreviouslyKnownAs{'x': 'y'}\n", []TokenKind{TokenAnnotation}}, + {"annotation with comment", "@deprecation.Deprecated{}\n// note\n", []TokenKind{TokenAnnotation, TokenLineComment}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -438,14 +447,9 @@ func TestLexTriviaKinds(t *testing.T) { t.Fatalf("unexpected errors: %v", errs) } - eof := toks[len(toks)-1] - if eof.Kind != TokenEOF { - t.Fatalf("last token is %v, want eof", eof.Kind) - } - - var got []TriviaKind - for _, trivia := range eof.Leading { - got = append(got, trivia.Kind) + var got []TokenKind + for _, tok := range toks[:len(toks)-1] { + got = append(got, tok.Kind) } if !reflect.DeepEqual(got, tt.want) { @@ -520,13 +524,13 @@ func TestLexErrors(t *testing.T) { name: "unterminated comment", src: "/* x", wantErrs: []string{"unterminated comment"}, - wantKinds: []TokenKind{TokenEOF}, + wantKinds: []TokenKind{TokenBlockComment, TokenEOF}, }, { name: "unterminated doc comment", src: "/** x", wantErrs: []string{"unterminated comment"}, - wantKinds: []TokenKind{TokenEOF}, + wantKinds: []TokenKind{TokenBlockComment, TokenEOF}, }, } diff --git a/syntax/parser.go b/syntax/parser.go index 92d195f..b94eb30 100644 --- a/syntax/parser.go +++ b/syntax/parser.go @@ -34,17 +34,32 @@ type parser struct { // --- token helpers --------------------------------------------------------- -func (p *parser) cur() Token { return p.toks[p.pos] } +// nextReal returns the index of the next non-comment token at or after i. +func (p *parser) nextReal(i int) int { + for i < len(p.toks) && IsComment(p.toks[i].Kind) { + i++ + } + + return i +} + +func (p *parser) cur() Token { return p.toks[p.nextReal(p.pos)] } func (p *parser) at(k TokenKind) bool { return p.cur().Kind == k } func (p *parser) advance() *Token { - t := &p.toks[p.pos] - p.pos++ + i := p.nextReal(p.pos) + t := &p.toks[i] + p.pos = i + 1 return t } +// peekAfter returns the kind of the real token after the token at index i. +func (p *parser) peekAfter(i int) TokenKind { + return p.toks[p.nextReal(i+1)].Kind +} + // accept consumes and returns the current token when its kind matches. func (p *parser) accept(k TokenKind) *Token { if p.at(k) { @@ -152,7 +167,7 @@ func (p *parser) parseDocument() *Document { // --- headers --------------------------------------------------------------- func (p *parser) parseInclude() *Include { - n := &Include{nodeBase: nodeBase{first: p.pos}} + n := &Include{nodeBase: nodeBase{first: p.nextReal(p.pos)}} p.advance() // include if !p.at(TokenStringLiteral) { @@ -172,7 +187,7 @@ func (p *parser) parseInclude() *Include { } func (p *parser) parseCPPInclude() *CPPInclude { - n := &CPPInclude{nodeBase: nodeBase{first: p.pos}} + n := &CPPInclude{nodeBase: nodeBase{first: p.nextReal(p.pos)}} p.advance() // cpp_include if !p.at(TokenStringLiteral) { @@ -192,7 +207,7 @@ func (p *parser) parseCPPInclude() *CPPInclude { } func (p *parser) parseNamespace() *Namespace { - n := &Namespace{nodeBase: nodeBase{first: p.pos}} + n := &Namespace{nodeBase: nodeBase{first: p.nextReal(p.pos)}} p.advance() // namespace switch { @@ -221,7 +236,7 @@ func (p *parser) parseNamespace() *Namespace { // --- definitions ----------------------------------------------------------- func (p *parser) parseConst() *Const { - n := &Const{nodeBase: nodeBase{first: p.pos}} + n := &Const{nodeBase: nodeBase{first: p.nextReal(p.pos)}} p.advance() // const n.Type = p.parseFieldType() @@ -261,7 +276,7 @@ func (p *parser) parseConst() *Const { } func (p *parser) parseTypedef() *Typedef { - n := &Typedef{nodeBase: nodeBase{first: p.pos}} + n := &Typedef{nodeBase: nodeBase{first: p.nextReal(p.pos)}} p.advance() // typedef n.Type = p.parseFieldType() @@ -289,7 +304,7 @@ func (p *parser) parseTypedef() *Typedef { } func (p *parser) parseEnum() *Enum { - n := &Enum{nodeBase: nodeBase{first: p.pos}} + n := &Enum{nodeBase: nodeBase{first: p.nextReal(p.pos)}} p.advance() // enum n.Name = p.expectIdentifier("enum name") @@ -333,7 +348,7 @@ func (p *parser) parseEnumValue() *EnumValue { return nil } - v := &EnumValue{nodeBase: nodeBase{first: p.pos}} + v := &EnumValue{nodeBase: nodeBase{first: p.nextReal(p.pos)}} v.Name = p.identifier() if p.at(TokenEqual) { @@ -357,7 +372,7 @@ func (p *parser) parseEnumValue() *EnumValue { } func (p *parser) parseStruct() *Struct { - n := &Struct{nodeBase: nodeBase{first: p.pos}, Kind: StructKind(p.cur().Kind)} + n := &Struct{nodeBase: nodeBase{first: p.nextReal(p.pos)}, Kind: StructKind(p.cur().Kind)} p.advance() // struct | union | exception n.Name = p.expectIdentifier("struct name") @@ -385,7 +400,7 @@ func (p *parser) parseStruct() *Struct { } func (p *parser) parseService() *Service { - n := &Service{nodeBase: nodeBase{first: p.pos}} + n := &Service{nodeBase: nodeBase{first: p.nextReal(p.pos)}} p.advance() // service n.Name = p.expectIdentifier("service name") @@ -436,7 +451,7 @@ func (p *parser) parseService() *Service { } func (p *parser) parseFunction() *Function { - f := &Function{nodeBase: nodeBase{first: p.pos}} + f := &Function{nodeBase: nodeBase{first: p.nextReal(p.pos)}} switch p.cur().Kind { case TokenOneway: @@ -534,9 +549,9 @@ func (p *parser) parseFieldList(term TokenKind) []*Field { } func (p *parser) parseField() (*Field, bool) { - f := &Field{nodeBase: nodeBase{first: p.pos}} + f := &Field{nodeBase: nodeBase{first: p.nextReal(p.pos)}} - if p.at(TokenIntConstant) && p.toks[p.pos+1].Kind == TokenColon { + if p.at(TokenIntConstant) && p.peekAfter(p.nextReal(p.pos)) == TokenColon { f.FieldID = p.advance() if !p.expect(TokenColon, "':' after field id") { p.synchronizeTo(TokenComma, TokenSemicolon, TokenRBrace, TokenRParen) @@ -598,7 +613,7 @@ func (p *parser) parseField() (*Field, bool) { // --- types ----------------------------------------------------------------- func (p *parser) parseFieldType() *FieldType { - t := &FieldType{nodeBase: nodeBase{first: p.pos}} + t := &FieldType{nodeBase: nodeBase{first: p.nextReal(p.pos)}} switch p.cur().Kind { case TokenMap, TokenList, TokenSet: @@ -709,7 +724,7 @@ func isBaseType(k TokenKind) bool { // --- constant values ------------------------------------------------------- func (p *parser) parseConstValue() *ConstValue { - v := &ConstValue{nodeBase: nodeBase{first: p.pos}} + v := &ConstValue{nodeBase: nodeBase{first: p.nextReal(p.pos)}} switch p.cur().Kind { case TokenIntConstant, TokenTrue, TokenFalse: @@ -831,7 +846,7 @@ func (p *parser) parseAnnotationsIfPresent() *Annotations { // literal (a bare name means an implicit value of "1"), and each may end // with an optional ',' or ';'. func (p *parser) parseAnnotations() *Annotations { - a := &Annotations{nodeBase: nodeBase{first: p.pos}} + a := &Annotations{nodeBase: nodeBase{first: p.nextReal(p.pos)}} p.advance() // ( for !p.at(TokenRParen) && !p.at(TokenEOF) { @@ -846,7 +861,7 @@ func (p *parser) parseAnnotations() *Annotations { continue } - item := &Annotation{nodeBase: nodeBase{first: p.pos}} + item := &Annotation{nodeBase: nodeBase{first: p.nextReal(p.pos)}} item.Name = p.identifier() if p.at(TokenEqual) { @@ -894,7 +909,7 @@ func (p *parser) acceptSeparator() TokenKind { } func (p *parser) identifier() *Identifier { - i := p.pos + i := p.nextReal(p.pos) t := p.advance() return &Identifier{nodeBase: nodeBase{first: i, last: i}, Text: t.Text}