From fabc4fd4d31dc7aa6c566662d37f76f9cfe4f325 Mon Sep 17 00:00:00 2001 From: karitham Date: Fri, 7 Aug 2026 21:18:10 +0200 Subject: [PATCH] lsp: semantic tokens --- lsp/initialize.go | 7 +- lsp/semantic.go | 28 ++++ lsp/semantic/semantic.go | 236 ++++++++++++++++++++++++++++++++++ lsp/semantic/semantic_test.go | 175 +++++++++++++++++++++++++ lsp/server.go | 13 +- 5 files changed, 456 insertions(+), 3 deletions(-) create mode 100644 lsp/semantic.go create mode 100644 lsp/semantic/semantic.go create mode 100644 lsp/semantic/semantic_test.go diff --git a/lsp/initialize.go b/lsp/initialize.go index 05cc8ca..ca0893d 100644 --- a/lsp/initialize.go +++ b/lsp/initialize.go @@ -11,6 +11,7 @@ import ( "go.lsp.dev/uri" "github.com/karitham/thrift-ls/lsp/cache" + "github.com/karitham/thrift-ls/lsp/semantic" ) func (s *Server) initialize(ctx context.Context, params *protocol.InitializeParams) (result *protocol.InitializeResult, err error) { @@ -224,9 +225,13 @@ func initializeResult() *protocol.InitializeResult { WorkDoneProgress: new(true), }, Legend: protocol.SemanticTokensLegend{ - TokenTypes: []string{}, + TokenTypes: semantic.Legend(), TokenModifiers: []string{}, }, + Full: &protocol.SemanticTokensFullDelta{ + Delta: new(true), + }, + Range: protocol.Boolean(false), }, StaticRegistrationOptions: protocol.StaticRegistrationOptions{ ID: new("thriftls"), diff --git a/lsp/semantic.go b/lsp/semantic.go new file mode 100644 index 0000000..80d63ce --- /dev/null +++ b/lsp/semantic.go @@ -0,0 +1,28 @@ +package lsp + +import ( + "context" + + "go.lsp.dev/protocol" + + "github.com/karitham/thrift-ls/lsp/semantic" +) + +func (s *Server) semanticTokensFull(ctx context.Context, params *protocol.SemanticTokensParams) (*protocol.SemanticTokens, error) { + file := params.TextDocument.URI + + view, err := s.session.ViewOf(file) + if err != nil { + return nil, err + } + + ss, release := view.Snapshot() + defer release() + + data, err := semantic.Tokens(ctx, ss, file) + if err != nil { + return nil, err + } + + return &protocol.SemanticTokens{Data: data}, nil +} diff --git a/lsp/semantic/semantic.go b/lsp/semantic/semantic.go new file mode 100644 index 0000000..bb718f9 --- /dev/null +++ b/lsp/semantic/semantic.go @@ -0,0 +1,236 @@ +// Package semantic computes LSP semantic tokens for a thrift document: +// keywords, types, definition names, comments, strings, and numbers. +// Pure over the snapshot: parsing and file I/O happen in the caller. +package semantic + +import ( + "context" + + "go.lsp.dev/uri" + + "github.com/karitham/thrift-ls/lsp/cache" + "github.com/karitham/thrift-ls/syntax" +) + +// legend is the ordered list of token types; indexes into it are the +// encoded token types. The server advertises it in the registration +// options, so it must stay in sync with the constants below. +var legend = []string{ + "keyword", "string", "number", "comment", + "type", "struct", "enum", "interface", + "property", "function", "enumMember", "variable", +} + +const ( + tokKeyword = iota + tokString + tokNumber + tokComment + tokType + tokStruct + tokEnum + tokInterface + tokProperty + tokFunction + tokEnumMember + tokVariable +) + +// Legend returns the semantic token types the server emits. +func Legend() []string { + return legend +} + +// Tokens returns the delta-encoded semantic tokens of a file, in source +// order. +func Tokens(ctx context.Context, ss *cache.Snapshot, file uri.URI) ([]uint32, error) { + pf, err := ss.Parse(ctx, file) + if err != nil || pf.AST() == nil { + return nil, err + } + + doc := pf.AST() + names := definitionNames(doc) + types := typeReferences(doc) + + data := make([]uint32, 0, len(doc.Tokens)*5) + prevLine, prevChar := 0, 0 + + for i, tok := range doc.Tokens { + typ, ok := classify(i, tok, names, types) + if !ok { + continue + } + + line := tok.Line - 1 + char := tok.Col - 1 + + deltaChar := char + if line == prevLine { + deltaChar = char - prevChar + } + + data = append(data, + uint32(line-prevLine), + uint32(deltaChar), + uint32(len(tok.Text)), + uint32(typ), + 0, // no token modifiers + ) + + prevLine, prevChar = line, char + } + + return data, nil +} + +// classify maps a token to its semantic type. Definition names win over +// type keywords so a field named "string" stays a property; type +// references win over keywords so "string" in a type position is a type. +func classify(i int, tok syntax.Token, names map[int]int, types map[int]bool) (int, bool) { + if syntax.IsComment(tok.Kind) { + return tokComment, true + } + + if t, ok := names[i]; ok { + return t, true + } + + if types[i] { + return tokType, true + } + + switch tok.Kind { + case syntax.TokenStringLiteral: + return tokString, true + case syntax.TokenIntConstant, syntax.TokenDoubleConstant: + return tokNumber, true + } + + if isTypeKeyword(tok.Kind) { + return tokType, true + } + + if isKeyword(tok.Kind) { + return tokKeyword, true + } + + return 0, false +} + +// definitionNames maps every definition name token to its semantic type: +// structs, enums, services, consts, typedefs, and their members. +func definitionNames(doc *syntax.Document) map[int]int { + names := map[int]int{} + + for _, n := range doc.Nodes { + switch v := n.(type) { + case *syntax.Struct: + names[v.Name.TokStart()] = tokStruct + for _, f := range v.Fields { + names[f.Name.TokStart()] = tokProperty + } + case *syntax.Enum: + names[v.Name.TokStart()] = tokEnum + for _, value := range v.Values { + names[value.Name.TokStart()] = tokEnumMember + } + case *syntax.Service: + names[v.Name.TokStart()] = tokInterface + for _, fn := range v.Functions { + names[fn.Name.TokStart()] = tokFunction + for _, arg := range fn.Args { + names[arg.Name.TokStart()] = tokProperty + } + + if fn.Throws != nil { + for _, f := range fn.Throws.Fields { + names[f.Name.TokStart()] = tokProperty + } + } + } + case *syntax.Const: + names[v.Name.TokStart()] = tokVariable + case *syntax.Typedef: + names[v.Name.TokStart()] = tokType + } + } + + return names +} + +// typeReferences maps every type reference's first token to the type +// semantic type: field types, function return types, argument and throws +// types, const and typedef types, and the nested container types. +func typeReferences(doc *syntax.Document) map[int]bool { + types := map[int]bool{} + + var add func(t *syntax.FieldType) + add = func(t *syntax.FieldType) { + if t == nil { + return + } + + types[t.TokStart()] = true + add(t.KeyType) + add(t.ValueType) + } + + for _, n := range doc.Nodes { + switch v := n.(type) { + case *syntax.Struct: + for _, f := range v.Fields { + add(f.Type) + } + case *syntax.Service: + for _, fn := range v.Functions { + add(fn.Type) + for _, arg := range fn.Args { + add(arg.Type) + } + + if fn.Throws != nil { + for _, f := range fn.Throws.Fields { + add(f.Type) + } + } + } + case *syntax.Const: + add(v.Type) + case *syntax.Typedef: + add(v.Type) + } + } + + return types +} + +// isTypeKeyword reports whether the kind is a base or container type +// keyword, which always appears in a type position. +func isTypeKeyword(k syntax.TokenKind) bool { + switch k { + case syntax.TokenMap, syntax.TokenList, syntax.TokenSet, syntax.TokenVoid, + syntax.TokenBool, syntax.TokenByte, syntax.TokenI8, syntax.TokenI16, + syntax.TokenI32, syntax.TokenI64, syntax.TokenDouble, + syntax.TokenString, syntax.TokenBinary, syntax.TokenSlist, syntax.TokenUUID: + return true + } + + return false +} + +// isKeyword reports whether the kind is a reserved word. +func isKeyword(k syntax.TokenKind) bool { + switch k { + case syntax.TokenInclude, syntax.TokenCPPInclude, syntax.TokenCPPType, + syntax.TokenNamespace, syntax.TokenStruct, syntax.TokenUnion, + syntax.TokenException, syntax.TokenService, syntax.TokenEnum, + syntax.TokenConst, syntax.TokenTypedef, syntax.TokenOneway, + syntax.TokenAsync, syntax.TokenThrows, syntax.TokenExtends, + syntax.TokenRequired, syntax.TokenOptional, syntax.TokenTrue, + syntax.TokenFalse: + return true + } + + return false +} diff --git a/lsp/semantic/semantic_test.go b/lsp/semantic/semantic_test.go new file mode 100644 index 0000000..0150021 --- /dev/null +++ b/lsp/semantic/semantic_test.go @@ -0,0 +1,175 @@ +package semantic + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.lsp.dev/protocol" + + "github.com/karitham/thrift-ls/lsp/cache" +) + +// decodedToken is one semantic token in absolute coordinates. +type decodedToken struct { + line, char, length uint32 + typ int +} + +// decode converts the delta-encoded token data back into absolute +// coordinates. +func decode(data []uint32) []decodedToken { + var out []decodedToken + line, char := uint32(0), uint32(0) + + for i := 0; i+4 < len(data); i += 5 { + line += data[i] + if data[i] == 0 { + char += data[i+1] + } else { + char = data[i+1] + } + + out = append(out, decodedToken{ + line: line, + char: char, + length: data[i+2], + typ: int(data[i+3]), + }) + } + + return out +} + +func semanticTokens(t *testing.T, src string) []decodedToken { + t.Helper() + + ss := cache.BuildSnapshotForTest([]*cache.FileChange{ + {URI: "file:///tmp/main.thrift", Version: 0, Content: []byte(src), From: cache.FileChangeTypeDidOpen}, + }) + + data, err := Tokens(t.Context(), ss, "file:///tmp/main.thrift") + require.NoError(t, err) + + return decode(data) +} + +func TestSemanticTokens(t *testing.T) { + tests := []struct { + name string + src string + want []decodedToken + }{ + { + name: "keywords, comments, strings, numbers", + src: "// doc\nconst i32 LIMIT = 10\ninclude \"base.thrift\"", + want: []decodedToken{ + {0, 0, 6, tokComment}, + {1, 0, 5, tokKeyword}, // const + {1, 6, 3, tokType}, // i32 + {1, 10, 5, tokVariable}, + {1, 18, 2, tokNumber}, + {2, 0, 7, tokKeyword}, // include + {2, 8, 13, tokString}, + }, + }, + { + name: "definition names and members", + src: `struct MobileSuit { + 1: required string Name +} + +service Federation { + void Deploy(1: string suitName), +} + +enum ZeonForces { + ZAKU_I = 1, +}`, + want: []decodedToken{ + {0, 0, 6, tokKeyword}, // struct + {0, 7, 10, tokStruct}, // MobileSuit + {1, 1, 1, tokNumber}, // 1 + {1, 4, 8, tokKeyword}, // required + {1, 13, 6, tokType}, // string + {1, 20, 4, tokProperty}, // Name + {4, 0, 7, tokKeyword}, // service + {4, 8, 10, tokInterface}, // Federation + {5, 1, 4, tokType}, // void + {5, 6, 6, tokFunction}, // Deploy + {5, 13, 1, tokNumber}, // 1 + {5, 16, 6, tokType}, // string + {5, 23, 8, tokProperty}, // suitName + {8, 0, 4, tokKeyword}, // enum + {8, 5, 10, tokEnum}, // ZeonForces + {9, 1, 6, tokEnumMember}, // ZAKU_I + {9, 10, 1, tokNumber}, // 1 + }, + }, + { + name: "type references and nested containers", + src: `struct StrikeRouge { + 1: required map packs +}`, + want: []decodedToken{ + {0, 0, 6, tokKeyword}, // struct + {0, 7, 11, tokStruct}, // StrikeRouge + {1, 1, 1, tokNumber}, // 1 + {1, 4, 8, tokKeyword}, // required + {1, 13, 3, tokType}, // map + {1, 17, 6, tokType}, // string + {1, 25, 6, tokType}, // Gundam + {1, 33, 5, tokProperty}, // packs + }, + }, + { + name: "a keyword used as a field name stays a property", + src: "struct S {\n\t1: string string\n}", + want: []decodedToken{ + {0, 0, 6, tokKeyword}, // struct + {0, 7, 1, tokStruct}, // S + {1, 1, 1, tokNumber}, // 1 + {1, 4, 6, tokType}, // string (type position) + {1, 11, 6, tokProperty}, // string (field name) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, semanticTokens(t, tt.src)) + }) + } +} + +// TestSemanticTokensEncoding pins the delta encoding: tokens on the same +// line carry relative characters, tokens on new lines carry absolute ones. +func TestSemanticTokensEncoding(t *testing.T) { + ss := cache.BuildSnapshotForTest([]*cache.FileChange{ + {URI: "file:///tmp/main.thrift", Version: 0, Content: []byte("const i32 A = 1\nconst i32 B = 2"), From: cache.FileChangeTypeDidOpen}, + }) + + data, err := Tokens(t.Context(), ss, "file:///tmp/main.thrift") + require.NoError(t, err) + + // Token 0: line 0, char 0. Token 1 (i32): same line, relative char 6. + // Token 4 (const on line 1): delta line 1, absolute char 0. + require.GreaterOrEqual(t, len(data), 25) + assert.Equal(t, uint32(0), data[0]) + assert.Equal(t, uint32(0), data[1]) + assert.Equal(t, uint32(0), data[5]) // same line as the previous token + assert.Equal(t, uint32(6), data[6]) // relative char + assert.Equal(t, uint32(1), data[20]) + assert.Equal(t, uint32(0), data[21]) // absolute char on the new line +} + +// TestSemanticTokensLegend pins the advertised legend. +func TestSemanticTokensLegend(t *testing.T) { + assert.Equal(t, []string{ + "keyword", "string", "number", "comment", + "type", "struct", "enum", "interface", + "property", "function", "enumMember", "variable", + }, Legend()) +} + +var _ = protocol.SemanticTokens{} diff --git a/lsp/server.go b/lsp/server.go index 462557c..bedf4c2 100644 --- a/lsp/server.go +++ b/lsp/server.go @@ -307,11 +307,20 @@ func (s *Server) OutgoingCalls(ctx context.Context, params *protocol.CallHierarc } func (s *Server) SemanticTokensFull(ctx context.Context, params *protocol.SemanticTokensParams) (result *protocol.SemanticTokens, err error) { - return nil, nil + return s.semanticTokensFull(ctx, params) } func (s *Server) SemanticTokensFullDelta(ctx context.Context, params *protocol.SemanticTokensDeltaParams) (result protocol.SemanticTokensDeltaResult, err error) { - return nil, nil + // No delta tracking: answer every request with the full token set, + // which is a valid delta response. + tokens, err := s.semanticTokensFull(ctx, &protocol.SemanticTokensParams{ + TextDocument: params.TextDocument, + }) + if err != nil { + return nil, err + } + + return tokens, nil } func (s *Server) SemanticTokensRange(ctx context.Context, params *protocol.SemanticTokensRangeParams) (result *protocol.SemanticTokens, err error) { -- 2.51.2