Something went wrong. Try again.
Lightweight AST-based codebase indexer for AI coding assistants. Generates a `relationships.json` that Claude Code / Cursor / etc. can reference instead of exploring the entire codebase. paragraph.com/@metaend
ai llm eco ast golang
Something went wrong. Try again.
codebase-indexer main.go
23 kB · 884 lines
Go
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885package main
import ( "context" "encoding/json" "flag" "fmt" "io/fs" "os" "path/filepath" "strings"
sitter "github.com/smacker/go-tree-sitter" "github.com/smacker/go-tree-sitter/golang" "github.com/smacker/go-tree-sitter/javascript" "github.com/smacker/go-tree-sitter/python" "github.com/smacker/go-tree-sitter/typescript/typescript")
type FileInfo struct { Language string `json:"language"` Exports []string `json:"exports,omitempty"` Defines []Symbol `json:"defines,omitempty"`}
type Symbol struct { Name string `json:"name"` Type string `json:"type"` Line uint32 `json:"line"`}
type Index struct { Files map[string]FileInfo `json:"files"` Symbols map[string]SymbolRef `json:"symbols"` Graph map[string][]string `json:"dependencyGraph"`}
type SymbolRef struct { File string `json:"file"` Line uint32 `json:"line"` Type string `json:"type"`}
var ( ignoreDirs = map[string]bool{ "node_modules": true, ".git": true, "target": true, "dist": true, "build": true, ".clj-kondo": true, ".shadow-cljs": true, ".cpcache": true, }
extToLang = map[string]string{ ".js": "javascript", ".jsx": "javascript", ".ts": "typescript", ".tsx": "typescript", ".astro": "astro", ".cljs": "clojurescript", ".clj": "clojure", ".cljc": "clojure", ".go": "go", ".py": "python", })
// parseResult is the internal result before we trim for output.type parseResult struct { Imports []string Exports []string Defines []Symbol}
func main() { root := flag.String("root", ".", "Project root directory") output := flag.String("out", ".context/relationships.json", "Output file") flag.Parse()
index := &Index{ Files: make(map[string]FileInfo), Symbols: make(map[string]SymbolRef), Graph: make(map[string][]string), }
parser := sitter.NewParser()
err := filepath.WalkDir(*root, func(path string, d fs.DirEntry, err error) error { if err != nil { return nil }
if d.IsDir() { if ignoreDirs[d.Name()] { return filepath.SkipDir } return nil }
ext := filepath.Ext(path) lang, ok := extToLang[ext] if !ok { return nil }
relPath, _ := filepath.Rel(*root, path) content, err := os.ReadFile(path) if err != nil { return nil }
result := parseFile(parser, content, lang)
// Deduplicate result.Imports = unique(result.Imports) result.Exports = unique(result.Exports) result.Defines = uniqueSymbols(result.Defines)
// Build file entry — only include defines if file has exports fileInfo := FileInfo{Language: lang} fileInfo.Exports = result.Exports if len(result.Exports) > 0 { fileInfo.Defines = result.Defines } index.Files[relPath] = fileInfo
// Global symbol index — only exported/module-level symbols exportSet := toSet(result.Exports) for _, sym := range result.Defines { if exportSet[sym.Name] || sym.Type == "class" || sym.Type == "struct" || sym.Type == "interface" || sym.Type == "type" || sym.Type == "protocol" || sym.Type == "multimethod" { index.Symbols[sym.Name] = SymbolRef{ File: relPath, Line: sym.Line, Type: sym.Type, } } }
// Dependency graph if len(result.Imports) > 0 { index.Graph[relPath] = result.Imports }
return nil })
if err != nil { fmt.Fprintf(os.Stderr, "Error walking directory: %v\n", err) os.Exit(1) }
// Ensure output directory exists outDir := filepath.Dir(*output) if outDir != "." { os.MkdirAll(outDir, 0755) }
jsonOut, _ := json.MarshalIndent(index, "", " ") err = os.WriteFile(*output, jsonOut, 0644) if err != nil { fmt.Fprintf(os.Stderr, "Error writing output: %v\n", err) os.Exit(1) }
fmt.Printf("Indexed %d files, %d symbols\n", len(index.Files), len(index.Symbols))}
func toSet(slice []string) map[string]bool { s := make(map[string]bool, len(slice)) for _, v := range slice { s[v] = true } return s}
func parseFile(parser *sitter.Parser, content []byte, lang string) parseResult { var treeSitterLang *sitter.Language
switch lang { case "javascript": treeSitterLang = javascript.GetLanguage() case "typescript": treeSitterLang = typescript.GetLanguage() case "astro": return parseAstro(parser, content) case "clojure", "clojurescript": return parseResult{} case "go": treeSitterLang = golang.GetLanguage() case "python": treeSitterLang = python.GetLanguage() default: return parseResult{} }
parser.SetLanguage(treeSitterLang) tree, err := parser.ParseCtx(context.Background(), nil, content) if err != nil { return parseResult{} } defer tree.Close()
root := tree.RootNode()
switch lang { case "javascript", "typescript": return parseJSTS(root, content) case "clojure", "clojurescript": return parseClojure(root, content) case "go": return parseGo(root, content) case "python": return parsePython(root, content) }
return parseResult{}}
// isTopLevel checks if a node is a direct child of the program/module root.func isTopLevel(node *sitter.Node) bool { parent := node.Parent() if parent == nil { return true } t := parent.Type() return t == "program" || t == "module" || t == "source_file" || t == "export_statement"}
func parseJSTS(root *sitter.Node, content []byte) parseResult { var result parseResult
var walk func(*sitter.Node) walk = func(node *sitter.Node) { switch node.Type() { case "import_statement": for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "string" { importPath := strings.Trim(child.Content(content), "\"'") result.Imports = append(result.Imports, importPath) } }
case "export_statement": hasDefault := false for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) switch child.Type() { case "identifier": // export default SomeName or export SomeName result.Exports = append(result.Exports, child.Content(content)) case "function_declaration", "class_declaration": // export function foo / export class C walk(child) nameNode := child.ChildByFieldName("name") if nameNode != nil { result.Exports = append(result.Exports, nameNode.Content(content)) } case "lexical_declaration", "variable_declaration": // export const x = ... / export let y = ... walk(child) for j := 0; j < int(child.ChildCount()); j++ { decl := child.Child(j) if decl.Type() == "variable_declarator" { nameNode := decl.ChildByFieldName("name") if nameNode != nil && nameNode.Type() == "identifier" { result.Exports = append(result.Exports, nameNode.Content(content)) } } } case "export_clause": // export { name1, name2 } or export { name1 as alias } extractExportClause(child, content, &result) case "type_alias_declaration", "interface_declaration": // export type Foo = ... / export interface Bar { ... } nameNode := child.ChildByFieldName("name") if nameNode != nil { name := nameNode.Content(content) result.Exports = append(result.Exports, name) result.Defines = append(result.Defines, Symbol{ Name: name, Type: child.Type()[:len(child.Type())-len("_declaration")], Line: child.StartPoint().Row + 1, }) } case "default": hasDefault = true } } // For "export default function name" / "export default class name", // the function/class name was already captured by the walk(child) above. // For "export default <expression>", mark the file's default export. if hasDefault { // Check if a named declaration was already captured foundNamed := false for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "function_declaration" || child.Type() == "class_declaration" { nameNode := child.ChildByFieldName("name") if nameNode != nil { result.Exports = append(result.Exports, nameNode.Content(content)) foundNamed = true } } } if !foundNamed { // export default <identifier> already handled above // export default <expression> — nothing useful to name } }
case "function_declaration": if !isTopLevel(node) { break } nameNode := node.ChildByFieldName("name") if nameNode != nil { result.Defines = append(result.Defines, Symbol{ Name: nameNode.Content(content), Type: "function", Line: node.StartPoint().Row + 1, }) }
case "class_declaration": if !isTopLevel(node) { break } nameNode := node.ChildByFieldName("name") if nameNode != nil { result.Defines = append(result.Defines, Symbol{ Name: nameNode.Content(content), Type: "class", Line: node.StartPoint().Row + 1, }) }
case "lexical_declaration", "variable_declaration": if !isTopLevel(node) { break } for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "variable_declarator" { nameNode := child.ChildByFieldName("name") if nameNode == nil { continue } // Skip destructuring patterns if nameNode.Type() != "identifier" { continue } symType := "variable" valueNode := child.ChildByFieldName("value") if valueNode != nil && valueNode.Type() == "arrow_function" { symType = "function" } result.Defines = append(result.Defines, Symbol{ Name: nameNode.Content(content), Type: symType, Line: child.StartPoint().Row + 1, }) } } }
for i := 0; i < int(node.ChildCount()); i++ { walk(node.Child(i)) } }
walk(root) return result}
// parseAstro handles .astro files by extracting the frontmatter (TypeScript)// and parsing it for imports. Astro components are always exported by their file.func parseAstro(parser *sitter.Parser, content []byte) parseResult { var result parseResult
// Extract frontmatter between --- markers frontmatter := extractAstroFrontmatter(content) if len(frontmatter) > 0 { // Parse frontmatter as TypeScript parser.SetLanguage(typescript.GetLanguage()) tree, err := parser.ParseCtx(context.Background(), nil, frontmatter) if err == nil { defer tree.Close() fmResult := parseJSTS(tree.RootNode(), frontmatter) result.Imports = fmResult.Imports // Include defines from frontmatter (exported functions, etc.) result.Defines = fmResult.Defines } }
// Astro components are always exported as the default export result.Exports = append(result.Exports, "default") result.Defines = append(result.Defines, Symbol{ Name: "default", Type: "component", Line: 1, })
return result}
// extractAstroFrontmatter extracts content between --- markers at the start of an Astro filefunc extractAstroFrontmatter(content []byte) []byte { str := string(content)
// Find the first --- firstIdx := strings.Index(str, "---") if firstIdx == -1 { return nil }
// Check that --- is at the start (allowing leading whitespace/newlines) prefix := strings.TrimSpace(str[:firstIdx]) if prefix != "" { return nil }
// Find the closing --- rest := str[firstIdx+3:] secondIdx := strings.Index(rest, "---") if secondIdx == -1 { return nil }
return []byte(rest[:secondIdx])}
func extractExportClause(node *sitter.Node, content []byte, result *parseResult) { for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "export_specifier" { // export { name } or export { name as alias } nameNode := child.ChildByFieldName("name") if nameNode != nil { result.Exports = append(result.Exports, nameNode.Content(content)) } } }}
func parseClojure(root *sitter.Node, content []byte) parseResult { var result parseResult
var walk func(*sitter.Node) walk = func(node *sitter.Node) { if node.Type() == "list_lit" && node.ChildCount() > 0 { first := findFirstSymbol(node, content)
switch first { case "ns": parseNsForm(node, content, &result)
case "def", "defonce": name := findNthSymbol(node, content, 1) if name != "" { result.Defines = append(result.Defines, Symbol{ Name: name, Type: "var", Line: node.StartPoint().Row + 1, }) result.Exports = append(result.Exports, name) }
case "defn", "defn-": name := findNthSymbol(node, content, 1) if name != "" { symType := "function" if first == "defn-" { symType = "private-function" } else { result.Exports = append(result.Exports, name) } result.Defines = append(result.Defines, Symbol{ Name: name, Type: symType, Line: node.StartPoint().Row + 1, }) }
case "defmulti": name := findNthSymbol(node, content, 1) if name != "" { result.Defines = append(result.Defines, Symbol{ Name: name, Type: "multimethod", Line: node.StartPoint().Row + 1, }) result.Exports = append(result.Exports, name) }
case "defprotocol": name := findNthSymbol(node, content, 1) if name != "" { result.Defines = append(result.Defines, Symbol{ Name: name, Type: "protocol", Line: node.StartPoint().Row + 1, }) result.Exports = append(result.Exports, name) }
case "defrecord", "deftype": name := findNthSymbol(node, content, 1) if name != "" { result.Defines = append(result.Defines, Symbol{ Name: name, Type: first, Line: node.StartPoint().Row + 1, }) result.Exports = append(result.Exports, name) } } }
for i := 0; i < int(node.ChildCount()); i++ { walk(node.Child(i)) } }
walk(root) return result}
func parseNsForm(node *sitter.Node, content []byte, result *parseResult) { var walkNs func(*sitter.Node) walkNs = func(n *sitter.Node) { if n.Type() == "kwd_lit" { kwd := n.Content(content) if kwd == ":require" || kwd == ":require-macros" { parent := n.Parent() if parent != nil { foundRequire := false for i := 0; i < int(parent.ChildCount()); i++ { child := parent.Child(i) if child == n { foundRequire = true continue } if foundRequire { extractRequires(child, content, result) } } } } } for i := 0; i < int(n.ChildCount()); i++ { walkNs(n.Child(i)) } } walkNs(node)}
func extractRequires(node *sitter.Node, content []byte, result *parseResult) { if node.Type() == "vec_lit" { sym := findFirstSymbol(node, content) if sym != "" { result.Imports = append(result.Imports, sym) } } else if node.Type() == "sym_lit" { sym := node.Content(content) result.Imports = append(result.Imports, sym) }}
func findFirstSymbol(node *sitter.Node, content []byte) string { for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "sym_lit" { return child.Content(content) } } return ""}
func findNthSymbol(node *sitter.Node, content []byte, n int) string { count := 0 for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "sym_lit" { if count == n { return child.Content(content) } count++ } } return ""}
func unique(slice []string) []string { seen := make(map[string]bool) result := []string{} for _, s := range slice { if !seen[s] { seen[s] = true result = append(result, s) } } return result}
func uniqueSymbols(syms []Symbol) []Symbol { seen := make(map[string]bool) result := []Symbol{} for _, s := range syms { if !seen[s.Name] { seen[s.Name] = true result = append(result, s) } } return result}
func parseGo(root *sitter.Node, content []byte) parseResult { var result parseResult
var walk func(*sitter.Node) walk = func(node *sitter.Node) { switch node.Type() { case "import_declaration": for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "import_spec" || child.Type() == "import_spec_list" { extractGoImports(child, content, &result) } }
case "function_declaration": nameNode := node.ChildByFieldName("name") if nameNode != nil { name := nameNode.Content(content) if len(name) > 0 && name[0] >= 'A' && name[0] <= 'Z' { result.Exports = append(result.Exports, name) } result.Defines = append(result.Defines, Symbol{ Name: name, Type: "function", Line: node.StartPoint().Row + 1, }) }
case "method_declaration": nameNode := node.ChildByFieldName("name") if nameNode != nil { name := nameNode.Content(content) receiverNode := node.ChildByFieldName("receiver") receiver := "" if receiverNode != nil { receiver = extractReceiverType(receiverNode, content) } fullName := name if receiver != "" { fullName = receiver + "." + name } if len(name) > 0 && name[0] >= 'A' && name[0] <= 'Z' { result.Exports = append(result.Exports, fullName) } result.Defines = append(result.Defines, Symbol{ Name: fullName, Type: "method", Line: node.StartPoint().Row + 1, }) }
case "type_declaration": for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "type_spec" { nameNode := child.ChildByFieldName("name") if nameNode != nil { name := nameNode.Content(content) typeNode := child.ChildByFieldName("type") typeKind := "type" if typeNode != nil { switch typeNode.Type() { case "struct_type": typeKind = "struct" case "interface_type": typeKind = "interface" } } if len(name) > 0 && name[0] >= 'A' && name[0] <= 'Z' { result.Exports = append(result.Exports, name) } result.Defines = append(result.Defines, Symbol{ Name: name, Type: typeKind, Line: child.StartPoint().Row + 1, }) } } }
case "var_declaration", "const_declaration": declType := "var" if node.Type() == "const_declaration" { declType = "const" } for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "var_spec" || child.Type() == "const_spec" { nameNode := child.ChildByFieldName("name") if nameNode != nil { name := nameNode.Content(content) if len(name) > 0 && name[0] >= 'A' && name[0] <= 'Z' { result.Exports = append(result.Exports, name) } result.Defines = append(result.Defines, Symbol{ Name: name, Type: declType, Line: child.StartPoint().Row + 1, }) } } } }
for i := 0; i < int(node.ChildCount()); i++ { walk(node.Child(i)) } }
walk(root) return result}
func extractGoImports(node *sitter.Node, content []byte, result *parseResult) { if node.Type() == "import_spec_list" { for i := 0; i < int(node.ChildCount()); i++ { extractGoImports(node.Child(i), content, result) } return }
if node.Type() == "import_spec" { pathNode := node.ChildByFieldName("path") if pathNode != nil { importPath := strings.Trim(pathNode.Content(content), "\"") result.Imports = append(result.Imports, importPath) } }}
func extractReceiverType(node *sitter.Node, content []byte) string { var walk func(*sitter.Node) string walk = func(n *sitter.Node) string { if n.Type() == "type_identifier" { return n.Content(content) } for i := 0; i < int(n.ChildCount()); i++ { if result := walk(n.Child(i)); result != "" { return result } } return "" } return walk(node)}
func parsePython(root *sitter.Node, content []byte) parseResult { var result parseResult
var walk func(*sitter.Node, bool) walk = func(node *sitter.Node, topLevel bool) { switch node.Type() { case "import_statement": for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "dotted_name" { result.Imports = append(result.Imports, child.Content(content)) } }
case "import_from_statement": moduleNode := node.ChildByFieldName("module_name") if moduleNode != nil { result.Imports = append(result.Imports, moduleNode.Content(content)) } else { for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "dotted_name" || child.Type() == "relative_import" { result.Imports = append(result.Imports, child.Content(content)) break } } }
case "function_definition": if !topLevel { break } nameNode := node.ChildByFieldName("name") if nameNode != nil { name := nameNode.Content(content) symType := "function" if hasPythonDecorator(node, content, "property") { symType = "property" } else if hasPythonDecorator(node, content, "staticmethod") { symType = "staticmethod" } else if hasPythonDecorator(node, content, "classmethod") { symType = "classmethod" } if !strings.HasPrefix(name, "_") { result.Exports = append(result.Exports, name) } result.Defines = append(result.Defines, Symbol{ Name: name, Type: symType, Line: node.StartPoint().Row + 1, }) }
case "class_definition": if !topLevel { break } nameNode := node.ChildByFieldName("name") if nameNode != nil { name := nameNode.Content(content) if !strings.HasPrefix(name, "_") { result.Exports = append(result.Exports, name) } result.Defines = append(result.Defines, Symbol{ Name: name, Type: "class", Line: node.StartPoint().Row + 1, }) }
case "assignment": if topLevel { leftNode := node.ChildByFieldName("left") if leftNode != nil && leftNode.Type() == "identifier" { name := leftNode.Content(content) if !strings.HasPrefix(name, "_") { result.Exports = append(result.Exports, name) } result.Defines = append(result.Defines, Symbol{ Name: name, Type: "variable", Line: node.StartPoint().Row + 1, }) } } }
// Only recurse as top-level for module-level children isModule := node.Type() == "module" for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) walk(child, isModule) } }
walk(root, false) return result}
func hasPythonDecorator(node *sitter.Node, content []byte, name string) bool { parent := node.Parent() if parent == nil { return false }
for i := 0; i < int(parent.ChildCount()); i++ { child := parent.Child(i) if child == node { break } if child.Type() == "decorator" { decoratorContent := child.Content(content) if strings.Contains(decoratorContent, name) { return true } } } return false}