Something went wrong. Try again.
A privacy-first, self-hosted, fully open source personal knowledge management software, written in typescript and golang. (PERSONAL FORK)
Something went wrong. Try again.
Go
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742// SiYuan - Refactor your thinking// Copyright (c) 2020-present, b3log.org//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as published by// the Free Software Foundation, either version 3 of the License, or// (at your option) any later version.//// This program is distributed in the hope that it will be useful,// but WITHOUT ANY WARRANTY; without even the implied warranty of// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.
package model
import ( "bytes" "math" "strings" "unicode/utf8"
"github.com/88250/gulu" "github.com/88250/lute/ast" "github.com/88250/lute/html" "github.com/88250/lute/parse" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util")
type GraphNode struct { ID string `json:"id"` Box string `json:"box"` Path string `json:"path"` Size float64 `json:"size"` Title string `json:"title,omitempty"` Label string `json:"label"` Type string `json:"type"` Refs int `json:"refs"` Defs int `json:"defs"`}
type GraphLink struct { From string `json:"from"` To string `json:"to"` Ref bool `json:"ref"` Arrows *GraphArrows `json:"arrows"`}
type GraphArrows struct { To *GraphArrowsTo `json:"to"`}
type GraphArrowsTo struct { Enabled bool `json:"enabled"`}
func BuildTreeGraph(id, query string) (boxID string, nodes []*GraphNode, links []*GraphLink) { nodes = []*GraphNode{} links = []*GraphLink{}
tree, err := LoadTreeByBlockID(id) if err != nil { return } node := treenode.GetNodeInTree(tree, id) if nil == node { return } sqlBlock := sql.BuildBlockFromNode(node, tree) boxID = sqlBlock.Box block := fromSQLBlock(sqlBlock, "", 0)
stmt := query2Stmt(query) stmt += graphTypeFilter(true) stmt += graphDailyNoteFilter(true) stmt = strings.ReplaceAll(stmt, "content", "ref.content") forwardlinks, backlinks := buildFullLinks(stmt)
var sqlBlocks []*sql.Block var rootID string if ast.NodeDocument == node.Type { sqlBlocks = sql.GetAllChildBlocks([]string{block.ID}, stmt, Conf.Graph.MaxBlocks) rootID = block.ID } else { sqlBlocks = sql.GetChildBlocks(block.ID, stmt, Conf.Graph.MaxBlocks) } blocks := fromSQLBlocks(&sqlBlocks, "", 0) if "" != rootID { // 局部关系图中添加文档链接关系 https://github.com/siyuan-note/siyuan/issues/4996 rootBlock := getBlockIn(blocks, rootID) if nil != rootBlock { // 按引用处理 sqlRootDefs := sql.QueryDefRootBlocksByRefRootID(rootID) rootDefBlocks := fromSQLBlocks(&sqlRootDefs, "", 0) var rootIDs []string for _, rootDef := range rootDefBlocks { blocks = append(blocks, rootDef) rootIDs = append(rootIDs, rootDef.ID) }
sqlRefBlocks := sql.QueryRefRootBlocksByDefRootIDs(rootIDs) for defRootID, sqlRefBs := range sqlRefBlocks { rootB := getBlockIn(rootDefBlocks, defRootID) if nil == rootB { continue }
blocks = append(blocks, rootB) refBlocks := fromSQLBlocks(&sqlRefBs, "", 0) rootB.Refs = append(rootB.Refs, refBlocks...) blocks = append(blocks, refBlocks...) }
// 按定义处理 blocks = append(blocks, rootBlock) sqlRefBlocks = sql.QueryRefRootBlocksByDefRootIDs([]string{rootID})
// 关系图日记过滤失效 https://github.com/siyuan-note/siyuan/issues/7547 dailyNotesPaths := dailyNotePaths(true) for _, sqlRefBs := range sqlRefBlocks { refBlocks := fromSQLBlocks(&sqlRefBs, "", 0)
if 0 < len(dailyNotesPaths) { isDailyNote := false var tmp []*Block for _, refBlock := range refBlocks { for _, dailyNotePath := range dailyNotesPaths { if strings.HasPrefix(refBlock.HPath, dailyNotePath) { isDailyNote = true break } }
if !isDailyNote { tmp = append(tmp, refBlock) } } refBlocks = tmp }
rootBlock.Refs = append(rootBlock.Refs, refBlocks...) blocks = append(blocks, refBlocks...) } } }
blocks = filterDailyNote(blocks, true) genTreeNodes(blocks, &nodes, &links, true) growTreeGraph(&forwardlinks, &backlinks, &nodes) blocks = append(blocks, forwardlinks...) blocks = append(blocks, backlinks...) buildLinks(&blocks, &links, true) if Conf.Graph.Local.Tag { p := sqlBlock.Path linkTagBlocks(&blocks, &nodes, &links, p) } markLinkedNodes(&nodes, &links, true) nodes = removeDuplicatedUnescape(nodes) return}
func BuildGraph(query string) (boxID string, nodes []*GraphNode, links []*GraphLink) { nodes = []*GraphNode{} links = []*GraphLink{}
stmt := query2Stmt(query) stmt = strings.TrimPrefix(stmt, "select * from blocks where") stmt += graphTypeFilter(false) stmt += graphDailyNoteFilter(false) stmt = strings.ReplaceAll(stmt, "content", "ref.content") forwardlinks, backlinks := buildFullLinks(stmt)
var blocks []*Block roots := sql.GetAllRootBlocks() if 0 < len(roots) { boxID = roots[0].Box } var rootIDs []string for _, root := range roots { rootIDs = append(rootIDs, root.ID) } rootIDs = gulu.Str.RemoveDuplicatedElem(rootIDs)
sqlBlocks := sql.GetAllChildBlocks(rootIDs, stmt, Conf.Graph.MaxBlocks) treeBlocks := fromSQLBlocks(&sqlBlocks, "", 0) treeBlocks = filterDailyNote(treeBlocks, false) genTreeNodes(treeBlocks, &nodes, &links, false) blocks = append(blocks, treeBlocks...)
// 文档块关联 sqlRootRefBlocks := sql.QueryRefRootBlocksByDefRootIDs(rootIDs) for defRootID, sqlRefBlocks := range sqlRootRefBlocks { rootBlock := getBlockIn(treeBlocks, defRootID) if nil == rootBlock { continue }
refBlocks := fromSQLBlocks(&sqlRefBlocks, "", 0) rootBlock.Refs = append(rootBlock.Refs, refBlocks...) }
growTreeGraph(&forwardlinks, &backlinks, &nodes) blocks = append(blocks, forwardlinks...) blocks = append(blocks, backlinks...) buildLinks(&blocks, &links, false) if Conf.Graph.Global.Tag { linkTagBlocks(&blocks, &nodes, &links, "") } markLinkedNodes(&nodes, &links, false) pruneUnref(&nodes, &links) nodes = removeDuplicatedUnescape(nodes) return}
func linkTagBlocks(blocks *[]*Block, nodes *[]*GraphNode, links *[]*GraphLink, p string) { tagSpans := sql.QueryTagSpans(p) if 1 > len(tagSpans) { return }
isGlobal := "" == p nodeSize := Conf.Graph.Global.NodeSize if !isGlobal { nodeSize = Conf.Graph.Local.NodeSize }
// 构造标签节点 var tagNodes []*GraphNode for _, tagSpan := range tagSpans { if nil == tagNodeIn(tagNodes, tagSpan.Content) { node := &GraphNode{ ID: tagSpan.Content, Label: tagSpan.Content, Size: nodeSize, Type: tagSpan.Type, } *nodes = append(*nodes, node) tagNodes = append(tagNodes, node) } }
// 连接标签和块 for _, block := range *blocks { for _, tagSpan := range tagSpans { if isGlobal { // 全局关系图将标签链接到文档块上 if block.RootID == tagSpan.RootID { // 局部关系图将标签链接到子块上 *links = append(*links, &GraphLink{ From: tagSpan.Content, To: block.RootID, }) } } else { if block.ID == tagSpan.BlockID { // 局部关系图将标签链接到子块上 *links = append(*links, &GraphLink{ From: tagSpan.Content, To: block.ID, }) } } } }
// 连接层级标签 for _, tagNode := range tagNodes { ids := strings.Split(tagNode.ID, "/") if 2 > len(ids) { continue }
for _, targetID := range ids[:len(ids)-1] { if targetTag := tagNodeIn(tagNodes, targetID); nil != targetTag {
*links = append(*links, &GraphLink{ From: tagNode.ID, To: targetID, }) } } }}
func tagNodeIn(tagNodes []*GraphNode, content string) *GraphNode { for _, tagNode := range tagNodes { if tagNode.Label == content { return tagNode } } return nil}
func growTreeGraph(forwardlinks, backlinks *[]*Block, nodes *[]*GraphNode) { forwardDepth, backDepth := 0, 0 growLinkedNodes(forwardlinks, backlinks, nodes, nodes, &forwardDepth, &backDepth)}
func growLinkedNodes(forwardlinks, backlinks *[]*Block, nodes, all *[]*GraphNode, forwardDepth, backDepth *int) { if 1 > len(*nodes) { return }
forwardGeneration := &[]*GraphNode{} if 16 > *forwardDepth { for _, ref := range *forwardlinks { for _, node := range *nodes { if node.ID == ref.ID { var defs []*Block for _, refDef := range ref.Defs { if existNodes(all, refDef.ID) || existNodes(forwardGeneration, refDef.ID) || existNodes(nodes, refDef.ID) { continue } defs = append(defs, refDef) }
for _, refDef := range defs { defNode := &GraphNode{ ID: refDef.ID, Box: refDef.Box, Path: refDef.Path, Size: Conf.Graph.Local.NodeSize, Type: refDef.Type, } nodeTitleLabel(defNode, nodeContentByBlock(refDef)) *forwardGeneration = append(*forwardGeneration, defNode) } } } } }
backGeneration := &[]*GraphNode{} if 16 > *backDepth { for _, def := range *backlinks { for _, node := range *nodes { if node.ID == def.ID { for _, ref := range def.Refs { if existNodes(all, ref.ID) || existNodes(backGeneration, ref.ID) || existNodes(nodes, ref.ID) { continue }
refNode := &GraphNode{ ID: ref.ID, Box: ref.Box, Path: ref.Path, Size: Conf.Graph.Local.NodeSize, Type: ref.Type, } nodeTitleLabel(refNode, nodeContentByBlock(ref)) *backGeneration = append(*backGeneration, refNode) } } } } }
generation := &[]*GraphNode{} *generation = append(*generation, *forwardGeneration...) *generation = append(*generation, *backGeneration...) *forwardDepth++ *backDepth++ growLinkedNodes(forwardlinks, backlinks, generation, nodes, forwardDepth, backDepth) *nodes = append(*nodes, *generation...)}
func existNodes(nodes *[]*GraphNode, id string) bool { for _, node := range *nodes { if node.ID == id { return true } } return false}
func buildLinks(defs *[]*Block, links *[]*GraphLink, local bool) { for _, def := range *defs { for _, ref := range def.Refs { link := &GraphLink{ From: ref.ID, To: def.ID, Ref: true, } if local { if Conf.Graph.Local.Arrow { link.Arrows = &GraphArrows{To: &GraphArrowsTo{Enabled: true}} } } else { if Conf.Graph.Global.Arrow { link.Arrows = &GraphArrows{To: &GraphArrowsTo{Enabled: true}} } } *links = append(*links, link) } }}
func genTreeNodes(blocks []*Block, nodes *[]*GraphNode, links *[]*GraphLink, local bool) { nodeSize := Conf.Graph.Local.NodeSize if !local { nodeSize = Conf.Graph.Global.NodeSize }
for _, block := range blocks { node := &GraphNode{ ID: block.ID, Box: block.Box, Path: block.Path, Type: block.Type, Size: nodeSize, } nodeTitleLabel(node, nodeContentByBlock(block)) *nodes = append(*nodes, node)
*links = append(*links, &GraphLink{ From: block.ParentID, To: block.ID, Ref: false, }) }}
func markLinkedNodes(nodes *[]*GraphNode, links *[]*GraphLink, local bool) { nodeSize := Conf.Graph.Local.NodeSize if !local { nodeSize = Conf.Graph.Global.NodeSize }
tmpLinks := (*links)[:0] for _, link := range *links { var sourceFound, targetFound bool for _, node := range *nodes { if link.To == node.ID { if link.Ref { size := nodeSize node.Defs++ size = math.Log2(float64(node.Defs))*nodeSize + nodeSize node.Size = size } targetFound = true } else if link.From == node.ID { node.Refs++ sourceFound = true } if targetFound && sourceFound { break } } if sourceFound && targetFound { tmpLinks = append(tmpLinks, link) } } *links = tmpLinks}
func removeDuplicatedUnescape(nodes []*GraphNode) (ret []*GraphNode) { m := map[string]*GraphNode{} for _, n := range nodes { if nil == m[n.ID] { n.Title = html.UnescapeString(n.Title) n.Label = html.UnescapeString(n.Label) ret = append(ret, n) m[n.ID] = n } } return ret}
func pruneUnref(nodes *[]*GraphNode, links *[]*GraphLink) { maxBlocks := Conf.Graph.MaxBlocks tmpNodes := (*nodes)[:0] for _, node := range *nodes { if 0 == Conf.Graph.Global.MinRefs { tmpNodes = append(tmpNodes, node) } else { if Conf.Graph.Global.MinRefs <= node.Refs { tmpNodes = append(tmpNodes, node) continue }
if Conf.Graph.Global.MinRefs <= node.Defs { tmpNodes = append(tmpNodes, node) continue } }
if maxBlocks < len(tmpNodes) { logging.LogWarnf("exceeded the maximum number of render nodes [%d]", maxBlocks) break } } *nodes = tmpNodes
tmpLinks := (*links)[:0] for _, link := range *links { var sourceFound, targetFound bool for _, node := range *nodes { if link.To == node.ID { targetFound = true } else if link.From == node.ID { sourceFound = true } } if sourceFound && targetFound { tmpLinks = append(tmpLinks, link) } } *links = tmpLinks}
func nodeContentByBlock(block *Block) (ret string) { if ret = block.Name; "" != ret { return } ret = block.Content if maxLen := 48; maxLen < utf8.RuneCountInString(ret) { ret = gulu.Str.SubStr(ret, maxLen) + "..." } return}
func graphTypeFilter(local bool) string { var inList []string
paragraph := Conf.Graph.Local.Paragraph if !local { paragraph = Conf.Graph.Global.Paragraph } if paragraph { inList = append(inList, "'p'") }
heading := Conf.Graph.Local.Heading if !local { heading = Conf.Graph.Global.Heading } if heading { inList = append(inList, "'h'") }
math := Conf.Graph.Local.Math if !local { math = Conf.Graph.Global.Math } if math { inList = append(inList, "'m'") }
code := Conf.Graph.Local.Code if !local { code = Conf.Graph.Global.Code } if code { inList = append(inList, "'c'") }
table := Conf.Graph.Local.Table if !local { table = Conf.Graph.Global.Table } if table { inList = append(inList, "'t'") }
list := Conf.Graph.Local.List if !local { list = Conf.Graph.Global.List } if list { inList = append(inList, "'l'") }
listItem := Conf.Graph.Local.ListItem if !local { listItem = Conf.Graph.Global.ListItem } if listItem { inList = append(inList, "'i'") }
blockquote := Conf.Graph.Local.Blockquote if !local { blockquote = Conf.Graph.Global.Blockquote } if blockquote { inList = append(inList, "'b'") }
super := Conf.Graph.Local.Super if !local { super = Conf.Graph.Global.Super } if super { inList = append(inList, "'s'") }
callout := Conf.Graph.Local.Callout if !local { callout = Conf.Graph.Global.Callout } if callout { inList = append(inList, "'callout'") }
inList = append(inList, "'d'") return " AND ref.type IN (" + strings.Join(inList, ",") + ")"}
func filterDailyNote(blocks []*Block, local bool) (ret []*Block) { // Graph dailynote filtering not working https://github.com/siyuan-note/siyuan/issues/16463
dailyNote := Conf.Graph.Local.DailyNote if !local { dailyNote = Conf.Graph.Global.DailyNote }
if dailyNote { ret = blocks return }
for _, block := range blocks { isDailyNote := false for k, _ := range block.IAL { isDailyNote = strings.HasPrefix(k, DailyNoteAttrPrefix) if isDailyNote { break } } if !isDailyNote { ret = append(ret, block) } } return}
func graphDailyNoteFilter(local bool) string { dailyNotesPaths := dailyNotePaths(local) if 1 > len(dailyNotesPaths) { return "" }
buf := bytes.Buffer{} for _, p := range dailyNotesPaths { buf.WriteString(" AND ref.hpath NOT LIKE '" + p + "%'") } return buf.String()}
func dailyNotePaths(local bool) (ret []string) { dailyNote := Conf.Graph.Local.DailyNote if !local { dailyNote = Conf.Graph.Global.DailyNote }
if dailyNote { return }
for _, box := range Conf.GetOpenedBoxes() { boxConf := box.GetConf() if 1 < strings.Count(boxConf.DailyNoteSavePath, "/") { dailyNoteSaveDir := strings.Split(boxConf.DailyNoteSavePath, "/")[1] ret = append(ret, "/"+dailyNoteSaveDir) } } ret = gulu.Str.RemoveDuplicatedElem(ret) return}
func nodeTitleLabel(node *GraphNode, blockContent string) { if "NodeDocument" != node.Type && "NodeHeading" != node.Type { node.Title = blockContent } else { node.Label = blockContent }}
func query2Stmt(queryStr string) (ret string) { buf := bytes.Buffer{} if ast.IsNodeIDPattern(queryStr) { buf.WriteString("id = '" + queryStr + "'") } else { var tags []string luteEngine := util.NewLute() t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions) ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus { if !entering { return ast.WalkContinue } if n.IsTextMarkType("tag") { tags = append(tags, n.Text()) } return ast.WalkContinue })
for _, tag := range tags { queryStr = strings.ReplaceAll(queryStr, "#"+tag+"#", "") } parts := strings.Split(queryStr, " ")
for i, part := range parts { if "" == part { continue } part = strings.ReplaceAll(part, "'", "''") buf.WriteString("(content LIKE '%" + part + "%'") buf.WriteString(Conf.Search.NAMFilter(part)) buf.WriteString(")") if i < len(parts)-1 { buf.WriteString(" AND ") } }
if 0 < len(tags) { if 0 < buf.Len() { buf.WriteString(" OR ") } for i, tag := range tags { buf.WriteString("(content LIKE '%#" + tag + "#%')") if i < len(tags)-1 { buf.WriteString(" AND ") } } buf.WriteString(" OR ") for i, tag := range tags { buf.WriteString("ial LIKE '%tags=\"%" + tag + "%\"%'") if i < len(tags)-1 { buf.WriteString(" AND ") } } } } if 1 > buf.Len() { buf.WriteString("1=1") } ret = buf.String() return}