Something went wrong. Try again.
A privacy-first, self-hosted, fully open source personal knowledge management software, written in typescript and golang. (PERSONAL FORK)
Something went wrong. Try again.
Go
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694// SiYuan - Refactor your thinking// Copyright (c) 2020-present, b3log.org//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as published by// the Free Software Foundation, either version 3 of the License, or// (at your option) any later version.//// This program is distributed in the hope that it will be useful,// but WITHOUT ANY WARRANTY; without even the implied warranty of// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.
package treenode
import ( "bytes" "database/sql" "errors" "os" "runtime" "runtime/debug" "strings" "sync" "time"
"github.com/88250/gulu" "github.com/88250/lute/ast" "github.com/88250/lute/parse" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util")
type BlockTree struct { ID string // 块 ID RootID string // 根 ID ParentID string // 父 ID BoxID string // 笔记本 ID Path string // 文档数据路径 HPath string // 文档可读路径 Updated string // 更新时间 Type string // 类型}
var ( db *sql.DB)
func initDatabase(forceRebuild bool) (err error) { initDBConnection()
if !forceRebuild { if !gulu.File.IsExist(util.BlockTreeDBPath) { forceRebuild = true } } if !forceRebuild { return }
closeDatabase() if gulu.File.IsExist(util.BlockTreeDBPath) { if err = removeDatabaseFile(); err != nil { logging.LogErrorf("remove database file [%s] failed: %s", util.BlockTreeDBPath, err) err = nil } }
initDBConnection() initDBTables()
logging.LogInfof("reinitialized database [%s]", util.BlockTreeDBPath) return}
func initDBTables() { _, err := db.Exec("DROP TABLE IF EXISTS blocktrees") if err != nil { logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "drop table [blocks] failed: %s", err) } _, err = db.Exec("CREATE TABLE blocktrees (id, root_id, parent_id, box_id, path, hpath, updated, type)") if err != nil { logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create table [blocktrees] failed: %s", err) }
_, err = db.Exec("CREATE INDEX idx_blocktrees_id ON blocktrees(id)") if err != nil { logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create index [idx_blocktrees_id] failed: %s", err) }
_, err = db.Exec("CREATE INDEX idx_blocktrees_root_id ON blocktrees(root_id)") if err != nil { logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create index [idx_blocktrees_root_id] failed: %s", err) }}
func initDBConnection() { if nil != db { closeDatabase() }
util.LogDatabaseSize(util.BlockTreeDBPath) dsn := util.BlockTreeDBPath + "?_journal_mode=WAL" + "&_synchronous=OFF" + "&_mmap_size=2684354560" + "&_secure_delete=OFF" + "&_cache_size=-20480" + "&_page_size=32768" + "&_busy_timeout=7000" + "&_ignore_check_constraints=ON" + "&_temp_store=MEMORY" + "&_case_sensitive_like=OFF" var err error db, err = sql.Open("sqlite3_extended", dsn) if err != nil { logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create database failed: %s", err) } db.SetMaxIdleConns(7) db.SetMaxOpenConns(7) db.SetConnMaxLifetime(365 * 24 * time.Hour)}
func CloseDatabase() { closeDatabase()}
func closeDatabase() { if nil == db { return }
if err := db.Close(); err != nil { logging.LogErrorf("close database failed: %s", err) } debug.FreeOSMemory() db = nil runtime.GC() return}
func removeDatabaseFile() (err error) { err = os.RemoveAll(util.BlockTreeDBPath) if err != nil { return } err = os.RemoveAll(util.BlockTreeDBPath + "-shm") if err != nil { return } err = os.RemoveAll(util.BlockTreeDBPath + "-wal") if err != nil { return } return}
func GetBlockTreesByType(typ string) (ret []*BlockTree) { sqlStmt := "SELECT * FROM blocktrees WHERE type = ?" rows, err := db.Query(sqlStmt, typ) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return } defer rows.Close() for rows.Next() { var block BlockTree if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); err != nil { logging.LogErrorf("query scan field failed: %s", err) return } ret = append(ret, &block) } return}
func GetBlockTreeByPath(path string) (ret *BlockTree) { ret = &BlockTree{} sqlStmt := "SELECT * FROM blocktrees WHERE path = ?" err := db.QueryRow(sqlStmt, path).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) if err != nil { ret = nil if errors.Is(err, sql.ErrNoRows) { return } logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return } return}
func CountTrees() (ret int) { sqlStmt := "SELECT COUNT(*) FROM blocktrees WHERE type = 'd'" err := db.QueryRow(sqlStmt).Scan(&ret) if err != nil { if errors.Is(err, sql.ErrNoRows) { return 0 } logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) } return}
func CountBlocks() (ret int) { sqlStmt := "SELECT COUNT(*) FROM blocktrees" err := db.QueryRow(sqlStmt).Scan(&ret) if err != nil { if errors.Is(err, sql.ErrNoRows) { return 0 } logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) } return}
func GetBlockTreeRootByPath(boxID, path string) (ret *BlockTree) { ret = &BlockTree{} sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND path = ? AND type = 'd'" err := db.QueryRow(sqlStmt, boxID, path).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) if err != nil { ret = nil if errors.Is(err, sql.ErrNoRows) { return } logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return } return}
func GetBlockTreeRootByHPath(boxID, hPath string) (ret *BlockTree) { ret = &BlockTree{} hPath = gulu.Str.RemoveInvisible(hPath) sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ? AND type = 'd'" err := db.QueryRow(sqlStmt, boxID, hPath).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) if err != nil { ret = nil if errors.Is(err, sql.ErrNoRows) { return } logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return } return}
func GetBlockTreeRootsByHPath(boxID, hPath string) (ret []*BlockTree) { hPath = gulu.Str.RemoveInvisible(hPath) sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ? AND type = 'd'" rows, err := db.Query(sqlStmt, boxID, hPath) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return } defer rows.Close() for rows.Next() { var block BlockTree if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); err != nil { logging.LogErrorf("query scan field failed: %s", err) return } ret = append(ret, &block) } return}
func GetBlockTreeByHPathPreferredParentID(boxID, hPath, preferredParentID string) (ret *BlockTree) { hPath = gulu.Str.RemoveInvisible(hPath) var roots []*BlockTree sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ? AND parent_id = ? LIMIT 1" rows, err := db.Query(sqlStmt, boxID, hPath, preferredParentID) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return } defer rows.Close() for rows.Next() { var block BlockTree if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); err != nil { logging.LogErrorf("query scan field failed: %s", err) return } if "" == preferredParentID { ret = &block return } roots = append(roots, &block) }
if 1 > len(roots) { return }
for _, root := range roots { if root.ID == preferredParentID { ret = root return } } ret = roots[0] return}
func ExistBlockTree(id string) bool { sqlStmt := "SELECT COUNT(*) FROM blocktrees WHERE id = ?" var count int err := db.QueryRow(sqlStmt, id).Scan(&count) if err != nil { if errors.Is(err, sql.ErrNoRows) { return false } logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return false } return 0 < count}
func ExistBlockTrees(ids []string) (ret map[string]bool) { ret = map[string]bool{} if 1 > len(ids) { return }
for _, id := range ids { ret[id] = false }
sqlStmt := "SELECT id FROM blocktrees WHERE id IN ('" + strings.Join(ids, "','") + "')" rows, err := db.Query(sqlStmt) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return } defer rows.Close() for rows.Next() { var id string if err = rows.Scan(&id); err != nil { logging.LogErrorf("query scan field failed: %s", err) return } ret[id] = true } return}
func GetBlockTrees(ids []string) (ret map[string]*BlockTree) { ret = map[string]*BlockTree{} if 1 > len(ids) { return }
stmtBuf := bytes.Buffer{} stmtBuf.WriteString("SELECT * FROM blocktrees WHERE id IN (") for i := range ids { stmtBuf.WriteString("?") if i == len(ids)-1 { stmtBuf.WriteString(")") } else { stmtBuf.WriteString(",") } } var args []any for _, id := range ids { args = append(args, id) } stmt := stmtBuf.String() rows, err := db.Query(stmt, args...) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", stmt, err) return } defer rows.Close() for rows.Next() { var block BlockTree if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); err != nil { logging.LogErrorf("query scan field failed: %s", err) return } ret[block.ID] = &block } return}
func GetBlockTree(id string) (ret *BlockTree) { if "" == id { return }
ret = &BlockTree{} sqlStmt := "SELECT * FROM blocktrees WHERE id = ?" err := db.QueryRow(sqlStmt, id).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) if err != nil { ret = nil if errors.Is(err, sql.ErrNoRows) { return } logging.LogErrorf("sql query [%s] failed: %v\n\t%s", sqlStmt, err, logging.ShortStack()) return } return}
func SetBlockTreePath(tree *parse.Tree) { RemoveBlockTreesByRootID(tree.ID) IndexBlockTree(tree)}
func RemoveBlockTreesByRootID(rootID string) { sqlStmt := "DELETE FROM blocktrees WHERE root_id = ?" _, err := db.Exec(sqlStmt, rootID) if err != nil { logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return }}
func CountBlockTreesByPathPrefix(pathPrefix string) (ret int) { sqlStmt := "SELECT COUNT(*) FROM blocktrees WHERE path LIKE ?" err := db.QueryRow(sqlStmt, pathPrefix+"%").Scan(&ret) if err != nil { if errors.Is(err, sql.ErrNoRows) { return 0 } logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) } return}
func GetBlockTreesByPathPrefix(pathPrefix string) (ret []*BlockTree) { sqlStmt := "SELECT * FROM blocktrees WHERE path LIKE ?" rows, err := db.Query(sqlStmt, pathPrefix+"%") if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return } defer rows.Close() for rows.Next() { var block BlockTree if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); err != nil { logging.LogErrorf("query scan field failed: %s", err) return } ret = append(ret, &block) } return}
func GetBlockTreesByRootID(rootID string) (ret []*BlockTree) { sqlStmt := "SELECT * FROM blocktrees WHERE root_id = ?" rows, err := db.Query(sqlStmt, rootID) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return } defer rows.Close() for rows.Next() { var block BlockTree if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); err != nil { logging.LogErrorf("query scan field failed: %s", err) return } ret = append(ret, &block) } return}
func RemoveBlockTreesByPathPrefix(pathPrefix string) { sqlStmt := "DELETE FROM blocktrees WHERE path LIKE ?" _, err := db.Exec(sqlStmt, pathPrefix+"%") if err != nil { logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return }}
func GetBlockTreesByBoxID(boxID string) (ret []*BlockTree) { sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ?" rows, err := db.Query(sqlStmt, boxID) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return } defer rows.Close() for rows.Next() { var block BlockTree if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); err != nil { logging.LogErrorf("query scan field failed: %s", err) return } ret = append(ret, &block) } return}
func RemoveBlockTreesByBoxID(boxID string) (ids []string) { sqlStmt := "SELECT id FROM blocktrees WHERE box_id = ?" rows, err := db.Query(sqlStmt, boxID) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return } defer rows.Close() for rows.Next() { var id string if err = rows.Scan(&id); err != nil { logging.LogErrorf("query scan field failed: %s", err) return } ids = append(ids, id) }
sqlStmt = "DELETE FROM blocktrees WHERE box_id = ?" _, err = db.Exec(sqlStmt, boxID) if err != nil { logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return } return}
func RemoveBlockTreesByIDs(ids []string) { if 1 > len(ids) { return }
sqlStmt := "DELETE FROM blocktrees WHERE id IN ('" + strings.Join(ids, "','") + "')" _, err := db.Exec(sqlStmt) if err != nil { logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return }}
func RemoveBlockTree(id string) { sqlStmt := "DELETE FROM blocktrees WHERE id = ?" _, err := db.Exec(sqlStmt, id) if err != nil { logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return }}
var indexBlockTreeLock = sync.Mutex{}
func IndexBlockTree(tree *parse.Tree) { var changedNodes []*ast.Node ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { if !entering || !n.IsBlock() || "" == n.ID { return ast.WalkContinue }
changedNodes = append(changedNodes, n) return ast.WalkContinue })
if 1 > len(changedNodes) { return }
indexBlockTreeLock.Lock() defer indexBlockTreeLock.Unlock()
tx, err := db.Begin() if err != nil { logging.LogErrorf("begin transaction failed: %s", err) return }
execInsertBlocktrees(tx, tree, changedNodes)}
func UpsertBlockTree(tree *parse.Tree) { oldBts := map[string]*BlockTree{} bts := GetBlockTreesByRootID(tree.ID) for _, bt := range bts { oldBts[bt.ID] = bt }
var changedNodes []*ast.Node ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { if !entering || !n.IsBlock() || "" == n.ID { return ast.WalkContinue }
if oldBt, found := oldBts[n.ID]; found { if oldBt.Updated != n.IALAttr("updated") || oldBt.Type != TypeAbbr(n.Type.String()) || oldBt.Path != tree.Path || oldBt.BoxID != tree.Box || oldBt.HPath != tree.HPath { children := ChildBlockNodes(n) // 需要考虑子块，因为一些操作（比如移动块）后需要同时更新子块 changedNodes = append(changedNodes, children...) } } else { children := ChildBlockNodes(n) changedNodes = append(changedNodes, children...) } return ast.WalkContinue })
if 1 > len(changedNodes) { return }
ids := bytes.Buffer{} for i, n := range changedNodes { ids.WriteString("'") ids.WriteString(n.ID) ids.WriteString("'") if i < len(changedNodes)-1 { ids.WriteString(",") } }
indexBlockTreeLock.Lock() defer indexBlockTreeLock.Unlock()
tx, err := db.Begin() if err != nil { logging.LogErrorf("begin transaction failed: %s", err) return }
sqlStmt := "DELETE FROM blocktrees WHERE id IN (" + ids.String() + ")" _, err = tx.Exec(sqlStmt) if err != nil { tx.Rollback() logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return }
execInsertBlocktrees(tx, tree, changedNodes)}
func execInsertBlocktrees(tx *sql.Tx, tree *parse.Tree, changedNodes []*ast.Node) { sqlStmt := "INSERT INTO blocktrees (id, root_id, parent_id, box_id, path, hpath, updated, type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" stmt, err := tx.Prepare(sqlStmt) if err != nil { tx.Rollback() logging.LogErrorf("prepare statement [%s] failed: %s", sqlStmt, err) return } defer stmt.Close()
for _, n := range changedNodes { var parentID string if nil != n.Parent { parentID = n.Parent.ID } if _, err = tx.Exec(sqlStmt, n.ID, tree.ID, parentID, tree.Box, tree.Path, tree.HPath, n.IALAttr("updated"), TypeAbbr(n.Type.String())); err != nil { tx.Rollback() logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return } } if err = tx.Commit(); err != nil { logging.LogErrorf("commit transaction failed: %s", err) }}
func InitBlockTree(force bool) { err := initDatabase(force) if err != nil { logging.LogErrorf("init database failed: %s", err) os.Exit(logging.ExitCodeUnavailableDatabase) return } return}
func CeilTreeCount(count int) int { if 100 > count { return 100 }
for i := 1; i < 40; i++ { if count < i*500 { return i * 500 } } return 500*40 + 1}
func CeilBlockCount(count int) int { if 5000 > count { return 5000 }
for i := 1; i < 100; i++ { if count < i*10000 { return i * 10000 } } return 10000*100 + 1}