Something went wrong. Try again.
A privacy-first, self-hosted, fully open source personal knowledge management software, written in typescript and golang. (PERSONAL FORK)
Something went wrong. Try again.
Go
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542// SiYuan - Refactor your thinking// Copyright (c) 2020-present, b3log.org//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as published by// the Free Software Foundation, either version 3 of the License, or// (at your option) any later version.//// This program is distributed in the hope that it will be useful,// but WITHOUT ANY WARRANTY; without even the implied warranty of// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import ( "bytes" "crypto/sha256" "database/sql" "fmt" "os" "path" "path/filepath" "strings" "sync"
"github.com/88250/gulu" "github.com/88250/lute/parse" "github.com/emirpasic/gods/sets/hashset" ignore "github.com/sabhiram/go-gitignore" "github.com/siyuan-note/eventbus" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util")
var luteEngine = util.NewLute()
func init() { luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md，但是要保留 span block ial}
const ( BlocksInsert = "INSERT INTO blocks (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s" BlocksFTSInsert = "INSERT INTO blocks_fts (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s" BlocksFTSCaseInsensitiveInsert = "INSERT INTO blocks_fts_case_insensitive (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s" BlocksPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
SpansInsert = "INSERT INTO spans (id, block_id, root_id, box, path, content, markdown, type, ial) VALUES %s" SpansPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
AssetsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)" AttributesPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?)" RefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" FileAnnotationRefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)")
func insertBlocks(tx *sql.Tx, blocks []*Block, context map[string]interface{}) (err error) { if 1 > len(blocks) { return }
var bulk []*Block for _, block := range blocks { bulk = append(bulk, block) if 512 > len(bulk) { continue }
if err = insertBlocks0(tx, bulk, context); err != nil { return } bulk = []*Block{} } if 0 < len(bulk) { if err = insertBlocks0(tx, bulk, context); err != nil { return } } return}
func insertBlocks0(tx *sql.Tx, bulk []*Block, context map[string]interface{}) (err error) { valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(BlocksPlaceholder, "?")) hashBuf := bytes.Buffer{} for _, b := range bulk { valueStrings = append(valueStrings, BlocksPlaceholder) valueArgs = append(valueArgs, b.ID) valueArgs = append(valueArgs, b.ParentID) valueArgs = append(valueArgs, b.RootID) valueArgs = append(valueArgs, b.Hash) valueArgs = append(valueArgs, b.Box) valueArgs = append(valueArgs, b.Path) valueArgs = append(valueArgs, b.HPath) valueArgs = append(valueArgs, b.Name) valueArgs = append(valueArgs, b.Alias) valueArgs = append(valueArgs, b.Memo) valueArgs = append(valueArgs, b.Tag) valueArgs = append(valueArgs, b.Content) valueArgs = append(valueArgs, b.FContent) valueArgs = append(valueArgs, b.Markdown) valueArgs = append(valueArgs, b.Length) valueArgs = append(valueArgs, b.Type) valueArgs = append(valueArgs, b.SubType) valueArgs = append(valueArgs, b.IAL) valueArgs = append(valueArgs, b.Sort) valueArgs = append(valueArgs, b.Created) valueArgs = append(valueArgs, b.Updated) putBlockCache(b)
hashBuf.WriteString(b.Hash) }
stmt := fmt.Sprintf(BlocksInsert, strings.Join(valueStrings, ",")) if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil { return } hashBuf.WriteString("blocks") evtHash := fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7] // 使用下面的 EvtSQLInsertBlocksFTS 就可以了 //eventbus.Publish(eventbus.EvtSQLInsertBlocks, context, current, total, len(bulk), evtHash)
stmt = fmt.Sprintf(BlocksFTSInsert, strings.Join(valueStrings, ",")) if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil { return }
if !caseSensitive { stmt = fmt.Sprintf(BlocksFTSCaseInsensitiveInsert, strings.Join(valueStrings, ",")) if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil { return } } hashBuf.WriteString("fts") evtHash = fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7] eventbus.Publish(eventbus.EvtSQLInsertBlocksFTS, context, len(bulk), evtHash) return}
func insertAttributes(tx *sql.Tx, attributes []*Attribute) (err error) { if 1 > len(attributes) { return }
var bulk []*Attribute for _, attr := range attributes { bulk = append(bulk, attr) if 512 > len(bulk) { continue }
if err = insertAttribute0(tx, bulk); err != nil { return } bulk = []*Attribute{} } if 0 < len(bulk) { if err = insertAttribute0(tx, bulk); err != nil { return } } return}
func insertAttribute0(tx *sql.Tx, bulk []*Attribute) (err error) { if 1 > len(bulk) { return }
valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AttributesPlaceholder, "?")) for _, attr := range bulk { valueStrings = append(valueStrings, AttributesPlaceholder) valueArgs = append(valueArgs, attr.ID) valueArgs = append(valueArgs, attr.Name) valueArgs = append(valueArgs, attr.Value) valueArgs = append(valueArgs, attr.Type) valueArgs = append(valueArgs, attr.BlockID) valueArgs = append(valueArgs, attr.RootID) valueArgs = append(valueArgs, attr.Box) valueArgs = append(valueArgs, attr.Path) } stmt := fmt.Sprintf("INSERT INTO attributes (id, name, value, type, block_id, root_id, box, path) VALUES %s", strings.Join(valueStrings, ",")) err = prepareExecInsertTx(tx, stmt, valueArgs) return}
func insertAssets(tx *sql.Tx, assets []*Asset) (err error) { if 1 > len(assets) { return }
var bulk []*Asset for _, asset := range assets { bulk = append(bulk, asset) if 512 > len(bulk) { continue }
if err = insertAsset0(tx, bulk); err != nil { return } bulk = []*Asset{} } if 0 < len(bulk) { if err = insertAsset0(tx, bulk); err != nil { return } } return}
func insertAsset0(tx *sql.Tx, bulk []*Asset) (err error) { if 1 > len(bulk) { return }
valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AssetsPlaceholder, "?")) for _, asset := range bulk { valueStrings = append(valueStrings, AssetsPlaceholder) valueArgs = append(valueArgs, asset.ID) valueArgs = append(valueArgs, asset.BlockID) valueArgs = append(valueArgs, asset.RootID) valueArgs = append(valueArgs, asset.Box) valueArgs = append(valueArgs, asset.DocPath) valueArgs = append(valueArgs, asset.Path) valueArgs = append(valueArgs, asset.Name) valueArgs = append(valueArgs, asset.Title) valueArgs = append(valueArgs, asset.Hash) } stmt := fmt.Sprintf("INSERT INTO assets (id, block_id, root_id, box, docpath, path, name, title, hash) VALUES %s", strings.Join(valueStrings, ",")) err = prepareExecInsertTx(tx, stmt, valueArgs) return}
func insertSpans(tx *sql.Tx, spans []*Span) (err error) { if 1 > len(spans) { return }
var bulk []*Span for _, span := range spans { bulk = append(bulk, span) if 512 > len(bulk) { continue }
if err = insertSpans0(tx, bulk); err != nil { return } bulk = []*Span{} } if 0 < len(bulk) { if err = insertSpans0(tx, bulk); err != nil { return } } return}
func insertSpans0(tx *sql.Tx, bulk []*Span) (err error) { if 1 > len(bulk) { return }
valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(SpansPlaceholder, "?")) for _, span := range bulk { valueStrings = append(valueStrings, SpansPlaceholder) valueArgs = append(valueArgs, span.ID) valueArgs = append(valueArgs, span.BlockID) valueArgs = append(valueArgs, span.RootID) valueArgs = append(valueArgs, span.Box) valueArgs = append(valueArgs, span.Path) valueArgs = append(valueArgs, span.Content) valueArgs = append(valueArgs, span.Markdown) valueArgs = append(valueArgs, span.Type) valueArgs = append(valueArgs, span.IAL) } stmt := fmt.Sprintf(SpansInsert, strings.Join(valueStrings, ",")) err = prepareExecInsertTx(tx, stmt, valueArgs) return}
func insertBlockRefs(tx *sql.Tx, refs []*Ref) (err error) { if 1 > len(refs) { return }
var bulk []*Ref for _, ref := range refs { bulk = append(bulk, ref) if 512 > len(bulk) { continue }
if err = insertRefs0(tx, bulk); err != nil { return } bulk = []*Ref{} } if 0 < len(bulk) { if err = insertRefs0(tx, bulk); err != nil { return } } return}
func insertRefs0(tx *sql.Tx, bulk []*Ref) (err error) { if 1 > len(bulk) { return }
valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(RefsPlaceholder, "?")) for _, ref := range bulk { valueStrings = append(valueStrings, RefsPlaceholder) valueArgs = append(valueArgs, ref.ID) valueArgs = append(valueArgs, ref.DefBlockID) valueArgs = append(valueArgs, ref.DefBlockParentID) valueArgs = append(valueArgs, ref.DefBlockRootID) valueArgs = append(valueArgs, ref.DefBlockPath) valueArgs = append(valueArgs, ref.BlockID) valueArgs = append(valueArgs, ref.RootID) valueArgs = append(valueArgs, ref.Box) valueArgs = append(valueArgs, ref.Path) valueArgs = append(valueArgs, ref.Content) valueArgs = append(valueArgs, ref.Markdown) valueArgs = append(valueArgs, ref.Type)
putRefCache(ref) } stmt := fmt.Sprintf("INSERT INTO refs (id, def_block_id, def_block_parent_id, def_block_root_id, def_block_path, block_id, root_id, box, path, content, markdown, type) VALUES %s", strings.Join(valueStrings, ",")) err = prepareExecInsertTx(tx, stmt, valueArgs) return}
func insertFileAnnotationRefs(tx *sql.Tx, refs []*FileAnnotationRef) (err error) { if 1 > len(refs) { return }
var bulk []*FileAnnotationRef for _, ref := range refs { bulk = append(bulk, ref) if 512 > len(bulk) { continue }
if err = insertFileAnnotationRefs0(tx, bulk); err != nil { return } bulk = []*FileAnnotationRef{} } if 0 < len(bulk) { if err = insertFileAnnotationRefs0(tx, bulk); err != nil { return } } return}
func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error) { if 1 > len(bulk) { return }
valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(FileAnnotationRefsPlaceholder, "?")) for _, ref := range bulk { valueStrings = append(valueStrings, FileAnnotationRefsPlaceholder) valueArgs = append(valueArgs, ref.ID) valueArgs = append(valueArgs, ref.FilePath) valueArgs = append(valueArgs, ref.AnnotationID) valueArgs = append(valueArgs, ref.BlockID) valueArgs = append(valueArgs, ref.RootID) valueArgs = append(valueArgs, ref.Box) valueArgs = append(valueArgs, ref.Path) valueArgs = append(valueArgs, ref.Content) valueArgs = append(valueArgs, ref.Type) } stmt := fmt.Sprintf("INSERT INTO file_annotation_refs (id, file_path, annotation_id, block_id, root_id, box, path, content, type) VALUES %s", strings.Join(valueStrings, ",")) err = prepareExecInsertTx(tx, stmt, valueArgs) return}
func indexTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { blocks, spans, assets, attributes := fromTree(tree.Root, tree) refs, fileAnnotationRefs := refsFromTree(tree) err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs) return}
func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { oldBlockHashes := queryBlockHashes(tree.ID) blocks, spans, assets, attributes := fromTree(tree.Root, tree) newBlockHashes := map[string]string{} for _, block := range blocks { newBlockHashes[block.ID] = block.Hash } unChanges := hashset.New() var toRemoves []string for id, hash := range oldBlockHashes { if newHash, ok := newBlockHashes[id]; ok { if newHash == hash { unChanges.Add(id) } } else { toRemoves = append(toRemoves, id) } } tmp := blocks[:0] for _, b := range blocks { if !unChanges.Contains(b.ID) { tmp = append(tmp, b) } } blocks = tmp for _, b := range blocks { toRemoves = append(toRemoves, b.ID) }
if err = deleteBlocksByIDs(tx, toRemoves); err != nil { return }
if err = deleteSpansByRootID(tx, tree.ID); err != nil { return } if err = deleteAssetsByRootID(tx, tree.ID); err != nil { return } if err = deleteAttributesByRootID(tx, tree.ID); err != nil { return } if err = deleteRefsByPathTx(tx, tree.Box, tree.Path); err != nil { return } if err = deleteFileAnnotationRefsByPathTx(tx, tree.Box, tree.Path); err != nil { return }
refs, fileAnnotationRefs := refsFromTree(tree) if err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs); err != nil { return } return err}
func insertTree0(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}, blocks []*Block, spans []*Span, assets []*Asset, attributes []*Attribute, refs []*Ref, fileAnnotationRefs []*FileAnnotationRef) (err error) { if ignoreLines := getIndexIgnoreLines(); 0 < len(ignoreLines) { // Support ignore index https://github.com/siyuan-note/siyuan/issues/9198 matcher := ignore.CompileIgnoreLines(ignoreLines...) if matcher.MatchesPath("/" + path.Join(tree.Box, tree.Path)) { return } }
if err = insertBlocks(tx, blocks, context); err != nil { return }
if err = insertBlockRefs(tx, refs); err != nil { return } if err = insertFileAnnotationRefs(tx, fileAnnotationRefs); err != nil { return }
if 0 < len(spans) { // 移除文档标签，否则会重复添加 https://github.com/siyuan-note/siyuan/issues/3723 if err = deleteSpansByRootID(tx, tree.Root.ID); err != nil { return } if err = insertSpans(tx, spans); err != nil { return } } if err = insertAssets(tx, assets); err != nil { return } if err = insertAttributes(tx, attributes); err != nil { return } return}
var ( IndexIgnoreCached bool indexIgnore []string indexIgnoreLock = sync.Mutex{})
func getIndexIgnoreLines() (ret []string) { // Support ignore index https://github.com/siyuan-note/siyuan/issues/9198
if IndexIgnoreCached { return indexIgnore }
indexIgnoreLock.Lock() defer indexIgnoreLock.Unlock()
IndexIgnoreCached = true indexIgnorePath := filepath.Join(util.DataDir, ".siyuan", "indexignore") err := os.MkdirAll(filepath.Dir(indexIgnorePath), 0755) if err != nil { return } if !gulu.File.IsExist(indexIgnorePath) { if err = gulu.File.WriteFileSafer(indexIgnorePath, nil, 0644); err != nil { logging.LogErrorf("create indexignore [%s] failed: %s", indexIgnorePath, err) return } } data, err := os.ReadFile(indexIgnorePath) if err != nil { logging.LogErrorf("read indexignore [%s] failed: %s", indexIgnorePath, err) return } dataStr := string(data) dataStr = strings.ReplaceAll(dataStr, "\r\n", "\n") ret = strings.Split(dataStr, "\n")
ret = gulu.Str.RemoveDuplicatedElem(ret) if 0 < len(ret) && "" == ret[0] { ret = ret[1:] } indexIgnore = nil for _, line := range ret { indexIgnore = append(indexIgnore, line) } return}