Something went wrong. Try again.
A privacy-first, self-hosted, fully open source personal knowledge management software, written in typescript and golang. (PERSONAL FORK)
Something went wrong. Try again.
Go
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740// SiYuan - Refactor your thinking// Copyright (c) 2020-present, b3log.org//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as published by// the Free Software Foundation, either version 3 of the License, or// (at your option) any later version.//// This program is distributed in the hope that it will be useful,// but WITHOUT ANY WARRANTY; without even the implied warranty of// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.
package model
import ( "bytes" "errors" "fmt" "io/fs" "mime" "net/http" "net/url" "os" "path" "path/filepath" "sort" "strings" "time"
"github.com/88250/go-humanize" "github.com/88250/gulu" "github.com/88250/lute/ast" "github.com/88250/lute/editor" "github.com/88250/lute/html" "github.com/88250/lute/parse" "github.com/disintegration/imaging" "github.com/gabriel-vasile/mimetype" "github.com/siyuan-note/filelock" "github.com/siyuan-note/httpclient" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/av" "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util")
func GetAssetImgSize(assetPath string) (width, height int) { absPath, err := GetAssetAbsPath(assetPath) if err != nil { logging.LogErrorf("get asset [%s] abs path failed: %s", assetPath, err) return }
img, err := imaging.Open(absPath) if err != nil { logging.LogErrorf("open asset image [%s] failed: %s", absPath, err) return }
width = img.Bounds().Dx() height = img.Bounds().Dy() return}
func GetAssetPathByHash(hash string) string { assetHash := cache.GetAssetHash(hash) if nil == assetHash { sqlAsset := sql.QueryAssetByHash(hash) if nil == sqlAsset { return "" } cache.SetAssetHash(sqlAsset.Hash, sqlAsset.Path) return sqlAsset.Path } return assetHash.Path}
func HandleAssetsRemoveEvent(assetAbsPath string) { if !filelock.IsExist(assetAbsPath) { return } if gulu.File.IsDir(assetAbsPath) { return } if filelock.IsHidden(assetAbsPath) { return } if strings.HasSuffix(assetAbsPath, ".tmp") { return }
removeIndexAssetContent(assetAbsPath) removeAssetThumbnail(assetAbsPath)
hash, err := util.GetEtag(assetAbsPath) if nil != err { logging.LogErrorf("calc asset [%s] hash failed: %s", assetAbsPath, err) } else { cache.RemoveAssetHash(hash) }}
func HandleAssetsChangeEvent(assetAbsPath string) { if !filelock.IsExist(assetAbsPath) { return } if gulu.File.IsDir(assetAbsPath) { return } if filelock.IsHidden(assetAbsPath) { return } if strings.HasSuffix(assetAbsPath, ".tmp") { return }
indexAssetContent(assetAbsPath) removeAssetThumbnail(assetAbsPath)
hash, err := util.GetEtag(assetAbsPath) if nil != err { logging.LogErrorf("calc asset [%s] hash failed: %s", assetAbsPath, err) } else { p := strings.TrimPrefix(assetAbsPath, util.DataDir) p = strings.TrimPrefix(filepath.ToSlash(p), "/") cache.SetAssetHash(hash, p) }}
func removeAssetThumbnail(assetAbsPath string) { if util.IsCompressibleAssetImage(assetAbsPath) { p := filepath.ToSlash(assetAbsPath) idx := strings.Index(p, "assets/") if -1 == idx { return } thumbnailPath := filepath.Join(util.TempDir, "thumbnails", "assets", p[idx+7:]) os.RemoveAll(thumbnailPath) }}
func NeedGenerateAssetsThumbnail(sourceImgPath string) bool { info, err := os.Stat(sourceImgPath) if err != nil { return false } if info.IsDir() { return false } return info.Size() > 1024*1024}
func GenerateAssetsThumbnail(sourceImgPath, resizedImgPath string) (err error) { start := time.Now() img, err := imaging.Open(sourceImgPath) if err != nil { return }
// 获取原图宽高 originalWidth := img.Bounds().Dx() originalHeight := img.Bounds().Dy()
// 固定最大宽度为 520，计算缩放比例 maxWidth := 520 scale := float64(maxWidth) / float64(originalWidth)
// 按比例计算新的宽高 newWidth := maxWidth newHeight := int(float64(originalHeight) * scale)
// 缩放图片 resizedImg := imaging.Resize(img, newWidth, newHeight, imaging.Lanczos)
// 保存缩放后的图片 err = os.MkdirAll(filepath.Dir(resizedImgPath), 0755) if err != nil { return } err = imaging.Save(resizedImg, resizedImgPath) if err != nil { return } logging.LogDebugf("generated thumbnail image [%s] to [%s], cost [%d]ms", sourceImgPath, resizedImgPath, time.Since(start).Milliseconds()) return}
func DocImageAssets(rootID string) (ret []string, err error) { tree, err := LoadTreeByBlockID(rootID) if err != nil { return }
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { if !entering { return ast.WalkContinue } if ast.NodeImage == n.Type { linkDest := n.ChildByType(ast.NodeLinkDest) dest := linkDest.Tokens if 1 > len(dest) { // 双击打开图片不对 https://github.com/siyuan-note/siyuan/issues/5876 return ast.WalkContinue } ret = append(ret, gulu.Str.FromBytes(dest)) } return ast.WalkContinue }) return}
func DocAssets(rootID string) (ret []string, err error) { tree, err := LoadTreeByBlockID(rootID) if err != nil { return }
ret = getAssetsLinkDests(tree.Root, false) return}
func NetAssets2LocalAssets(rootID string, onlyImg bool, originalURL string) (err error) { tree, err := LoadTreeByBlockID(rootID) if err != nil { return }
docDirLocalPath := filepath.Join(util.DataDir, tree.Box, path.Dir(tree.Path)) assetsDirPath := getAssetsDir(filepath.Join(util.DataDir, tree.Box), docDirLocalPath) if !gulu.File.IsExist(assetsDirPath) { if err = os.MkdirAll(assetsDirPath, 0755); err != nil { return } }
err = netAssets2LocalAssets0(tree, onlyImg, originalURL, assetsDirPath, true) return}
func netAssets2LocalAssets0(tree *parse.Tree, onlyImg bool, originalURL string, assetsDirPath string, needWriteTree bool) (err error) { var files int var size int64 msgId := gulu.Rand.String(7)
browserClient := util.NewCustomReqClient() // 自定义了 TLS 指纹，增加下载成功率
forbiddenCount := 0 destNodes := getRemoteAssetsLinkDestsInTree(tree, onlyImg) assetsMap := map[string]string{} for _, destNode := range destNodes { dests := getRemoteAssetsLinkDests(destNode, onlyImg) if 1 > len(dests) { continue }
for _, dest := range dests { if strings.HasPrefix(strings.ToLower(dest), "file://") { // 处理本地文件链接 u := dest[7:] unescaped, _ := url.PathUnescape(u) if unescaped != u { // `Convert network images/assets to local` supports URL-encoded local file names https://github.com/siyuan-note/siyuan/issues/9929 u = unescaped } if strings.Contains(u, ":") { u = strings.TrimPrefix(u, "/") } if strings.Contains(u, "?") { u = u[:strings.Index(u, "?")] }
if !gulu.File.IsExist(u) { logging.LogErrorf("local file asset [%s] not exist", u) continue }
if gulu.File.IsDir(u) { logging.LogWarnf("ignore converting directory path [%s] to local asset", u) continue }
if util.IsSensitivePath(u) { logging.LogWarnf("ignore converting sensitive path [%s] to local asset", u) continue }
name := assetsMap[u] if "" != name { setAssetsLinkDest(destNode, dest, "assets/"+name) continue }
name = filepath.Base(u) name = util.FilterUploadFileName(name) name = "network-asset-" + name name = util.AssetName(name, ast.NewNodeID()) writePath := filepath.Join(assetsDirPath, name) if err = filelock.Copy(u, writePath); err != nil { logging.LogErrorf("copy [%s] to [%s] failed: %s", u, writePath, err) continue }
setAssetsLinkDest(destNode, dest, "assets/"+name) assetsMap[u] = name files++ size += gulu.File.GetFileSize(writePath) continue }
if strings.HasPrefix(strings.ToLower(dest), "https://") || strings.HasPrefix(strings.ToLower(dest), "http://") || strings.HasPrefix(dest, "//") { if strings.HasPrefix(dest, "//") { // `Convert network images to local` supports `//` https://github.com/siyuan-note/siyuan/issues/10598 dest = "https:" + dest }
u := dest if strings.Contains(u, "qpic.cn") { // 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/5052 if strings.Contains(u, "http://") { u = strings.Replace(u, "http://", "https://", 1) }
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/6431 // 下面这部分需要注释掉，否则会导致响应 400 //if strings.HasSuffix(u, "/0") { // u = strings.Replace(u, "/0", "/640", 1) //} else if strings.Contains(u, "/0?") { // u = strings.Replace(u, "/0?", "/640?", 1) //} }
name := assetsMap[u] if "" != name { setAssetsLinkDest(destNode, dest, "assets/"+name) continue }
displayU := u if 64 < len(displayU) { displayU = displayU[:64] + "..." }
util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), displayU), 15000) request := browserClient.R() request.SetRetryCount(1).SetRetryFixedInterval(3 * time.Second) if "" != originalURL { request.SetHeader("Referer", originalURL) // 改进浏览器剪藏扩展转换本地图片成功率 https://github.com/siyuan-note/siyuan/issues/7464 } resp, reqErr := request.Get(u) if nil != reqErr { logging.LogErrorf("download network asset [%s] failed: %s", u, reqErr) continue } if http.StatusForbidden == resp.StatusCode || http.StatusUnauthorized == resp.StatusCode { forbiddenCount++ } if strings.Contains(strings.ToLower(resp.GetContentType()), "text/html") { // 忽略超链接网页 `Convert network assets to local` no longer process webpage https://github.com/siyuan-note/siyuan/issues/9965 continue } if 200 != resp.StatusCode { logging.LogErrorf("download network asset [%s] failed: %d", u, resp.StatusCode) continue }
if 1024*1024*96 < resp.ContentLength { logging.LogWarnf("network asset [%s]' size [%s] is large then [96 MB], ignore it", u, humanize.IBytes(uint64(resp.ContentLength))) continue }
data, repErr := resp.ToBytes() if nil != repErr { logging.LogErrorf("download network asset [%s] failed: %s", u, repErr) continue } if strings.Contains(u, "?") { name = u[:strings.Index(u, "?")] name = path.Base(name) } else { name = path.Base(u) } if strings.Contains(name, "#") { name = name[:strings.Index(name, "#")] } name, _ = url.PathUnescape(name) name = util.FilterUploadFileName(name) ext := util.Ext(name) if !util.IsCommonExt(ext) { if mtype := mimetype.Detect(data); nil != mtype { ext = mtype.Extension() name += ext } } if "" == ext && bytes.HasPrefix(data, []byte("<svg ")) && bytes.HasSuffix(data, []byte("</svg>")) { ext = ".svg" name += ext } if "" == ext { contentType := resp.Header.Get("Content-Type") exts, _ := mime.ExtensionsByType(contentType) if 0 < len(exts) { ext = exts[0] name += ext } } name = util.AssetName(name, ast.NewNodeID()) name = "network-asset-" + name writePath := filepath.Join(assetsDirPath, name) if err = filelock.WriteFile(writePath, data); err != nil { logging.LogErrorf("write downloaded network asset [%s] to local asset [%s] failed: %s", u, writePath, err) continue }
setAssetsLinkDest(destNode, dest, "assets/"+name) assetsMap[u] = name files++ size += int64(len(data)) continue } } }
util.PushClearMsg(msgId)
if needWriteTree { if 0 < files { msgId = util.PushMsg(Conf.Language(113), 7000) if err = writeTreeUpsertQueue(tree); err != nil { return } util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(120), files, humanize.BytesCustomCeil(uint64(size), 2)), 5000)
if 0 < forbiddenCount { util.PushErrMsg(fmt.Sprintf(Conf.Language(255), forbiddenCount), 5000) } } else { if 0 < forbiddenCount { util.PushErrMsg(fmt.Sprintf(Conf.Language(255), forbiddenCount), 5000) } else { util.PushMsg(Conf.Language(121), 3000) } } } return}
func SearchAssetsByName(keyword string, exts []string) (ret []*cache.Asset) { ret = []*cache.Asset{} var keywords []string keywords = append(keywords, keyword) if "" != keyword { keywords = append(keywords, strings.Split(keyword, " ")...) } pathHitCount := map[string]int{} filterByExt := 0 < len(exts) matchedAssets := cache.FilterAssets(func(path string, asset *cache.Asset) bool {
// 扩展名过滤 if filterByExt { ext := filepath.Ext(asset.HName) includeExt := false for _, e := range exts { if strings.ToLower(ext) == strings.ToLower(e) { includeExt = true break } } if !includeExt { return false } }
// 关键字匹配 lowerHName := strings.ToLower(asset.HName) lowerPath := strings.ToLower(asset.Path) var hitNameCount, hitPathCount int for i, k := range keywords { lowerKeyword := strings.ToLower(k) if 0 == i { // 第一个是完全匹配，权重最高 if strings.Contains(lowerHName, lowerKeyword) { hitNameCount += 64 } if strings.Contains(lowerPath, lowerKeyword) { hitPathCount += 64 } }
hitNameCount += strings.Count(lowerHName, lowerKeyword) hitPathCount += strings.Count(lowerPath, lowerKeyword) if 1 > hitNameCount && 1 > hitPathCount { continue } }
// 只返回有匹配的资源 if 1 > hitNameCount+hitPathCount { return false }
// 记录命中次数用于排序 pathHitCount[asset.Path] = hitNameCount + hitPathCount return true })
// 添加高亮 for _, asset := range matchedAssets { hitCount := pathHitCount[asset.Path] hName := asset.HName if hitCount > 0 { _, hName = search.MarkText(asset.HName, strings.Join(keywords, search.TermSep), 64, Conf.Search.CaseSensitive) } ret = append(ret, &cache.Asset{ HName: hName, Path: asset.Path, Updated: asset.Updated, }) }
if 0 < len(pathHitCount) { sort.Slice(ret, func(i, j int) bool { return pathHitCount[ret[i].Path] > pathHitCount[ret[j].Path] }) } else { sort.Slice(ret, func(i, j int) bool { return ret[i].Updated > ret[j].Updated }) }
if Conf.Search.Limit <= len(ret) { ret = ret[:Conf.Search.Limit] } return}
func GetAssetAbsPath(relativePath string) (ret string, err error) { relativePath = strings.TrimSpace(relativePath) if strings.Contains(relativePath, "?") { relativePath = relativePath[:strings.Index(relativePath, "?")] }
// 在全局 assets 路径下搜索 p := filepath.Join(util.DataDir, relativePath) if gulu.File.IsExist(p) { ret = p if !util.IsSubPath(util.WorkspaceDir, ret) { err = fmt.Errorf("[%s] is not sub path of workspace", ret) return } return }
// 在笔记本下搜索 notebooks, err := ListNotebooks() if err != nil { err = errors.New(Conf.Language(0)) return } for _, notebook := range notebooks { notebookAbsPath := filepath.Join(util.DataDir, notebook.ID) filelock.Walk(notebookAbsPath, func(path string, d fs.DirEntry, err error) error { if isSkipFile(d.Name()) { if d.IsDir() { return filepath.SkipDir } return nil } if p := filepath.ToSlash(path); strings.HasSuffix(p, relativePath) { if gulu.File.IsExist(path) { ret = path return fs.SkipAll } } return nil })
if "" != ret { if !util.IsSubPath(util.WorkspaceDir, ret) { err = fmt.Errorf("[%s] is not sub path of workspace", ret) return } return } } return "", errors.New(fmt.Sprintf(Conf.Language(12), relativePath))}
func UploadAssets2Cloud(id string, ignorePushMsg bool) (count int, err error) { if !IsSubscriber() { return }
tree, err := LoadTreeByBlockID(id) if err != nil { return }
node := treenode.GetNodeInTree(tree, id) if nil == node { err = ErrBlockNotFound return }
nodes := []*ast.Node{node} if ast.NodeHeading == node.Type { nodes = append(nodes, treenode.HeadingChildren(node)...) }
var assets []string for _, n := range nodes { assets = append(assets, getAssetsLinkDests(n, false)...) assets = append(assets, getQueryEmbedNodesAssetsLinkDests(n)...) } assets = gulu.Str.RemoveDuplicatedElem(assets) count, err = uploadAssets2Cloud(assets, bizTypeUploadAssets, ignorePushMsg) if err != nil { return } return}
func UploadAssets2CloudByAssetsPaths(assetPaths []string, ignorePushMsg bool) (count int, err error) { if !IsSubscriber() { return }
count, err = uploadAssets2Cloud(assetPaths, bizTypeUploadAssets, ignorePushMsg) return}
const ( bizTypeUploadAssets = "upload-assets" bizTypeExport2Liandi = "export-liandi")
// uploadAssets2Cloud 将资源文件上传到云端图床。func uploadAssets2Cloud(assetPaths []string, bizType string, ignorePushMsg bool) (count int, err error) { var uploadAbsAssets []string for _, assetPath := range assetPaths { var absPath string absPath, err = GetAssetAbsPath(assetPath) if err != nil { logging.LogWarnf("get asset [%s] abs path failed: %s", assetPath, err) return } if "" == absPath { logging.LogErrorf("not found asset [%s]", assetPath) continue }
uploadAbsAssets = append(uploadAbsAssets, absPath) }
uploadAbsAssets = gulu.Str.RemoveDuplicatedElem(uploadAbsAssets) if 1 > len(uploadAbsAssets) { return }
logging.LogInfof("uploading [%d] assets", len(uploadAbsAssets))
var msgId string if !ignorePushMsg { msgId = util.PushMsg(fmt.Sprintf(Conf.Language(27), len(uploadAbsAssets)), 3000) } if loadErr := LoadUploadToken(); nil != loadErr { util.PushMsg(loadErr.Error(), 5000) return }
limitSize := uint64(3 * 1024 * 1024) // 3MB if IsSubscriber() { limitSize = 10 * 1024 * 1024 // 10MB }
// metaType 为服务端 Filemeta.FILEMETA_TYPE，这里只有两个值： // // 5: SiYuan，表示为 SiYuan 上传图床 // 4: Client，表示作为客户端分享发布帖子时上传的文件 var metaType = "5" if bizTypeUploadAssets == bizType { metaType = "5" } else if bizTypeExport2Liandi == bizType { metaType = "4" }
pushErrMsgCount := 0 var completedUploadAssets []string for _, absAsset := range uploadAbsAssets { fi, statErr := os.Stat(absAsset) if nil != statErr { logging.LogErrorf("stat file [%s] failed: %s", absAsset, statErr) return count, statErr }
if limitSize < uint64(fi.Size()) { logging.LogWarnf("file [%s] larger than limit size [%s], ignore uploading it", absAsset, humanize.IBytes(limitSize)) if 3 > pushErrMsgCount { msg := fmt.Sprintf(Conf.Language(247), filepath.Base(absAsset), humanize.IBytes(limitSize)) util.PushErrMsg(msg, 30000) } pushErrMsgCount++ continue }
if !ignorePushMsg { msg := fmt.Sprintf(Conf.Language(27), html.EscapeString(absAsset)) util.PushStatusBar(msg) util.PushUpdateMsg(msgId, msg, 3000) }
requestResult := gulu.Ret.NewResult() request := httpclient.NewCloudFileRequest2m() resp, reqErr := request. SetSuccessResult(requestResult). SetFile("file[]", absAsset). SetCookies(&http.Cookie{Name: "symphony", Value: uploadToken}). SetHeader("meta-type", metaType). SetHeader("biz-type", bizType). Post(util.GetCloudServer() + "/apis/siyuan/upload?ver=" + util.Ver) if nil != reqErr { logging.LogErrorf("upload assets failed: %s", reqErr) return count, ErrFailedToConnectCloudServer }
if 401 == resp.StatusCode { err = errors.New(Conf.Language(31)) return }
if 0 != requestResult.Code { logging.LogErrorf("upload assets failed: %s", requestResult.Msg) err = errors.New(fmt.Sprintf(Conf.Language(94), requestResult.Msg)) return }
absAsset = filepath.ToSlash(absAsset) relAsset := absAsset[strings.Index(absAsset, "assets/"):] completedUploadAssets = append(completedUploadAssets, relAsset) logging.LogInfof("uploaded asset [%s]", relAsset) count++ }
if !ignorePushMsg { util.PushClearMsg(msgId) }
if 0 < len(completedUploadAssets) { logging.LogInfof("uploaded [%d] assets", len(completedUploadAssets)) } return}
func RemoveUnusedAssets() (ret []string) { ret = []string{} var size int64
msgId := util.PushMsg(Conf.Language(100), 30*1000) defer func() { msg := fmt.Sprintf(Conf.Language(91), len(ret), humanize.BytesCustomCeil(uint64(size), 2)) util.PushUpdateMsg(msgId, msg, 7000) }()
unusedAssets := UnusedAssets(false)
historyDir, err := GetHistoryDir(HistoryOpClean) if err != nil { logging.LogErrorf("get history dir failed: %s", err) return }
var hashes []string for _, unusedAsset := range unusedAssets { p := unusedAsset.Item historyPath := filepath.Join(historyDir, p) if p = filepath.Join(util.DataDir, p); filelock.IsExist(p) { if filelock.IsHidden(p) { continue }
if err = filelock.Copy(p, historyPath); err != nil { return }
hash, _ := util.GetEtag(p) hashes = append(hashes, hash) cache.RemoveAssetHash(hash) } }
sql.BatchRemoveAssetsQueue(hashes)
for _, unusedAsset := range unusedAssets { p := unusedAsset.Item absPath := filepath.Join(util.DataDir, p) if filelock.IsExist(absPath) { info, statErr := os.Stat(absPath) if statErr == nil { if info.IsDir() { dirSize, _ := util.SizeOfDirectory(absPath) size += dirSize } else { size += info.Size() } }
if !isFileWatcherAvailable() { HandleAssetsRemoveEvent(absPath) }
if removeErr := filelock.RemoveWithoutFatal(absPath); removeErr != nil { logging.LogErrorf("remove unused asset [%s] failed: %s", absPath, removeErr) util.PushErrMsg(fmt.Sprintf("%s", removeErr), 7000) return }
util.RemoveAssetText(p) } ret = append(ret, absPath) } if 0 < len(ret) { IncSync() }
indexHistoryDir(filepath.Base(historyDir), util.NewLute()) cache.LoadAssets() return}
func RemoveUnusedAsset(p string) (ret string) { absPath := filepath.Join(util.DataDir, p) if !filelock.IsExist(absPath) { return absPath }
historyDir, err := GetHistoryDir(HistoryOpClean) if err != nil { logging.LogErrorf("get history dir failed: %s", err) return }
newP := strings.TrimPrefix(absPath, util.DataDir) historyPath := filepath.Join(historyDir, newP) if filelock.IsExist(absPath) { if err = filelock.Copy(absPath, historyPath); err != nil { return }
hash, _ := util.GetEtag(absPath) sql.BatchRemoveAssetsQueue([]string{hash}) cache.RemoveAssetHash(hash) }
if !isFileWatcherAvailable() { HandleAssetsRemoveEvent(absPath) }
if err = filelock.RemoveWithoutFatal(absPath); err != nil { logging.LogErrorf("remove unused asset [%s] failed: %s", absPath, err) util.PushErrMsg(fmt.Sprintf("%s", err), 7000) return } ret = absPath
util.RemoveAssetText(p)
IncSync()
indexHistoryDir(filepath.Base(historyDir), util.NewLute()) cache.RemoveAsset(p) return}
func RenameAsset(oldPath, newName string) (newPath string, err error) { util.PushEndlessProgress(Conf.Language(110)) defer util.PushClearProgress()
newName = strings.TrimSpace(newName) newName = util.FilterUploadFileName(newName) if path.Base(oldPath) == newName { return } if "" == newName { return }
if !gulu.File.IsValidFilename(newName) { err = errors.New(Conf.Language(151)) return }
newName = util.AssetName(newName+filepath.Ext(oldPath), ast.NewNodeID()) parentDir := path.Dir(oldPath) newPath = path.Join(parentDir, newName) oldAbsPath, getErr := GetAssetAbsPath(oldPath) if getErr != nil { logging.LogErrorf("get asset [%s] abs path failed: %s", oldPath, getErr) return } newAbsPath := filepath.Join(filepath.Dir(oldAbsPath), newName) if err = filelock.Copy(oldAbsPath, newAbsPath); err != nil { logging.LogErrorf("copy asset [%s] failed: %s", oldAbsPath, err) return }
if filelock.IsExist(filepath.Join(util.DataDir, oldPath+".sya")) { // Rename the .sya annotation file when renaming a PDF asset https://github.com/siyuan-note/siyuan/issues/9390 if err = filelock.Copy(filepath.Join(util.DataDir, oldPath+".sya"), filepath.Join(util.DataDir, newPath+".sya")); err != nil { logging.LogErrorf("copy PDF annotation [%s] failed: %s", oldPath+".sya", err) return } }
oldName := path.Base(oldPath)
notebooks, err := ListNotebooks() if err != nil { return }
luteEngine := util.NewLute() for _, notebook := range notebooks { pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32)
for _, paths := range pages { for _, treeAbsPath := range paths { data, readErr := filelock.ReadFile(treeAbsPath) if nil != readErr { logging.LogErrorf("get data [path=%s] failed: %s", treeAbsPath, readErr) err = readErr return }
if !bytes.Contains(data, []byte(oldName)) { continue }
data = bytes.Replace(data, []byte(oldName), []byte(newName), -1) if writeErr := filelock.WriteFile(treeAbsPath, data); nil != writeErr { logging.LogErrorf("write data [path=%s] failed: %s", treeAbsPath, writeErr) err = writeErr return }
cache.RemoveTreeData(util.GetTreeID(treeAbsPath)) p := filepath.ToSlash(strings.TrimPrefix(treeAbsPath, filepath.Join(util.DataDir, notebook.ID))) tree, parseErr := filesys.LoadTreeByData(data, notebook.ID, p, luteEngine) if nil != parseErr { logging.LogWarnf("parse json to tree [%s] failed: %s", treeAbsPath, parseErr) continue }
treenode.UpsertBlockTree(tree) sql.UpsertTreeQueue(tree)
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title")))) } } }
storageAvDir := filepath.Join(util.DataDir, "storage", "av") if gulu.File.IsDir(storageAvDir) { entries, readErr := os.ReadDir(storageAvDir) if nil != readErr { logging.LogErrorf("read dir [%s] failed: %s", storageAvDir, readErr) err = readErr return }
for _, entry := range entries { if !strings.HasSuffix(entry.Name(), ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(entry.Name(), ".json")) { continue }
data, readDataErr := filelock.ReadFile(filepath.Join(util.DataDir, "storage", "av", entry.Name())) if nil != readDataErr { logging.LogErrorf("read file [%s] failed: %s", entry.Name(), readDataErr) err = readDataErr return }
if bytes.Contains(data, []byte(oldPath)) { data = bytes.ReplaceAll(data, []byte(oldPath), []byte(newPath)) if writeDataErr := filelock.WriteFile(filepath.Join(util.DataDir, "storage", "av", entry.Name()), data); nil != writeDataErr { logging.LogErrorf("write file [%s] failed: %s", entry.Name(), writeDataErr) err = writeDataErr return } }
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(entry.Name()))) } }
if ocrText := util.GetAssetText(oldPath); "" != ocrText { // 图片重命名后 ocr-texts.json 需要更新 https://github.com/siyuan-note/siyuan/issues/12974 util.SetAssetText(newPath, ocrText) }
IncSync() return}
type UnusedItem struct { Item string `json:"item"` Name string `json:"name"` ModTime time.Time `json:"-"`}
func UnusedAssets(sorted bool) (ret []*UnusedItem) { defer logging.Recover() ret = []*UnusedItem{}
assetsPathMap, err := allAssetAbsPaths() if err != nil { return } linkDestMap := map[string]bool{} notebooks, err := ListNotebooks() if err != nil { return } luteEngine := util.NewLute() for _, notebook := range notebooks { dests := map[string]bool{}
// 分页加载，优化清理未引用资源内存占用 https://github.com/siyuan-note/siyuan/issues/5200 pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32) for _, paths := range pages { var trees []*parse.Tree for _, localPath := range paths { tree, loadTreeErr := loadTree(localPath, luteEngine) if nil != loadTreeErr { continue } trees = append(trees, tree) } for _, tree := range trees { for _, d := range getAssetsLinkDests(tree.Root, false) { dests[d] = true }
if titleImgPath := treenode.GetDocTitleImgPath(tree.Root); "" != titleImgPath { // 题头图计入 if !util.IsAssetLinkDest([]byte(titleImgPath), false) { continue } dests[titleImgPath] = true } } }
var linkDestFolderPaths, linkDestFilePaths []string for dest := range dests { if !strings.HasPrefix(dest, "assets/") { continue }
if idx := strings.Index(dest, "?"); 0 < idx { // `pdf?page` 资源文件链接会被判定为未引用资源 https://github.com/siyuan-note/siyuan/issues/5649 dest = dest[:idx] }
if "" == assetsPathMap[dest] { continue } if strings.HasSuffix(dest, "/") { linkDestFolderPaths = append(linkDestFolderPaths, dest) } else { linkDestFilePaths = append(linkDestFilePaths, dest) } }
// 排除文件夹链接 var toRemoves []string for asset := range assetsPathMap { for _, linkDestFolder := range linkDestFolderPaths { if strings.HasPrefix(asset, linkDestFolder) { toRemoves = append(toRemoves, asset) } } for _, linkDestPath := range linkDestFilePaths { if strings.HasPrefix(linkDestPath, asset) { toRemoves = append(toRemoves, asset) } } } for _, toRemove := range toRemoves { delete(assetsPathMap, toRemove) }
for _, dest := range linkDestFilePaths { linkDestMap[dest] = true
if strings.HasSuffix(dest, ".pdf") { linkDestMap[dest+".sya"] = true } } }
var toRemoves []string for asset := range assetsPathMap { if strings.HasSuffix(asset, "ocr-texts.json") { // 排除 OCR 结果文本 toRemoves = append(toRemoves, asset) continue }
if strings.HasSuffix(asset, "android-notification-texts.txt") { // 排除 Android 通知文本 toRemoves = append(toRemoves, asset) continue } }
// 排除数据库中引用的资源文件 storageAvDir := filepath.Join(util.DataDir, "storage", "av") if gulu.File.IsDir(storageAvDir) { entries, readErr := os.ReadDir(storageAvDir) if nil != readErr { logging.LogErrorf("read dir [%s] failed: %s", storageAvDir, readErr) err = readErr return }
for _, entry := range entries { if !strings.HasSuffix(entry.Name(), ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(entry.Name(), ".json")) { continue }
data, readDataErr := filelock.ReadFile(filepath.Join(util.DataDir, "storage", "av", entry.Name())) if nil != readDataErr { logging.LogErrorf("read file [%s] failed: %s", entry.Name(), readDataErr) err = readDataErr return }
for asset := range assetsPathMap { if bytes.Contains(data, []byte(asset)) { toRemoves = append(toRemoves, asset) } } } }
for _, toRemove := range toRemoves { delete(assetsPathMap, toRemove) }
dataAssetsAbsPath := util.GetDataAssetsAbsPath() for dest, assetAbsPath := range assetsPathMap { if _, ok := linkDestMap[dest]; ok { continue }
var p string if strings.HasPrefix(dataAssetsAbsPath, assetAbsPath) { p = assetAbsPath[strings.Index(assetAbsPath, "assets"):] } else { p = strings.TrimPrefix(assetAbsPath, filepath.Dir(dataAssetsAbsPath)) } p = filepath.ToSlash(p) if strings.HasPrefix(p, "/") { p = p[1:] } name := util.RemoveID(path.Base(p))
var modTime time.Time if sorted { if info, statErr := os.Stat(assetAbsPath); nil == statErr { modTime = info.ModTime() } }
ret = append(ret, &UnusedItem{Item: p, Name: name, ModTime: modTime}) }
if sorted { sort.Slice(ret, func(i, j int) bool { if !ret[i].ModTime.Equal(ret[j].ModTime) { return ret[i].ModTime.After(ret[j].ModTime) } return ret[i].Item > ret[j].Item }) } return}
func MissingAssets() (ret []*UnusedItem) { defer logging.Recover() ret = []*UnusedItem{}
assetsPathMap, err := allAssetAbsPaths() if err != nil { return } notebooks, err := ListNotebooks() if err != nil { return } luteEngine := util.NewLute() for _, notebook := range notebooks { if notebook.Closed { continue }
dests := map[string]bool{} pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32) for _, paths := range pages { var trees []*parse.Tree for _, localPath := range paths { tree, loadTreeErr := loadTree(localPath, luteEngine) if nil != loadTreeErr { continue } trees = append(trees, tree) } for _, tree := range trees { for _, d := range getAssetsLinkDests(tree.Root, false) { dests[d] = true }
if titleImgPath := treenode.GetDocTitleImgPath(tree.Root); "" != titleImgPath { // 题头图计入 if !util.IsAssetLinkDest([]byte(titleImgPath), false) { continue } dests[titleImgPath] = true } } }
for dest := range dests { if !strings.HasPrefix(dest, "assets/") { continue }
if idx := strings.Index(dest, "?"); 0 < idx { dest = dest[:idx] }
if strings.HasSuffix(dest, "/") { continue }
if strings.Contains(strings.ToLower(dest), ".pdf/") { if idx := strings.LastIndex(dest, "/"); -1 < idx { if ast.IsNodeIDPattern(dest[idx+1:]) { // PDF 标注不计入 https://github.com/siyuan-note/siyuan/issues/13891 continue } } }
if "" == assetsPathMap[dest] { if strings.HasPrefix(dest, "assets/.") { // Assets starting with `.` should not be considered missing assets https://github.com/siyuan-note/siyuan/issues/8821 if !filelock.IsExist(filepath.Join(util.DataDir, dest)) { name := util.RemoveID(path.Base(dest)) ret = append(ret, &UnusedItem{Item: dest, Name: name}) } } else { name := util.RemoveID(path.Base(dest)) ret = append(ret, &UnusedItem{Item: dest, Name: name}) } continue } } } return}
func emojisInTree(tree *parse.Tree) (ret []string) { if icon := tree.Root.IALAttr("icon"); "" != icon { if !strings.Contains(icon, "://") && !strings.HasPrefix(icon, "api/icon/") && !util.NativeEmojiChars[icon] { ret = append(ret, "/emojis/"+icon) } }
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { if !entering { return ast.WalkContinue } if ast.NodeEmojiImg == n.Type { tokens := n.Tokens idx := bytes.Index(tokens, []byte("src=\"")) if -1 == idx { return ast.WalkContinue } src := tokens[idx+len("src=\""):] src = src[:bytes.Index(src, []byte("\""))] ret = append(ret, string(src)) } return ast.WalkContinue }) ret = gulu.Str.RemoveDuplicatedElem(ret) return}
func getQueryEmbedNodesAssetsLinkDests(node *ast.Node) (ret []string) { // The images in the embed blocks are not uploaded to the community hosting https://github.com/siyuan-note/siyuan/issues/10042
ret = []string{} ast.Walk(node, func(n *ast.Node, entering bool) ast.WalkStatus { if !entering || ast.NodeBlockQueryEmbedScript != n.Type { return ast.WalkContinue }
stmt := n.TokensStr() stmt = html.UnescapeString(stmt) stmt = strings.ReplaceAll(stmt, editor.IALValEscNewLine, "\n") sqlBlocks := sql.SelectBlocksRawStmt(stmt, 1, Conf.Search.Limit) for _, sqlBlock := range sqlBlocks { subtree, _ := LoadTreeByBlockID(sqlBlock.ID) if nil == subtree { continue } embedNode := treenode.GetNodeInTree(subtree, sqlBlock.ID) if nil == embedNode { continue }
ret = append(ret, getAssetsLinkDests(embedNode, false)...) } return ast.WalkContinue }) ret = gulu.Str.RemoveDuplicatedElem(ret) return}
func getAssetsLinkDests(node *ast.Node, includeServePath bool) (ret []string) { ret = []string{} ast.Walk(node, func(n *ast.Node, entering bool) ast.WalkStatus { if n.IsBlock() { // 以 custom-data-assets 开头的块属性值可能是多个资源文件链接，需要计入 // Ignore assets associated with the `custom-data-assets` block attribute when cleaning unreferenced assets https://github.com/siyuan-note/siyuan/issues/12574 for _, kv := range n.KramdownIAL { k := kv[0] if strings.HasPrefix(k, "custom-data-assets") { dest := kv[1] if "" == dest || !util.IsAssetLinkDest([]byte(dest), includeServePath) { continue } ret = append(ret, dest) } } }
// 修改以下代码时需要同时修改 database 构造行级元素实现，增加必要的类型 if !entering || (ast.NodeLinkDest != n.Type && ast.NodeHTMLBlock != n.Type && ast.NodeInlineHTML != n.Type && ast.NodeIFrame != n.Type && ast.NodeWidget != n.Type && ast.NodeAudio != n.Type && ast.NodeVideo != n.Type && ast.NodeAttributeView != n.Type && !n.IsTextMarkType("a") && !n.IsTextMarkType("file-annotation-ref")) { return ast.WalkContinue }
if ast.NodeLinkDest == n.Type { if !util.IsAssetLinkDest(n.Tokens, includeServePath) { return ast.WalkContinue }
dest := strings.TrimSpace(string(n.Tokens)) ret = append(ret, dest) } else if n.IsTextMarkType("a") { if !util.IsAssetLinkDest(gulu.Str.ToBytes(n.TextMarkAHref), includeServePath) { return ast.WalkContinue }
dest := strings.TrimSpace(n.TextMarkAHref) ret = append(ret, dest) } else if n.IsTextMarkType("file-annotation-ref") { if !util.IsAssetLinkDest(gulu.Str.ToBytes(n.TextMarkFileAnnotationRefID), includeServePath) { return ast.WalkContinue }
if !strings.Contains(n.TextMarkFileAnnotationRefID, "/") { return ast.WalkContinue }
dest := n.TextMarkFileAnnotationRefID[:strings.LastIndexByte(n.TextMarkFileAnnotationRefID, '/')] dest = strings.TrimSpace(dest) ret = append(ret, dest) } else if ast.NodeAttributeView == n.Type { attrView, _ := av.ParseAttributeView(n.AttributeViewID) if nil == attrView { return ast.WalkContinue }
for _, keyValues := range attrView.KeyValues { if av.KeyTypeMAsset == keyValues.Key.Type { for _, value := range keyValues.Values { if 1 > len(value.MAsset) { continue }
for _, asset := range value.MAsset { dest := asset.Content if !util.IsAssetLinkDest([]byte(dest), includeServePath) { continue } ret = append(ret, strings.TrimSpace(dest)) } } } else if av.KeyTypeURL == keyValues.Key.Type { for _, value := range keyValues.Values { if nil != value.URL { dest := value.URL.Content if !util.IsAssetLinkDest([]byte(dest), includeServePath) { continue } ret = append(ret, strings.TrimSpace(dest)) } } } } } else { if ast.NodeWidget == n.Type { dataAssets := n.IALAttr("custom-data-assets") if "" == dataAssets { // 兼容两种属性名 custom-data-assets 和 data-assets https://github.com/siyuan-note/siyuan/issues/4122#issuecomment-1154796568 dataAssets = n.IALAttr("data-assets") } if !util.IsAssetLinkDest([]byte(dataAssets), includeServePath) { return ast.WalkContinue } ret = append(ret, dataAssets) } else { // HTMLBlock/InlineHTML/IFrame/Audio/Video dest := treenode.GetNodeSrcTokens(n) if !util.IsAssetLinkDest([]byte(dest), includeServePath) { return ast.WalkContinue } ret = append(ret, dest) } } return ast.WalkContinue }) ret = gulu.Str.RemoveDuplicatedElem(ret) for i, dest := range ret { // 对于 macOS 的 rtfd 文件夹格式需要特殊处理，为其加上结尾 / if strings.HasSuffix(dest, ".rtfd") { ret[i] = dest + "/" } } return}
func getAssetsLinkDestsInTree(tree *parse.Tree, includeServePath bool) (nodes []*ast.Node) { ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { if !entering { return ast.WalkContinue }
dests := getAssetsLinkDests(n, includeServePath) if 1 > len(dests) { return ast.WalkContinue }
nodes = append(nodes, n) return ast.WalkContinue }) return}
func setAssetsLinkDest(node *ast.Node, oldDest, dest string) { if ast.NodeLinkDest == node.Type { if bytes.HasPrefix(node.Tokens, []byte("//")) { node.Tokens = append([]byte("https:"), node.Tokens...) } node.Tokens = bytes.ReplaceAll(node.Tokens, []byte(oldDest), []byte(dest)) } else if node.IsTextMarkType("a") { if strings.HasPrefix(node.TextMarkAHref, "//") { node.TextMarkAHref = "https:" + node.TextMarkAHref } node.TextMarkAHref = strings.ReplaceAll(node.TextMarkAHref, oldDest, dest) } else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type { if strings.HasPrefix(node.TextMarkAHref, "//") { node.TextMarkAHref = "https:" + node.TextMarkAHref } node.Tokens = bytes.ReplaceAll(node.Tokens, []byte(oldDest), []byte(dest)) } else if ast.NodeAttributeView == node.Type { needWrite := false attrView, _ := av.ParseAttributeView(node.AttributeViewID) if nil == attrView { return }
for _, keyValues := range attrView.KeyValues { if av.KeyTypeMAsset != keyValues.Key.Type { continue }
for _, value := range keyValues.Values { if 1 > len(value.MAsset) { continue }
for _, asset := range value.MAsset { if oldDest == asset.Content && oldDest != dest { asset.Content = dest needWrite = true } } } }
if needWrite { av.SaveAttributeView(attrView) } }}
func getRemoteAssetsLinkDests(node *ast.Node, onlyImg bool) (ret []string) { if onlyImg { if ast.NodeLinkDest == node.Type { if node.ParentIs(ast.NodeImage) { if !util.IsAssetLinkDest(node.Tokens, false) { ret = append(ret, string(node.Tokens)) }
} } else if ast.NodeAttributeView == node.Type { attrView, _ := av.ParseAttributeView(node.AttributeViewID) if nil == attrView { return }
for _, keyValues := range attrView.KeyValues { if av.KeyTypeMAsset != keyValues.Key.Type { continue }
for _, value := range keyValues.Values { if 1 > len(value.MAsset) { continue }
for _, asset := range value.MAsset { if av.AssetTypeImage != asset.Type { continue }
dest := asset.Content if !util.IsAssetLinkDest([]byte(dest), false) { ret = append(ret, strings.TrimSpace(dest)) } } } } } } else { if ast.NodeLinkDest == node.Type { if !util.IsAssetLinkDest(node.Tokens, false) { ret = append(ret, string(node.Tokens)) } } else if node.IsTextMarkType("a") { if !util.IsAssetLinkDest([]byte(node.TextMarkAHref), false) { ret = append(ret, node.TextMarkAHref) } } else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type { src := treenode.GetNodeSrcTokens(node) if !util.IsAssetLinkDest([]byte(src), false) { ret = append(ret, src) } } else if ast.NodeAttributeView == node.Type { attrView, _ := av.ParseAttributeView(node.AttributeViewID) if nil == attrView { return }
for _, keyValues := range attrView.KeyValues { if av.KeyTypeMAsset != keyValues.Key.Type { continue }
for _, value := range keyValues.Values { if 1 > len(value.MAsset) { continue }
for _, asset := range value.MAsset { dest := asset.Content if !util.IsAssetLinkDest([]byte(dest), false) { ret = append(ret, strings.TrimSpace(dest)) } } } } } } return}
func getRemoteAssetsLinkDestsInTree(tree *parse.Tree, onlyImg bool) (nodes []*ast.Node) { ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { if !entering { return ast.WalkContinue }
dests := getRemoteAssetsLinkDests(n, onlyImg) if 1 > len(dests) { return ast.WalkContinue }
nodes = append(nodes, n) return ast.WalkContinue }) return}
// allAssetAbsPaths 返回 asset 相对路径（assets/xxx）到绝对路径（F:\SiYuan\data\assets\xxx）的映射。func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) { notebooks, err := ListNotebooks() if err != nil { return }
assetsAbsPathMap = map[string]string{} // 笔记本 assets for _, notebook := range notebooks { notebookAbsPath := filepath.Join(util.DataDir, notebook.ID) filelock.Walk(notebookAbsPath, func(path string, d fs.DirEntry, err error) error { if notebookAbsPath == path { return nil } if isSkipFile(d.Name()) { if d.IsDir() { return filepath.SkipDir } return nil }
if filelock.IsHidden(path) { // 清理资源文件时忽略隐藏文件 Ignore hidden files when cleaning unused assets https://github.com/siyuan-note/siyuan/issues/12172 return nil }
if d.IsDir() && "assets" == d.Name() { filelock.Walk(path, func(assetPath string, d fs.DirEntry, err error) error { if path == assetPath { return nil } if isSkipFile(d.Name()) { if d.IsDir() { return filepath.SkipDir } return nil } relPath := filepath.ToSlash(assetPath) relPath = relPath[strings.Index(relPath, "assets/"):] if d.IsDir() { relPath += "/" } assetsAbsPathMap[relPath] = assetPath return nil }) return filepath.SkipDir } return nil }) }
// 全局 assets dataAssetsAbsPath := util.GetDataAssetsAbsPath() filelock.Walk(dataAssetsAbsPath, func(assetPath string, d fs.DirEntry, err error) error { if dataAssetsAbsPath == assetPath { return nil }
if isSkipFile(d.Name()) { if d.IsDir() { return filepath.SkipDir } return nil }
if filelock.IsHidden(assetPath) { // 清理资源文件时忽略隐藏文件 Ignore hidden files when cleaning unused assets https://github.com/siyuan-note/siyuan/issues/12172 return nil }
relPath := filepath.ToSlash(assetPath) relPath = relPath[strings.Index(relPath, "assets/"):] if d.IsDir() { relPath += "/" } assetsAbsPathMap[relPath] = assetPath return nil }) return}
// copyBoxAssetsToDataAssets 将笔记本路径下所有（包括子文档）的 assets 复制一份到 data/assets 中。func copyBoxAssetsToDataAssets(boxID string) { boxLocalPath := filepath.Join(util.DataDir, boxID) copyAssetsToDataAssets(boxLocalPath)}
// copyDocAssetsToDataAssets 将文档路径下所有（包括子文档）的 assets 复制一份到 data/assets 中。func copyDocAssetsToDataAssets(boxID, parentDocPath string) { boxLocalPath := filepath.Join(util.DataDir, boxID) parentDocDirAbsPath := filepath.Dir(filepath.Join(boxLocalPath, parentDocPath)) copyAssetsToDataAssets(parentDocDirAbsPath)}
func copyAssetsToDataAssets(rootPath string) { var assetsDirPaths []string filelock.Walk(rootPath, func(path string, d fs.DirEntry, err error) error { if nil != err || rootPath == path || nil == d { return nil }
isDir, name := d.IsDir(), d.Name() if isSkipFile(name) { if isDir { return filepath.SkipDir } return nil }
if "assets" == name && isDir { assetsDirPaths = append(assetsDirPaths, path) } return nil })
dataAssetsPath := filepath.Join(util.DataDir, "assets") for _, assetsDirPath := range assetsDirPaths { if err := filelock.Copy(assetsDirPath, dataAssetsPath); err != nil { logging.LogErrorf("copy tree assets from [%s] to [%s] failed: %s", assetsDirPaths, dataAssetsPath, err) } }}
func isFileWatcherAvailable() bool { return util.ContainerAndroid != util.Container && util.ContainerIOS != util.Container && util.ContainerHarmony != util.Container}