Something went wrong. Try again.
A modern, network-enabled music player platform built on Rockbox technology. rockboxd.tsiry-sandratraina.com
rust deno navidrome airplay libadwaita zig mpris snapcast mpd rockbox audio subsonic
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527/*************************************************************************** * __________ __ ___. * Open \______ \ ____ ____ | | _\_ |__ _______ ___ * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ / * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < < * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \ * \/ \/ \/ \/ \/ * * Copyright (C) 2005 by Miika Pekkarinen * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY * KIND, either express or implied. * ****************************************************************************/
/* * TagCache API * * ----------x---------x------------------x----- * | | | External * +---------------x-------+ | TagCache | Libraries * | Modification routines | | Core | * +-x---------x-----------+ | | * | (R/W) | | | | * | +------x-------------x-+ +-------------x-----+ | * | | x==x Filters & clauses | | * | | Search routines | +-------------------+ | * | | x============================x DirCache * | +-x--------------------+ | (optional) * | | (R) | * | | +-------------------------------+ +---------+ | * | | | DB Commit (sort,unique,index) | | | | * | | +-x--------------------------x--+ | Control | | * | | | (R/W) | (R) | Thread | | * | | | +----------------------+ | | | | * | | | | TagCache DB Builder | | +---------+ | * | | | +-x-------------x------+ | | * | | | | (R) | (W) | | * | | | | +--x--------x---------+ | * | | | | | Temporary Commit DB | | * | | | | +---------------------+ | * +-x----x-------x--+ | * | TagCache RAM DB x==\(W) +-----------------+ | * +-----------------+ \===x | | * | | | | (R) | Ram DB Loader x============x DirCache * +-x----x---x---x---+ /==x | | (optional) * | Tagcache Disk DB x==/ +-----------------+ | * +------------------+ | * */
#if !defined(PLUGIN)
/*#define LOGF_ENABLE*//*#define LOGF_CLAUSES define to enable logf clause matching (LOGF_ENABLE req'd) */
#include <stdio.h>#include <stdlib.h>#include <ctype.h>#ifdef APPLICATION#include <unistd.h> /* readlink() */#include <limits.h> /* PATH_MAX */#endif#include "config.h"#include "ata_idle_notify.h"#include "thread.h"#include "kernel.h"#include "system.h"#include "logf.h"#include "string-extra.h"#include "usb.h"#include "metadata.h"#include "tagcache.h"#include "yesno.h"#include "core_alloc.h"#include "crc32.h"#include "misc.h"#include "settings.h"#include "dir.h"#include "pathfuncs.h"#include "debug.h"#include "dircache.h"#include "errno.h"
#ifndef __PCTOOL__#include "lang.h"#include "eeprom_settings.h"#endif#define USR_CANCEL false#else/*!defined(PLUGIN)*/#define USR_CANCEL (tc_stat.commit_delayed == true)#endif /*!defined(PLUGIN)*//* * Define this to support non-native endian tagcache files. * Databases are always written in native endian so this is * basically only necessary to support databases generated * by the PC database tool. * * Adds around 0.5-1.0k of code. */#define TAGCACHE_SUPPORT_FOREIGN_ENDIAN
/* Allow a little drift to the filename ordering (should not be too high/low). */#define POS_HISTORY_COUNT 4
/* How much to pre-load entries while committing to prevent seeking. */#define IDX_BUF_DEPTH 64
/* Tag Cache Header version 'TCHxx'. Increment when changing internal structures. */#define TAGCACHE_MAGIC 0x54434810
/* Dump store/restore header version 'TCSxx'. */#define TAGCACHE_STATEFILE_MAGIC 0x54435301
/* How much to allocate extra space for ramcache. */#define TAGCACHE_RESERVE 32768
/* * Define how long one entry must be at least (longer -> less memory at commit). * Must be at least 4 bytes in length for correct alignment. */#define TAGFILE_ENTRY_CHUNK_LENGTH 8
/* Used to guess the necessary buffer size at commit. */#define TAGFILE_ENTRY_AVG_LENGTH 16
/* Max events in the internal tagcache command queue. */#define TAGCACHE_COMMAND_QUEUE_LENGTH 32
/* Idle time before committing events in the command queue. */#define TAGCACHE_COMMAND_QUEUE_COMMIT_DELAY HZ*2
/* Dont commit database_tmp data. */#define TAGCACHE_FILE_NOCOMMIT "database_commit.ignore"
/* Temporary database containing new tags to be committed to the main db. */#define TAGCACHE_FILE_TEMP "database_tmp.tcd"
/* The main database master index and numeric data. */#define TAGCACHE_FILE_MASTER "database_idx.tcd"
/* The main database string data. */#define TAGCACHE_FILE_INDEX "database_%d.tcd"
/* ASCII dumpfile of the DB contents. */#define TAGCACHE_FILE_CHANGELOG "database_changelog.txt"
/* Serialized DB. */#define TAGCACHE_STATEFILE "database_state.tcd"
/* Flags */#define FLAG_DELETED 0x0001 /* Entry has been removed from db */#define FLAG_DIRCACHE 0x0002 /* Filename is a dircache pointer */#define FLAG_DIRTYNUM 0x0004 /* Numeric data has been modified */#define FLAG_TRKNUMGEN 0x0008 /* Track number has been generated */#define FLAG_RESURRECTED 0x0010 /* Statistics data has been resurrected */
#ifdef __PCTOOL__#define yield() do { } while(0)#define sim_sleep(timeout) do { } while(0)#define do_timed_yield() do { } while(0)static void db_log(const char *prefix, const char *msg);#endif
#ifdef __PCTOOL__#define DB_LOG(__prefix, __file) db_log(__prefix, __file)#else#define DB_LOG(...)#endif
#ifndef __PCTOOL__/* Tag Cache thread. */static struct event_queue tagcache_queue SHAREDBSS_ATTR;static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];static const char tagcache_thread_name[] = "tagcache";#endif
/* Previous path when scanning directory tree recursively. */static char curpath[TAGCACHE_BUFSZ];/* Shared buffer for several build_index fns to reduce stack usage */static char build_idx_buf[TAGCACHE_BUFSZ];static const long build_idx_bufsz = sizeof(build_idx_buf);/* Used when removing duplicates. */static char *tempbuf; /* Allocated when needed. */static long tempbufidx; /* Current location in buffer. */static size_t tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */static long tempbuf_left; /* Buffer space left. */static long tempbuf_pos;#ifndef __PCTOOL__static int tempbuf_handle;#endif
#define SORTED_TAGS_COUNT 9#define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)#define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)#define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \ (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))/* Tags we want to get sorted (loaded to the tempbuf). */#define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \ (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \ (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title) | \ (1LU << tag_virt_canonicalartist))
/* Uniqued tags (we can use these tags with filters and conditional clauses). */#define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \ (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \ (1LU << tag_albumartist) | (1LU << tag_grouping) | \ (1LU << tag_virt_canonicalartist))
/* String presentation of the tags defined in tagcache.h. Must be in correct order! */static const char * const tags_str[] = { "artist", "album", "genre", "title", "filename", "composer", "comment", "albumartist", "grouping", "year", "discnumber", "tracknumber", "canonicalartist", "bitrate", "length", "playcount", "rating", "playtime", "lastplayed", "commitid", "mtime", "lastelapsed", "lastoffset"#if !defined(LOGF_ENABLE) || !defined(LOGF_CLAUSES)};#define logf_clauses(...) do { } while(0)#else /* strings for logf debugging */ "tag_virt_basename", "tag_virt_length_min", "tag_virt_length_sec", "tag_virt_playtime_min", "tag_virt_playtime_sec", "tag_virt_entryage", "tag_virt_autoscore"};/* more debug strings */static const char * const tag_type_str[] = { [clause_none] = "clause_none", [clause_is] = "clause_is", [clause_is_not] = "clause_is_not", [clause_gt] = "clause_gt", [clause_gteq] = "clause_gteq", [clause_lt] = "clause_lt", [clause_lteq] = "clause_lteq", [clause_contains] = "clause_contains", [clause_not_contains] = "clause_not_contains", [clause_begins_with] = "clause_begins_with", [clause_not_begins_with] = "clause_not_begins_with", [clause_ends_with] = "clause_ends_with", [clause_not_ends_with] = "clause_not_ends_with", [clause_oneof] = "clause_oneof", [clause_contains_oneof] = "clause_contains_oneof", [clause_begins_oneof] = "clause_begins_oneof", [clause_ends_oneof] = "clause_ends_oneof", [clause_not_oneof] = "clause_not_oneof", [clause_not_contains_oneof] = "clause_not_contains_oneof", [clause_not_begins_oneof] = "clause_not_begins_oneof", [clause_not_ends_oneof] = "clause_not_ends_oneof", [clause_logical_or] = "clause_logical_or" };#define logf_clauses logf#endif /* !defined(LOGF_ENABLE) || !defined(LOGF_CLAUSES) */
#if defined(PLUGIN)char *itoa_buf(char *buf, size_t bufsz, long int i){ snprintf(buf, bufsz, "%ld", i); return buf;}#endif
/* Status information of the tagcache. */static struct tagcache_stat tc_stat;
/* Queue commands. */enum tagcache_queue { Q_STOP_SCAN = 0, Q_START_SCAN, Q_IMPORT_CHANGELOG, Q_UPDATE, Q_REBUILD,
/* Internal tagcache command queue. */ CMD_UPDATE_MASTER_HEADER, CMD_UPDATE_NUMERIC,};
struct tagcache_command_entry { int32_t command; int32_t idx_id; int32_t tag; int32_t data;};
#ifndef __PCTOOL__static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];static volatile int command_queue_widx = 0;static volatile int command_queue_ridx = 0;static struct mutex command_queue_mutex SHAREDBSS_ATTR;#endif
/* Tag database structures. */
/* Variable-length tag entry in tag files. */struct tagfile_entry { int32_t tag_length; /* Length of the data in bytes including '\0' */ int32_t idx_id; /* Corresponding entry location in index file of not unique tags */ char tag_data[0]; /* Begin of the tag data */};
/* Fixed-size tag entry in master db index. */struct index_entry { int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */ int32_t flag; /* Status flags */};
/* Header is the same in every file. */struct tagcache_header { int32_t magic; /* Header version number */ int32_t datasize; /* Data size in bytes */ int32_t entry_count; /* Number of entries in this file */};
struct master_header { struct tagcache_header tch; int32_t serial; /* Increasing counting number */ int32_t commitid; /* Number of commits so far */ int32_t dirty;};
static struct master_header current_tcmh;
#ifdef HAVE_TC_RAMCACHE
#define TC_ALIGN_PTR(p, type, gap_out_p) \ ({ typeof (p) __p = (p); \ typeof (p) __palgn = ALIGN_UP(__p, __alignof__(type)); \ *(gap_out_p) = (char *)__palgn - (char *)__p; \ __palgn; })
#define IF_TCRCDC(...) IF_DIRCACHE(__VA_ARGS__)
#ifdef HAVE_DIRCACHE#define tcrc_dcfrefs \ ((struct dircache_fileref *)(tcramcache.hdr->tags[tag_filename] + \ sizeof (struct tagcache_header)))#endif /* HAVE_DIRCACHE */
/* Header is created when loading database to ram. */struct ramcache_header { char *tags[TAG_COUNT]; /* Tag file content (dcfrefs if tag_filename) */ int entry_count[TAG_COUNT]; /* Number of entries in the indices. */ struct index_entry indices[0]; /* Master index file content */};
#ifdef HAVE_EEPROM_SETTINGSstruct statefile_header { int32_t magic; /* Statefile version number */ struct master_header mh; /* Header from the master index */ struct ramcache_header *hdr; /* Old load address of hdr for relocation */ struct tagcache_stat tc_stat;};#endif /* HAVE_EEPROM_SETTINGS */
/* In-RAM ramcache structure (not persisted) */static struct tcramcache{ struct ramcache_header *hdr; /* allocated ramcache_header */ int handle; /* buffer handle */} tcramcache;
static inline void tcrc_buffer_lock(void){ core_pin(tcramcache.handle);}
static inline void tcrc_buffer_unlock(void){ core_unpin(tcramcache.handle);}
#else /* ndef HAVE_TC_RAMCACHE */
#define IF_TCRCDC(...)
#endif /* HAVE_TC_RAMCACHE */
/** * Full tag entries stored in a temporary file waiting * for commit to the cache. */struct temp_file_entry { int32_t tag_offset[TAG_COUNT]; int16_t tag_length[TAG_COUNT]; int32_t flag; int32_t data_length;};
struct tempbuf_id_list { long id; struct tempbuf_id_list *next;};
struct tempbuf_searchidx { long idx_id; char *str; int seek; struct tempbuf_id_list idlist;};
/* Lookup buffer for fixing messed up index while after sorting. */static long commit_entry_count;static long lookup_buffer_depth;static struct tempbuf_searchidx **lookup;
/* Used when building the temporary file. */static int cachefd = -1, filenametag_fd;static int total_entry_count = 0;static int data_size = 0;static int processed_dir_count;
/* Thread safe locking */static volatile int write_lock;static volatile int read_lock;
static bool delete_entry(long idx_id);
static inline void str_setlen(char *buf, size_t len){ buf[len] = '\0';}
const char* tagcache_tag_to_str(int tag){ return tags_str[tag];}
#ifdef TAGCACHE_SUPPORT_FOREIGN_ENDIANstatic void swap_tagfile_entry(struct tagfile_entry *buf){ if (tc_stat.econ) { buf->tag_length = swap32(buf->tag_length); buf->idx_id = swap32(buf->idx_id); }}
static void swap_index_entry(struct index_entry *buf){ if (tc_stat.econ) { for (int i = 0; i < TAG_COUNT; ++i) buf->tag_seek[i] = swap32(buf->tag_seek[i]); buf->flag = swap32(buf->flag); }}
static void swap_tagcache_header(struct tagcache_header *buf){ if (tc_stat.econ) { buf->magic = swap32(buf->magic); buf->datasize = swap32(buf->datasize); buf->entry_count = swap32(buf->entry_count); }}
static void swap_master_header(struct master_header *buf){ if (tc_stat.econ) { swap_tagcache_header(&buf->tch); buf->serial = swap32(buf->serial); buf->commitid = swap32(buf->commitid); buf->dirty = swap32(buf->dirty); }}#elsestatic void swap_tagfile_entry(struct tagfile_entry *buf) { (void)buf; }static void swap_index_entry(struct index_entry *buf) { (void)buf; }static void swap_tagcache_header(struct tagcache_header *buf) { (void)buf; }static void swap_master_header(struct master_header *buf) { (void)buf; }#endif
static ssize_t read_tagfile_entry(int fd, struct tagfile_entry *buf){ ssize_t ret = read(fd, buf, sizeof(*buf)); if (ret == sizeof(*buf) && tc_stat.econ) swap_tagfile_entry(buf);
return ret;}
static ssize_t write_tagfile_entry(int fd, struct tagfile_entry *buf){ struct tagfile_entry e = *buf;
swap_tagfile_entry(&e);
return write(fd, &e, sizeof(e));}
enum e_read_errors { e_SUCCESS = 0, e_SUCCESS_LEN_ZERO = 1, e_ENTRY_SIZEMISMATCH, e_TAG_TOOLONG, e_TAG_SIZEMISMATCH};
static enum e_read_errorsread_tagfile_entry_and_tag(int fd, struct tagfile_entry *tfe, char* buf, int bufsz){ if (read_tagfile_entry(fd, tfe) != sizeof(struct tagfile_entry)) return e_ENTRY_SIZEMISMATCH;
long tag_length = tfe->tag_length; if (tag_length >= bufsz) return e_TAG_TOOLONG;
if (tag_length > 0 && read(fd, buf, tag_length) != tag_length) return e_TAG_SIZEMISMATCH;
str_setlen(buf, tag_length); return (tag_length > 0 && *buf) ? e_SUCCESS : e_SUCCESS_LEN_ZERO;}
static ssize_t read_index_entries(int fd, struct index_entry *buf, size_t count){ ssize_t ret = read(fd, buf, sizeof(*buf) * count); for (ssize_t i = 0; i < ret; i += sizeof(*buf)) swap_index_entry(buf++);
return ret;}
static ssize_t write_index_entries(int fd, struct index_entry *buf, size_t count){#ifdef TAGCACHE_SUPPORT_FOREIGN_ENDIAN ssize_t ret = 0; for (; count > 0; count--) { struct index_entry e = *buf++; swap_index_entry(&e);
ssize_t rc = write(fd, &e, sizeof(e)); if (rc < 0) return rc; ret += rc; }
return ret;#else return write(fd, buf, sizeof(*buf) * count);#endif}
static ssize_t read_tagcache_header(int fd, struct tagcache_header *buf){ ssize_t ret = read(fd, buf, sizeof(*buf)); if (ret == sizeof(*buf)) swap_tagcache_header(buf);
return ret;}
static ssize_t write_tagcache_header(int fd, struct tagcache_header *buf){ struct tagcache_header e = *buf; swap_tagcache_header(&e); return write(fd, &e, sizeof(e));}
static ssize_t read_master_header(int fd, struct master_header *buf){ ssize_t ret = read(fd, buf, sizeof(*buf)); if (ret == sizeof(*buf)) swap_master_header(buf);
return ret;}
static ssize_t write_master_header(int fd, struct master_header *buf){ struct master_header e = *buf; swap_master_header(&e); return write(fd, &e, sizeof(e));}
/* * open_db_fd and remove_db_file are noinline to minimize stack usage */static int NO_INLINE open_db_fd(const char* filename, int mode){ char buf[MAX_PATH];
if(mode & O_CREAT) { if (mkdir(tc_stat.db_path) < 0 && errno != EEXIST) return -1; }
return open_pathfmt(buf, sizeof(buf), mode, "%s/%s", tc_stat.db_path, filename);}
static int NO_INLINE remove_db_file(const char* filename){ char buf[MAX_PATH];
snprintf(buf, sizeof(buf), "%s/%s", tc_stat.db_path, filename);
return remove(buf);}
static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write){ int fd; char fname[MAX_PATH];
if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT) return -1;
fd = open_pathfmt(fname, sizeof(fname), write ? O_RDWR : O_RDONLY, "%s/" TAGCACHE_FILE_INDEX, tc_stat.db_path, tag); if (fd < 0) { logf("%s failed: tag=%d write=%d file= " TAGCACHE_FILE_INDEX, __func__, tag, write, tag); tc_stat.ready = false; return fd; }
/* Check the header. */ if (read_tagcache_header(fd, hdr) != sizeof(struct tagcache_header) || hdr->magic != TAGCACHE_MAGIC) { logf("header error"); tc_stat.ready = false; close(fd); return -2; }
return fd;}
static int open_master_fd(struct master_header *hdr, bool write){ int fd; int rc;
fd = open_db_fd(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY); if (fd < 0) { logf("master file open failed for R/W"); tc_stat.ready = false; return fd; }
rc = read(fd, hdr, sizeof(struct master_header)); if (rc != sizeof(struct master_header)) { logf("master file read failed"); close(fd); return -1; }
/* Tagcache files can have either endianness. A device will always * create files in its native endianness, but we accept non-native * endian files for compatibility reasons. */ if (hdr->tch.magic == TAGCACHE_MAGIC) tc_stat.econ = false;#ifdef TAGCACHE_SUPPORT_FOREIGN_ENDIAN else if (hdr->tch.magic == swap32(TAGCACHE_MAGIC)) { tc_stat.econ = true; swap_master_header(hdr); }#endif else { logf("master file bad magic: %08lx\n", (unsigned long)hdr->tch.magic); close(fd); return -2; }
return fd;}
static void remove_files(void){ int i; char buf[MAX_PATH]; const int bufsz = sizeof(buf); logf("%s", __func__);
tc_stat.ready = false; tc_stat.ramcache = false; tc_stat.econ = false; remove_db_file(TAGCACHE_FILE_MASTER); for (i = 0; i < TAG_COUNT; i++) { if (TAGCACHE_IS_NUMERIC(i)) continue;
snprintf(buf, bufsz, "%s/" TAGCACHE_FILE_INDEX, tc_stat.db_path, i); remove(buf); }}
static bool check_all_headers(void){ struct master_header myhdr; struct tagcache_header tch; int tag; int fd;
if ( (fd = open_master_fd(&myhdr, false)) < 0) return false;
close(fd); if (myhdr.dirty) { logf("tagcache is dirty!"); return false; }
memcpy(¤t_tcmh, &myhdr, sizeof(struct master_header));
for (tag = 0; tag < TAG_COUNT; tag++) { if (TAGCACHE_IS_NUMERIC(tag)) continue;
if ( (fd = open_tag_fd(&tch, tag, false)) < 0) return false;
close(fd); }
return true;}
static bool update_master_header(void){ struct master_header myhdr; int fd;
if (!tc_stat.ready) return false;
if ( (fd = open_master_fd(&myhdr, true)) < 0) return false;
myhdr.serial = current_tcmh.serial; myhdr.commitid = current_tcmh.commitid; myhdr.dirty = current_tcmh.dirty;
/* Write it back */ lseek(fd, 0, SEEK_SET); write_master_header(fd, &myhdr); close(fd);
return true;}
#if !defined(PLUGIN)#ifndef __PCTOOL__static bool do_timed_yield(void){ /* Sorting can lock up for quite a while, so yield occasionally */ static long wakeup_tick = 0; if (TIME_AFTER(current_tick, wakeup_tick)) { yield(); wakeup_tick = current_tick + (HZ/25); return true; } return false;}#endif /* __PCTOOL__ */
static void allocate_tempbuf(void){ /* Yeah, malloc would be really nice now :) */ size_t size; tempbuf_size = 0;
#ifdef __PCTOOL__ size = 32*1024*1024; tempbuf = malloc(size); if (tempbuf) tempbuf_size = size;#else /* !__PCTOOL__ */ /* Need to pass dummy ops to prevent the buffer being moved * out from under us, since we yield during the tagcache commit. */ tempbuf_handle = core_alloc_maximum(&size, &buflib_ops_locked); if (tempbuf_handle > 0) { tempbuf = core_get_data(tempbuf_handle); tempbuf_size = size; }#endif /* __PCTOOL__ */
}
static void free_tempbuf(void){ if (tempbuf_size == 0) return ;
#ifdef __PCTOOL__ free(tempbuf);#else tempbuf_handle = core_free(tempbuf_handle);#endif tempbuf = NULL; tempbuf_size = 0;}
#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)/* find the ramcache entry corresponding to the file indicated by * filename and dc (it's corresponding dircache id). */static long find_entry_ram(const char *filename){ static long last_pos = 0; struct dircache_fileref dcfref;
/* Check if tagcache is loaded into ram. */ if (!tc_stat.ramcache || global_settings.tagcache_ram != TAGCACHE_RAM_ON) return -1;
if (dircache_search(DCS_CACHED_PATH | DCS_UPDATE_FILEREF, &dcfref, filename) <= 0) { logf("tagcache: file not found."); return -1; }
/* Search references */ int end_pos = current_tcmh.tch.entry_count; while (1) { for (int i = last_pos; i < end_pos; i++) { do_timed_yield();
if (!(tcramcache.hdr->indices[i].flag & FLAG_DIRCACHE)) continue;
int cmp = dircache_fileref_cmp(&tcrc_dcfrefs[i], &dcfref); if (cmp < 3) continue;
last_pos = MAX(0, i - 3); return i; }
if (last_pos == 0) { last_pos = MAX(0, end_pos - 3); break; }
end_pos = last_pos; last_pos = 0; }
return -1;}#endif /* defined (HAVE_TC_RAMCACHE) && defined (HAVE_DIRCACHE) */
static long find_entry_disk(const char *filename_raw, bool localfd){ struct tagfile_entry tfe; struct tagcache_header tch; static long last_pos = -1; long pos_history[POS_HISTORY_COUNT]; unsigned int pos_history_idx = 0; unsigned int i;
char buf[TAGCACHE_BUFSZ]; const long bufsz = sizeof(buf);
int fd; int pos = -1; long idx = -1;
bool found = false;
const char *filename = filename_raw;
#ifdef APPLICATION char pathbuf[PATH_MAX]; /* Note: Don't use MAX_PATH here, it's too small */ if (realpath(filename, pathbuf) == pathbuf) filename = pathbuf;#endif /* APPLICATION */
if (!tc_stat.ready) return -2;
fd = filenametag_fd; if (fd < 0 || localfd) { last_pos = -1; if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0) return -1; }
check_again:
if (last_pos > 0) /* pos gets cached to prevent reading from beginning */ pos = lseek(fd, last_pos, SEEK_SET); else /* start back at beginning */ pos = lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
long tag_length = strlen(filename) + 1; /* include NULL */
if (tag_length < bufsz) { while (true) { for (i = pos_history_idx-1; i < pos_history_idx; i--) pos_history[i+1] = pos_history[i]; pos_history[0] = pos;
if (read_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry)) { logf("size mismatch find entry"); break; } else { pos += sizeof(struct tagfile_entry) + tfe.tag_length; /* don't read the entry unless the length matches */ if (tfe.tag_length == tag_length) { if(read(fd, buf, tfe.tag_length) != tag_length) { logf("read error #2"); close(fd); if (!localfd) filenametag_fd = -1; last_pos = -1; return -3; } if (!strncmp(filename, buf, tag_length)) { last_pos = pos_history[pos_history_idx]; found = true; idx = tfe.idx_id; break ; } } else lseek(fd, pos, SEEK_SET);
} } if (pos_history_idx < POS_HISTORY_COUNT - 1) pos_history_idx++; }
/* Not found? */ if (!found) { if (last_pos > 0) /* start back at the beginning */ { last_pos = -1; logf("seek again"); goto check_again; }
idx = -4; }
if (fd != filenametag_fd || localfd) close(fd);
return idx;}
static int find_index(const char *filename){ long idx_id = -1;
#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) idx_id = find_entry_ram(filename);#endif
if (idx_id < 0) idx_id = find_entry_disk(filename, true);
return idx_id;}
bool tagcache_find_index(struct tagcache_search *tcs, const char *filename){ /* NOTE: on ret==true you need to call tagcache_search_finish(tcs) yourself */ int idx_id;
if (!tc_stat.ready) return false;
idx_id = find_index(filename); if (idx_id < 0) return false;
if (!tagcache_search(tcs, tag_filename)) return false;
tcs->entry_count = 0; tcs->idx_id = idx_id;
return true;}
static bool get_index(int masterfd, int idxid, struct index_entry *idx, bool use_ram){ bool localfd = false;
if (idxid < 0) { logf("Incorrect idxid: %d", idxid); return false; }
#ifdef HAVE_TC_RAMCACHE if (tc_stat.ramcache && use_ram) { if (tcramcache.hdr->indices[idxid].flag & FLAG_DELETED) return false;
*idx = tcramcache.hdr->indices[idxid]; return true; }#endif /* HAVE_TC_RAMCACHE */
if (masterfd < 0) { struct master_header tcmh;
localfd = true; masterfd = open_master_fd(&tcmh, false); if (masterfd < 0) return false; }
lseek(masterfd, idxid * sizeof(struct index_entry) + sizeof(struct master_header), SEEK_SET); if (read_index_entries(masterfd, idx, 1) != sizeof(struct index_entry)) { logf("read error #3"); if (localfd) close(masterfd);
return false; }
if (localfd) close(masterfd);
if (idx->flag & FLAG_DELETED) return false;
return true;
(void)use_ram;}
#ifndef __PCTOOL__
static bool write_index(int masterfd, int idxid, struct index_entry *idx){ /* We need to exclude all memory only flags & tags when writing to disk. */ if (idx->flag & FLAG_DIRCACHE) { logf("memory only flags!"); return false; }
#ifdef HAVE_TC_RAMCACHE /* Only update numeric data. Writing the whole index to RAM by memcpy * destroys dircache pointers! */ if (tc_stat.ramcache) { struct index_entry *idx_ram = &tcramcache.hdr->indices[idxid];
for (int tag = 0; tag < TAG_COUNT; tag++) { if (TAGCACHE_IS_NUMERIC(tag)) { idx_ram->tag_seek[tag] = idx->tag_seek[tag]; } }
/* Don't touch the dircache flag or attributes. */ idx_ram->flag = (idx->flag & 0x0000ffff) | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE)); }#endif /* HAVE_TC_RAMCACHE */
lseek(masterfd, idxid * sizeof(struct index_entry) + sizeof(struct master_header), SEEK_SET); if (write_index_entries(masterfd, idx, 1) != sizeof(struct index_entry)) { logf("write error #3"); logf("idxid: %d", idxid); return false; }
return true;}
#endif /* !__PCTOOL__ */
static bool open_files(struct tagcache_search *tcs, int tag){ if (tcs->idxfd[tag] < 0) { char fname[MAX_PATH]; tcs->idxfd[tag] = open_pathfmt(fname, sizeof(fname), O_RDONLY, "%s/" TAGCACHE_FILE_INDEX, tc_stat.db_path, tag); if (tcs->idxfd[tag] < 0) { logf("File not open!"); return false; } }
return true;}
static bool retrieve(struct tagcache_search *tcs, IF_DIRCACHE(int idx_id,) struct index_entry *idx, int tag, char *buf, long bufsz){ bool success = false; bool is_basename = false; struct tagfile_entry tfe; long seek;
if (tag == tag_virt_basename) { tag = tag_filename; is_basename = true; }
if (TAGCACHE_IS_NUMERIC(tag)) goto failure;
seek = idx->tag_seek[tag]; if (seek < 0) { logf("Retrieve failed"); goto failure; }
#ifdef HAVE_TC_RAMCACHE if (tcs->ramsearch) {#ifdef HAVE_DIRCACHE if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)) { if (dircache_get_fileref_path(&tcrc_dcfrefs[idx_id], buf, bufsz) >= 0) success = true; } else#endif /* HAVE_DIRCACHE */ if (tag != tag_filename) { struct tagfile_entry *ep = (struct tagfile_entry *)&tcramcache.hdr->tags[tag][seek]; strmemccpy(buf, ep->tag_data, bufsz); success = true; } }#endif /* HAVE_TC_RAMCACHE */
if (!success && open_files(tcs, tag)) { lseek(tcs->idxfd[tag], seek, SEEK_SET); switch (read_tagfile_entry_and_tag(tcs->idxfd[tag], &tfe, buf, bufsz)) { case e_ENTRY_SIZEMISMATCH: logf("read error #5"); break; case e_TAG_TOOLONG: logf("too long tag #5"); break; case e_TAG_SIZEMISMATCH: logf("read error #6"); break; default: logf("unknown_error"); break; case e_SUCCESS_LEN_ZERO: case e_SUCCESS: success = true; break; } }
if (success) { if (is_basename) { char* basename = strrchr(buf, '/'); if (basename != NULL) memmove(buf, basename + 1, strlen(basename)); /* includes NULL */ } return true; }
failure: str_setlen(buf, 0); return false;}
#define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
static long tc_find_tag(int tag, int idx_id, const struct index_entry *idx){#ifndef __PCTOOL__ if (! COMMAND_QUEUE_IS_EMPTY && TAGCACHE_IS_NUMERIC(tag)) { /* Attempt to find tag data through store-to-load forwarding in command queue */ long result = -1;
mutex_lock(&command_queue_mutex);
int ridx = command_queue_widx;
while (ridx != command_queue_ridx) { if (--ridx < 0) ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
if (command_queue[ridx].command == CMD_UPDATE_NUMERIC && command_queue[ridx].idx_id == idx_id && command_queue[ridx].tag == tag) { result = command_queue[ridx].data; break; } }
mutex_unlock(&command_queue_mutex);
if (result >= 0) { logf("tc_find_tag: Recovered tag %d value %lX from write queue", tag, (unsigned long) result); return result; } }#else (void)idx_id;#endif
return idx->tag_seek[tag];}
static inline long sec_in_ms(long ms){ return (ms/1000) % 60;}
static inline long min_in_ms(long ms){ return (ms/1000) / 60;}
static long check_virtual_tags(int tag, int idx_id, const struct index_entry *idx){ long data = 0;
switch (tag) { case tag_virt_length_sec: data = sec_in_ms(tc_find_tag(tag_length, idx_id, idx)); break;
case tag_virt_length_min: data = min_in_ms(tc_find_tag(tag_length, idx_id, idx)); break;
case tag_virt_playtime_sec: data = sec_in_ms(tc_find_tag(tag_playtime, idx_id, idx)); break;
case tag_virt_playtime_min: data = min_in_ms(tc_find_tag(tag_playtime, idx_id, idx)); break;
case tag_virt_autoscore: if (tc_find_tag(tag_length, idx_id, idx) == 0 || tc_find_tag(tag_playcount, idx_id, idx) == 0) { data = 0; } else { /* A straight calculus gives: autoscore = 100 * playtime / length / playcout (1) Now, consider the euclidian division of playtime by length: playtime = alpha * length + beta With: 0 <= beta < length Now, (1) becomes: autoscore = 100 * (alpha / playcout + beta / length / playcount) Both terms should be small enough to avoid any overflow */ long playtime = tc_find_tag(tag_playtime, idx_id, idx); long length = tc_find_tag(tag_length, idx_id, idx); long playcount = tc_find_tag(tag_playcount, idx_id, idx); data = 100 * (playtime / length) + (100 * (playtime % length)) / length; data /= playcount; } break;
/* How many commits before the file has been added to the DB. */ case tag_virt_entryage: data = current_tcmh.commitid - tc_find_tag(tag_commitid, idx_id, idx) - 1; break;
case tag_virt_basename: tag = tag_filename; /* return filename; caller handles basename */ /* FALLTHRU */
default: data = tc_find_tag(tag, idx_id, idx); }
return data;}
long tagcache_get_numeric(const struct tagcache_search *tcs, int tag){ struct index_entry idx;
if (!tc_stat.ready) return false;
if (!TAGCACHE_IS_NUMERIC(tag)) return -1;
if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true)) return -2;
return check_virtual_tags(tag, tcs->idx_id, &idx);}
inline static bool str_ends_with(const char *str1, const char *str2){ logf_clauses("%s %s %s", str1, __func__, str2); int str_len = strlen(str1); int clause_len = strlen(str2);
if (clause_len > str_len) return false;
return !strcasecmp(&str1[str_len - clause_len], str2);}
inline static bool str_oneof(const char *str, const char *list){ logf_clauses("%s %s %s", str, __func__, list); const char *sep; int l, len = strlen(str);
while (*list) { sep = strchr(list, '|'); l = sep ? (intptr_t)sep - (intptr_t)list : (int)strlen(list); if ((l==len) && !strncasecmp(str, list, len)) return true; list += sep ? l + 1 : l; }
return false;}
inline static bool str_begins_ends_oneof(const char *str, const char *list, bool begins){ logf_clauses("%s %s (%s) %s", str, __func__, begins ? "begins" : "ends", list); const char *sep; int l, p, len = strlen(str);
while (*list) { sep = strchr(list, '|'); l = sep ? (intptr_t)sep - (intptr_t)list : (int)strlen(list); p = begins ? 0 : len - l; if (l <= len && !strncasecmp(&str[p], list, l)) return true; list += sep ? l + 1 : l; }
return false;}
inline static bool str_contains_oneof(const char *str, char *list){ logf_clauses("%s %s %s", str, __func__, list); const char *sep; int l, len = strlen(str); while (*list) { sep = strchr(list, '|'); l = sep ? (intptr_t)sep - (intptr_t)list : (int)strlen(list); list[l] = '\0'; if (l <= len && strcasestr(str, list) != NULL) { if (sep) list[l] = '|'; return true; } if (sep) list[l] = '|'; list += sep ? l + 1 : l; } return false;}
static bool check_against_clause(long numeric, const char *str, const struct tagcache_search_clause *clause){ if (clause->numeric) { switch (clause->type) { case clause_is: return numeric == clause->numeric_data; case clause_is_not: return numeric != clause->numeric_data; case clause_gt: return numeric > clause->numeric_data; case clause_gteq: return numeric >= clause->numeric_data; case clause_lt: return numeric < clause->numeric_data; case clause_lteq: return numeric <= clause->numeric_data; default: logf("Incorrect numeric tag: %d", clause->type); } } else { switch (clause->type) { case clause_is: return !strcasecmp(clause->str, str); case clause_is_not: return strcasecmp(clause->str, str); case clause_gt: return 0>strcasecmp(clause->str, str); case clause_gteq: return 0>=strcasecmp(clause->str, str); case clause_lt: return 0<strcasecmp(clause->str, str); case clause_lteq: return 0<=strcasecmp(clause->str, str); case clause_contains: return (strcasestr(str, clause->str) != NULL); case clause_not_contains: return (strcasestr(str, clause->str) == NULL); case clause_begins_with: return (strcasestr(str, clause->str) == str); case clause_not_begins_with: return (strcasestr(str, clause->str) != str); case clause_ends_with: return str_ends_with(str, clause->str); case clause_not_ends_with: return !str_ends_with(str, clause->str); case clause_oneof: return str_oneof(str, clause->str); case clause_not_oneof: return !str_oneof(str, clause->str); case clause_ends_oneof: /* Fall-Through */ case clause_begins_oneof: return str_begins_ends_oneof(str, clause->str, clause->type == clause_begins_oneof); case clause_not_ends_oneof: /* Fall-Through */ case clause_not_begins_oneof: return !str_begins_ends_oneof(str, clause->str, clause->type == clause_not_begins_oneof); case clause_contains_oneof: return str_contains_oneof(str, clause->str); case clause_not_contains_oneof: return !str_contains_oneof(str, clause->str); default: logf("Incorrect tag: %d", clause->type); } }
return false;}
static bool check_clauses(struct tagcache_search *tcs, struct index_entry *idx, struct tagcache_search_clause **clauses, int count){ int i;
/* Go through all conditional clauses. */ for (i = 0; i < count; i++) { int seek; char buf[256]; const int bufsz = sizeof(buf); char *str = buf; struct tagcache_search_clause *clause = clauses[i];
logf_clauses("%s clause %d %s %s [%ld] %s", "Checking", i, tag_type_str[clause->type], tags_str[clause->tag], clause->numeric_data, (clause->numeric || clause->str == NULL) ? "[NUMERIC?]" : clause->str);
if (clause->type == clause_logical_or) { logf_clauses("Bailing"); break; /* all conditions before logical-or satisfied -- stop processing clauses */ } seek = check_virtual_tags(clause->tag, tcs->idx_id, idx);
#ifdef HAVE_TC_RAMCACHE if (tcs->ramsearch) { struct tagfile_entry *tfe;
if (!TAGCACHE_IS_NUMERIC(clause->tag)) { if (clause->tag == tag_filename || clause->tag == tag_virt_basename) { retrieve(tcs, IF_DIRCACHE(tcs->idx_id,) idx, clause->tag, buf, bufsz); } else { tfe = (struct tagfile_entry *) &tcramcache.hdr->tags[clause->tag][seek]; /* str points to movable data, but no locking required here, * as no yield() is following */ str = tfe->tag_data; } } } else#endif /* HAVE_TC_RAMCACHE */ { struct tagfile_entry tfe;
if (!TAGCACHE_IS_NUMERIC(clause->tag)) { int tag = clause->tag; if (tag == tag_virt_basename) tag = tag_filename;
int fd = tcs->idxfd[tag]; lseek(fd, seek, SEEK_SET);
switch (read_tagfile_entry_and_tag(fd, &tfe, str, bufsz)) { case e_SUCCESS_LEN_ZERO: /* Check if entry has been deleted. */ return false; case e_SUCCESS: if (clause->tag == tag_virt_basename) { char *basename = strrchr(str, '/'); if (basename) str = basename + 1; } break; case e_ENTRY_SIZEMISMATCH: logf("read error #15"); return false; case e_TAG_TOOLONG: logf("too long tag #6"); return false; case e_TAG_SIZEMISMATCH: logf("read error #16"); return false; default: logf("unknown_error"); break;; } } }
if (!check_against_clause(seek, str, clause)) { /* Clause failed -- try finding a logical-or clause */ while (++i < count) { if (clauses[i]->type == clause_logical_or) break; }
if (i < count) /* Found logical-or? */ continue; /* Check clauses after logical-or */
return false; }
logf_clauses("%s clause %d %s %s [%ld] %s", "Found", i, tag_type_str[clause->type], tags_str[clause->tag], clause->numeric_data, (clause->numeric || clause->str == NULL) ? "[NUMERIC?]" : clause->str); }
return true;}
bool tagcache_check_clauses(struct tagcache_search *tcs, struct tagcache_search_clause **clause, int count){ struct index_entry idx;
if (count == 0) return true;
if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true)) return false;
return check_clauses(tcs, &idx, clause, count);}
static bool add_uniqbuf(struct tagcache_search *tcs, uint32_t id){ int i;
/* If uniq buffer is not defined we must return true for search to work. */ if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type) && !TAGCACHE_IS_NUMERIC(tcs->type))) { return true; }
for (i = 0; i < tcs->unique_list_count; i++) { /* Return false if entry is found. */ if (tcs->unique_list[i] == id) { //logf("%d Exists @ %d", id, i); return false; } }
if (tcs->unique_list_count < tcs->unique_list_capacity) { tcs->unique_list[i] = id; tcs->unique_list_count++; }
return true;}
static bool build_lookup_list(struct tagcache_search *tcs){ struct index_entry entry; int i, j;
tcs->seek_list_count = 0;
#ifdef HAVE_TC_RAMCACHE if (tcs->ramsearch) { tcrc_buffer_lock(); /* lock because below makes a pointer to movable data */
for (i = tcs->seek_pos; i < current_tcmh.tch.entry_count; i++) { struct tagcache_seeklist_entry *seeklist; /* idx points to movable data, don't yield or reload */ struct index_entry *idx = &tcramcache.hdr->indices[i]; if (tcs->seek_list_count == SEEK_LIST_SIZE) break ;
/* Skip deleted files. */ if (idx->flag & FLAG_DELETED) continue;
/* Go through all filters.. */ for (j = 0; j < tcs->filter_count; j++) { if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j]) { break ; } }
if (j < tcs->filter_count) continue ;
/* Check for conditions. */ if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count)) continue; /* Add to the seek list if not already in uniq buffer (doesn't yield)*/ if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type])) continue;
/* Lets add it. */ seeklist = &tcs->seeklist[tcs->seek_list_count]; seeklist->seek = idx->tag_seek[tcs->type]; seeklist->flag = idx->flag; seeklist->idx_id = i; tcs->seek_list_count++; }
tcrc_buffer_unlock();
tcs->seek_pos = i;
return tcs->seek_list_count > 0; }#endif /* HAVE_TC_RAMCACHE */
if (tcs->masterfd < 0) { struct master_header tcmh; tcs->masterfd = open_master_fd(&tcmh, false); }
lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) + sizeof(struct master_header), SEEK_SET);
while (read_index_entries(tcs->masterfd, &entry, 1) == sizeof(struct index_entry)) { struct tagcache_seeklist_entry *seeklist;
if (tcs->seek_list_count == SEEK_LIST_SIZE) break ;
i = tcs->seek_pos; tcs->seek_pos++;
/* Check if entry has been deleted. */ if (entry.flag & FLAG_DELETED) continue;
/* Go through all filters.. */ for (j = 0; j < tcs->filter_count; j++) { if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j]) break ; }
if (j < tcs->filter_count) continue ;
/* Check for conditions. */ if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count)) continue;
/* Add to the seek list if not already in uniq buffer. */ if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type])) continue;
/* Lets add it. */ seeklist = &tcs->seeklist[tcs->seek_list_count]; seeklist->seek = entry.tag_seek[tcs->type]; seeklist->flag = entry.flag; seeklist->idx_id = i; tcs->seek_list_count++;
yield(); }
return tcs->seek_list_count > 0;}
bool tagcache_search(struct tagcache_search *tcs, int tag){ /* NOTE: call tagcache_search_finish(&tcs) when finished or BAD things may happen (TM) */ struct tagcache_header tag_hdr; struct master_header master_hdr; int i;
while (read_lock) sleep(1);
memset(tcs, 0, sizeof(struct tagcache_search)); if (tc_stat.commit_step > 0 || !tc_stat.ready) return false;
tcs->position = sizeof(struct tagcache_header); tcs->type = tag; tcs->seek_pos = 0; tcs->list_position = 0; tcs->seek_list_count = 0; tcs->filter_count = 0; tcs->masterfd = -1;
for (i = 0; i < TAG_COUNT; i++) tcs->idxfd[i] = -1;
#ifndef HAVE_TC_RAMCACHE tcs->ramsearch = false;#else tcs->ramsearch = tc_stat.ramcache; if (tcs->ramsearch) { tcs->entry_count = tcramcache.hdr->entry_count[tcs->type]; } else#endif { /* Always open as R/W so we can pass tcs to functions that modify data also * without failing. */ tcs->masterfd = open_master_fd(&master_hdr, true); if (tcs->masterfd < 0) return false;
if (!TAGCACHE_IS_NUMERIC(tcs->type)) { tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false); if (tcs->idxfd[tcs->type] < 0) return false;
tcs->entry_count = tag_hdr.entry_count; } else { tcs->entry_count = master_hdr.tch.entry_count; } }
tcs->valid = true; tcs->initialized = true; write_lock++;
return true;}
void tagcache_search_set_uniqbuf(struct tagcache_search *tcs, void *buffer, long length){ tcs->unique_list = (uint32_t *)buffer; tcs->unique_list_capacity = length / sizeof(*tcs->unique_list); tcs->unique_list_count = 0; memset(tcs->unique_list, 0, tcs->unique_list_capacity);}
bool tagcache_search_add_filter(struct tagcache_search *tcs, int tag, int seek){ if (tcs->filter_count == TAGCACHE_MAX_FILTERS) return false;
if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag)) return false;
tcs->filter_tag[tcs->filter_count] = tag; tcs->filter_seek[tcs->filter_count] = seek; tcs->filter_count++;
return true;}
bool tagcache_search_add_clause(struct tagcache_search *tcs, struct tagcache_search_clause *clause){ int i; int clause_count = tcs->clause_count;
if (clause_count >= TAGCACHE_MAX_CLAUSES) { logf("Too many clauses"); return false; }
if (clause->type != clause_logical_or) { /* BUGFIX OR'd clauses seem to be mishandled once made into a filter */ if (clause_count <= 1 || tcs->clause[clause_count - 1]->type != clause_logical_or) { /* Check if there is already a similar filter in present (filters are * much faster than clauses). */ for (i = 0; i < tcs->filter_count; i++) { if (tcs->filter_tag[i] == clause->tag) { return true; } } }
if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0) { char fname[MAX_PATH]; tcs->idxfd[clause->tag] = open_pathfmt(fname, sizeof(fname), O_RDONLY, "%s/" TAGCACHE_FILE_INDEX, tc_stat.db_path, clause->tag); } }
tcs->clause[tcs->clause_count] = clause; tcs->clause_count++;
return true;}
static bool get_next(struct tagcache_search *tcs, bool is_numeric, char *buf, long bufsz){ struct tagfile_entry entry;#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) long flag = 0;#endif
if (tcs->idxfd[tcs->type] < 0 && !is_numeric#ifdef HAVE_TC_RAMCACHE && !tcs->ramsearch#endif ) return false;
/* Relative fetch. */ if (tcs->filter_count > 0 || tcs->clause_count > 0 || is_numeric#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) /* We need to retrieve flag status for dircache. */ || (tcs->ramsearch && tcs->type == tag_filename)#endif ) { struct tagcache_seeklist_entry *seeklist;
/* Check for end of list. */ if (tcs->list_position == tcs->seek_list_count) { tcs->list_position = 0;
/* Try to fetch more. */ if (!build_lookup_list(tcs)) { tcs->valid = false; return false; } }
seeklist = &tcs->seeklist[tcs->list_position];#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) flag = seeklist->flag;#endif tcs->position = seeklist->seek; tcs->idx_id = seeklist->idx_id; tcs->list_position++; } else { if (tcs->entry_count == 0) { tcs->valid = false; return false; }
tcs->entry_count--; }
tcs->result_seek = tcs->position;
if (is_numeric) { itoa_buf(buf, bufsz, tcs->position); tcs->result = buf; tcs->result_len = strlen(buf) + 1; return true; }
/* Direct fetch. */#ifdef HAVE_TC_RAMCACHE if (tcs->ramsearch) {#ifdef HAVE_DIRCACHE if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)) { ssize_t len = dircache_get_fileref_path(&tcrc_dcfrefs[tcs->idx_id], buf, bufsz); if (len >= 0) { tcs->result_len = len + 1; tcs->result = buf; tcs->ramresult = false; return true; } /* else do it the hard way */ }#endif /* HAVE_DIRCACHE */
if (tcs->type != tag_filename) { struct tagfile_entry *ep;
ep = (struct tagfile_entry *)&tcramcache.hdr->tags[tcs->type][tcs->position]; /* don't return ep->tag_data directly as it may move */ tcs->result_len = strlcpy(buf, ep->tag_data, bufsz) + 1; tcs->result = buf; tcs->idx_id = ep->idx_id; tcs->ramresult = false; /* was true before we copied to buf too */
/* Increase position for the next run. This may get overwritten. */ tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
return true; } }#endif /* HAVE_TC_RAMCACHE */
if (!open_files(tcs, tcs->type)) { tcs->valid = false; return false; }
/* Seek stream to the correct position and continue to direct fetch. */ lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
switch (read_tagfile_entry_and_tag(tcs->idxfd[tcs->type], &entry, buf, bufsz)) { case e_SUCCESS_LEN_ZERO: case e_SUCCESS: break; case e_ENTRY_SIZEMISMATCH: logf("read error #5"); tcs->valid = false; return false; case e_TAG_TOOLONG: tcs->valid = false; logf("too long tag #2"); logf("P:%lX/%" PRIX32, (unsigned long) tcs->position, entry.tag_length); return false; case e_TAG_SIZEMISMATCH: tcs->valid = false; logf("read error #4"); return false; }
/** Update the position for the next read (this may be overridden if filters or clauses are being used). */ tcs->position += sizeof(struct tagfile_entry) + entry.tag_length; str_setlen(buf, entry.tag_length);
tcs->result = buf; tcs->result_len = entry.tag_length + 1; tcs->idx_id = entry.idx_id; tcs->ramresult = false;
return true;}
bool tagcache_get_next(struct tagcache_search *tcs, char *buf, long size){ if (tcs->valid && tagcache_is_usable()) { bool is_numeric = TAGCACHE_IS_NUMERIC(tcs->type); while (get_next(tcs, is_numeric, buf, size)) { if (tcs->result_len > 1) return true; } }#ifdef LOGF_ENABLE if (tcs->unique_list_count > 0) logf(" uniqbuf: %d used / %d avail", tcs->unique_list_count, tcs->unique_list_capacity);#endif
return false;}
bool tagcache_retrieve(struct tagcache_search *tcs, int idxid, int tag, char *buf, long size){ struct index_entry idx;
*buf = '\0'; if (!get_index(tcs->masterfd, idxid, &idx, true)) return false;
return retrieve(tcs, IF_DIRCACHE(idxid,) &idx, tag, buf, size);}
void tagcache_search_finish(struct tagcache_search *tcs){ int i;
if (!tcs->initialized) return;
if (tcs->masterfd >= 0) { close(tcs->masterfd); tcs->masterfd = -1; }
for (i = 0; i < TAG_COUNT; i++) { if (tcs->idxfd[i] >= 0) { close(tcs->idxfd[i]); tcs->idxfd[i] = -1; } }
tcs->ramsearch = false; tcs->valid = false; tcs->initialized = 0; if (write_lock > 0) write_lock--;}
#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag){ return (struct tagfile_entry *)&tcramcache.hdr->tags[tag][entry->tag_seek[tag]];}
static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id){ return check_virtual_tags(tag, idx_id, entry);}
static char* get_tag_string(const struct index_entry *entry, int tag){ char* s = get_tag(entry, tag)->tag_data; return strcmp(s, UNTAGGED) ? s : NULL;}
bool tagcache_fill_tags(struct mp3entry *id3, const char *filename){ struct index_entry *entry; int idx_id;
if (!tc_stat.ready || !tc_stat.ramcache) return false;
/* Find the corresponding entry in tagcache. */
if (filename != NULL) memset(id3, 0, sizeof(struct mp3entry)); else /* Note: caller clears id3 prior to call */ filename = id3->path;
idx_id = find_entry_ram(filename); if (idx_id < 0) return false;
entry = &tcramcache.hdr->indices[idx_id];
char* buf = id3->id3v2buf; ssize_t remaining = sizeof(id3->id3v2buf);
/* this macro sets id3 strings by copying to the id3v2buf */#define SET(x, y) do \ { \ if (remaining > 0) \ { \ x = NULL; /* initialize with null if tag doesn't exist */ \ char* src = get_tag_string(entry, y); \ if (src) \ { \ x = buf; \ size_t len = strlcpy(buf, src, remaining) +1; \ buf += len; remaining -= len; \ } \ } \ } while(0)
SET(id3->title, tag_title); SET(id3->artist, tag_artist); SET(id3->album, tag_album); SET(id3->genre_string, tag_genre); SET(id3->composer, tag_composer); SET(id3->comment, tag_comment); SET(id3->albumartist, tag_albumartist); SET(id3->grouping, tag_grouping);
id3->length = get_tag_numeric(entry, tag_length, idx_id); id3->playcount = get_tag_numeric(entry, tag_playcount, idx_id); id3->rating = get_tag_numeric(entry, tag_rating, idx_id); id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id); id3->score = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10; id3->year = get_tag_numeric(entry, tag_year, idx_id);
id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id); id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id); id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id); if (id3->bitrate == 0) id3->bitrate = 1;
if (global_settings.autoresume_enable) { id3->elapsed = get_tag_numeric(entry, tag_lastelapsed, idx_id); logf("tagcache_fill_tags: Set elapsed for %s to %lX\n", id3->title, id3->elapsed);
id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id); logf("tagcache_fill_tags: Set offset for %s to %lX\n", id3->title, id3->offset); }
return true;}#endif /* defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) */
static inline void write_item(const char *item){ int len = strlen(item) + 1;
data_size += len; write(cachefd, item, len);}
static int check_if_empty(char **tag){ int length;
if (*tag == NULL || **tag == '\0') { *tag = UNTAGGED; return sizeof(UNTAGGED); /* Tag length */ }
length = strlen(*tag); if (length > TAG_MAXLEN) { logf("over length tag: %s", *tag); length = TAG_MAXLEN; str_setlen((*tag), length); }
return length + 1;}
#ifdef __PCTOOL__static void db_log(const char *prefix, const char *msg){ /* Crude logging for the sim - to aid in debugging */ int logfd = open(ROCKBOX_DIR "/database.log", O_WRONLY | O_APPEND | O_CREAT, 0666); if (logfd >= 0) { write(logfd, prefix, strlen(prefix)); write(logfd, ": ", 2); write(logfd, msg, strlen(msg)); write(logfd, "\n", 1); close(logfd); }}#endif
/* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good * idea, as it uses lots of stack and is called from a recursive function * (check_dir). */static void NO_INLINE add_tagcache(char *path, unsigned long mtime){ #define ADD_TAG(entry, tag, data) \ /* Adding tag */ \ entry.tag_length[tag] = check_if_empty(data); \ entry.tag_offset[tag] = offset; \ offset += entry.tag_length[tag]
struct mp3entry id3; struct temp_file_entry entry; bool ret; int idx_id = -1; char tracknumfix[3]; int offset = 0; int path_length = strlen(path); bool has_artist; bool has_grouping;
DB_LOG("file", path);
if (cachefd < 0) return ;
/* Check for overlength file path. */ if (path_length > MAX_PATH || path_length > TAG_MAXLEN) { /* Path can't be shortened. */ logf("Too long path: %s", path); DB_LOG("error", "path too long"); return ; }
/* Check if the file is supported. */ int afmt = probe_file_format(path); if (afmt == AFMT_UNKNOWN) return;
/* Check if the file is already cached. */#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) idx_id = find_entry_ram(path);#endif
/* Be sure the entry doesn't exist. */ if (filenametag_fd >= 0 && idx_id < 0) idx_id = find_entry_disk(path, false);
/* Check if file has been modified. */ if (idx_id >= 0) { struct index_entry idx;
/* TODO: Mark that the index exists (for fast reverse scan) */ //found_idx[idx_id/8] |= idx_id%8;
if (!get_index(-1, idx_id, &idx, true)) { logf("failed to retrieve index entry"); DB_LOG("error", "failed to retrieve index entry"); return ; }
if ((unsigned long)idx.tag_seek[tag_mtime] == mtime) { /* No changes to file. */ return ; }
/* Metadata might have been changed. Delete the entry. */ logf("Re-adding: %s", path); DB_LOG("info", "re-adding"); if (!delete_entry(idx_id)) { logf("delete_entry failed: %d", idx_id); DB_LOG("error", "delete entry failed"); return ; } }
/*memset(&id3, 0, sizeof(struct mp3entry)); -- get_metadata does this for us */ memset(&entry, 0, sizeof(struct temp_file_entry)); memset(&tracknumfix, 0, sizeof(tracknumfix)); ret = get_metadata_afmt(&id3, -1, path, afmt, METADATA_EXCLUDE_ID3_PATH);
if (!ret) { logf("get_metadata failed: %s", path); DB_LOG("error", "get_metadata failed"); return ; }
logf("-> %s", path);
if (id3.tracknum < 0) /* Track number missing? */ { id3.tracknum = -1; }
/* Numeric tags */ entry.tag_offset[tag_year] = id3.year; entry.tag_offset[tag_discnumber] = id3.discnum; entry.tag_offset[tag_tracknumber] = id3.tracknum; entry.tag_offset[tag_length] = id3.length; entry.tag_offset[tag_bitrate] = id3.bitrate; entry.tag_offset[tag_mtime] = mtime;
/* String tags. */ has_artist = id3.artist != NULL && strlen(id3.artist) > 0; has_grouping = id3.grouping != NULL && strlen(id3.grouping) > 0;
ADD_TAG(entry, tag_filename, &path); ADD_TAG(entry, tag_title, &id3.title); ADD_TAG(entry, tag_artist, &id3.artist); ADD_TAG(entry, tag_album, &id3.album); ADD_TAG(entry, tag_genre, &id3.genre_string); ADD_TAG(entry, tag_composer, &id3.composer); ADD_TAG(entry, tag_comment, &id3.comment); ADD_TAG(entry, tag_albumartist, &id3.albumartist); if (has_artist) { ADD_TAG(entry, tag_virt_canonicalartist, &id3.artist); } else { ADD_TAG(entry, tag_virt_canonicalartist, &id3.albumartist); } if (has_grouping) { ADD_TAG(entry, tag_grouping, &id3.grouping); } else { ADD_TAG(entry, tag_grouping, &id3.title); } entry.data_length = offset;
/* Write the header */ write(cachefd, &entry, sizeof(struct temp_file_entry));
/* And tags also... Correct order is critical */ write_item(path); write_item(id3.title); write_item(id3.artist); write_item(id3.album); write_item(id3.genre_string); write_item(id3.composer); write_item(id3.comment); write_item(id3.albumartist); if (has_artist) { write_item(id3.artist); } else { write_item(id3.albumartist); } if (has_grouping) { write_item(id3.grouping); } else { write_item(id3.title); }
total_entry_count++;
#undef ADD_TAG}#endif /*!defined(PLUGIN)*/
static bool tempbuf_insert(char *str, int id, int idx_id, bool unique){ struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; int len = strlen(str)+1; int i; unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4]; unsigned crc32 = 0xffffffff; char chr_lower; for (i = 0; str[i] != '\0' && i < len -1; i++) { chr_lower = tolower(str[i]); crc32 = crc_32(&chr_lower, 1, crc32); }
if (unique) { /* Check if the crc does not exist -> entry does not exist for sure. */ for (i = 0; i < tempbufidx; i++) { if (crcbuf[-i] != crc32) continue;
if (!strcasecmp(str, index[i].str)) { if (id < 0 || id >= lookup_buffer_depth) { logf("lookup buf overf.: %d", id); return false; }
lookup[id] = &index[i]; return true; } } }
/* Insert to CRC buffer. */ crcbuf[-tempbufidx] = crc32; tempbuf_left -= 4;
/* Insert it to the buffer. */ tempbuf_left -= len; if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count) { logf("temp buf error rem: %ld idx: %ld / %ld", tempbuf_left, tempbufidx, commit_entry_count-1); return false; } if (id >= lookup_buffer_depth) { logf("lookup buf overf. #2: %d", id); return false; }
if (id >= 0) { lookup[id] = &index[tempbufidx]; index[tempbufidx].idlist.id = id; } else index[tempbufidx].idlist.id = -1;
index[tempbufidx].idlist.next = NULL; index[tempbufidx].idx_id = idx_id; index[tempbufidx].seek = -1; index[tempbufidx].str = &tempbuf[tempbuf_pos]; memcpy(index[tempbufidx].str, str, len); tempbuf_pos += len; tempbufidx++;
return true;}
static int compare(const void *p1, const void *p2){ do_timed_yield();
struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1; struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
if (strcmp(e1->str, UNTAGGED) == 0) { if (strcmp(e2->str, UNTAGGED) == 0) return 0; return -1; } else if (strcmp(e2->str, UNTAGGED) == 0) return 1;
return strncasecmp(e1->str, e2->str, TAG_MAXLEN);}
static int tempbuf_sort(int fd){ struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; struct tagfile_entry fe; int i; int length;
/* Generate reverse lookup entries. */ for (i = 0; i < lookup_buffer_depth; i++) { struct tempbuf_id_list *idlist;
if (!lookup[i]) continue;
if (lookup[i]->idlist.id == i) continue;
idlist = &lookup[i]->idlist; while (idlist->next != NULL) idlist = idlist->next;
ALIGN_BUFFER(tempbuf_pos, tempbuf_left, alignof(struct tempbuf_id_list)); tempbuf_left -= sizeof(struct tempbuf_id_list); if (tempbuf_left < 0) return -1;
idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos]; tempbuf_pos += sizeof(struct tempbuf_id_list);
idlist = idlist->next; idlist->id = i; idlist->next = NULL;
do_timed_yield(); }
qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare); memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
for (i = 0; i < tempbufidx; i++) { struct tempbuf_id_list *idlist = &index[i].idlist;
/* Fix the lookup list. */ while (idlist != NULL) { if (idlist->id >= 0) lookup[idlist->id] = &index[i]; idlist = idlist->next; }
index[i].seek = lseek(fd, 0, SEEK_CUR); length = strlen(index[i].str) + 1; fe.tag_length = length; fe.idx_id = index[i].idx_id;
/* Check the chunk alignment. */ if ((fe.tag_length + sizeof(struct tagfile_entry)) % TAGFILE_ENTRY_CHUNK_LENGTH) { fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH - ((fe.tag_length + sizeof(struct tagfile_entry)) % TAGFILE_ENTRY_CHUNK_LENGTH); }
if (write_tagfile_entry(fd, &fe) != sizeof(struct tagfile_entry)) { logf("tempbuf_sort: write error #1"); return -1; }
if (write(fd, index[i].str, length) != length) { logf("tempbuf_sort: write error #2"); return -2; }
/* Write some padding. */ if (fe.tag_length - length > 0) write(fd, "XXXXXXXX", fe.tag_length - length); }
return i;}
inline static struct tempbuf_searchidx* tempbuf_locate(int id){ if (id < 0 || id >= lookup_buffer_depth) return NULL;
return lookup[id];}
inline static int tempbuf_find_location(int id){ struct tempbuf_searchidx *entry;
entry = tempbuf_locate(id); if (entry == NULL) return -1;
return entry->seek;}
static bool build_numeric_indices(struct tagcache_header *h, int tmpfd){ struct master_header tcmh; struct index_entry idx; int masterfd; int masterfd_pos; struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf; int max_entries; int entries_processed = 0; int i, j;
max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
logf("Building numeric indices..."); lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
if ( (masterfd = open_master_fd(&tcmh, true)) < 0) return false;
masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry), SEEK_CUR); if (masterfd_pos < 0) { logf("we can't append!"); close(masterfd); return false; }
while (entries_processed < h->entry_count && !USR_CANCEL) { int count = MIN(h->entry_count - entries_processed, max_entries);
/* Read in as many entries as possible. */ for (i = 0; i < count; i++) { struct temp_file_entry *tfe = &entrybuf[i]; int datastart;
/* Read in numeric data. */ if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) != sizeof(struct temp_file_entry)) { logf("read fail #1"); close(masterfd); return false; }
datastart = lseek(tmpfd, 0, SEEK_CUR);
/** * Read string data from the following tags: * - tag_filename * - tag_artist * - tag_album * - tag_title * * A crc32 hash is calculated from the read data * and stored back to the data offset field kept in memory. */#define tmpdb_read_string_tag(tag) \ lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \ if ((unsigned long)tfe->tag_length[tag] > (unsigned long)build_idx_bufsz) \ { \ logf("read fail: buffer overflow"); \ close(masterfd); \ return false; \ } \ \ if (read(tmpfd, build_idx_buf, tfe->tag_length[tag]) != \ tfe->tag_length[tag]) \ { \ logf("read fail #2"); \ close(masterfd); \ return false; \ } \ str_setlen(build_idx_buf, tfe->tag_length[tag]); \ \ tfe->tag_offset[tag] = crc_32(build_idx_buf, strlen(build_idx_buf), 0xffffffff); \ lseek(tmpfd, datastart, SEEK_SET)
tmpdb_read_string_tag(tag_filename); tmpdb_read_string_tag(tag_artist); tmpdb_read_string_tag(tag_album); tmpdb_read_string_tag(tag_title);
/* Seek to the end of the string data. */ lseek(tmpfd, tfe->data_length, SEEK_CUR); }
/* Backup the master index position. */ masterfd_pos = lseek(masterfd, 0, SEEK_CUR); lseek(masterfd, sizeof(struct master_header), SEEK_SET);
/* Check if we can resurrect some deleted runtime statistics data. */ for (i = 0; i < tcmh.tch.entry_count && !USR_CANCEL; i++) { /* Read the index entry. */ if (read_index_entries(masterfd, &idx, 1) != sizeof(struct index_entry)) { logf("read fail #3"); close(masterfd); return false; }
/** * Skip unless the entry is marked as being deleted * or the data has already been resurrected. */ if (!(idx.flag & FLAG_DELETED) || (idx.flag & FLAG_RESURRECTED)) continue;
/* Now try to match the entry. */ /** * To succesfully match a song, the following conditions * must apply: * * For numeric fields: tag_length * - Full identical match is required * * If tag_filename matches, no further checking necessary. * * For string hashes: tag_artist, tag_album, tag_title * - All three of these must match */ for (j = 0; j < count; j++) { struct temp_file_entry *tfe = &entrybuf[j];
/* Try to match numeric fields first. */ if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length]) continue;
/* Now it's time to do the hash matching. */ if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename]) { int match_count = 0;
/* No filename match, check if we can match two other tags. */#define tmpdb_match(tag) \ if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \ match_count++
tmpdb_match(tag_artist); tmpdb_match(tag_album); tmpdb_match(tag_title);
if (match_count < 3) { /* Still no match found, give up. */ continue; } }
/* A match found, now copy & resurrect the statistical data. */#define tmpdb_copy_tag(tag) \ tfe->tag_offset[tag] = idx.tag_seek[tag]
tmpdb_copy_tag(tag_playcount); tmpdb_copy_tag(tag_rating); tmpdb_copy_tag(tag_playtime); tmpdb_copy_tag(tag_lastplayed); tmpdb_copy_tag(tag_commitid); tmpdb_copy_tag(tag_lastelapsed); tmpdb_copy_tag(tag_lastoffset);
/* Avoid processing this entry again. */ idx.flag |= FLAG_RESURRECTED;
lseek(masterfd, -(off_t)sizeof(struct index_entry), SEEK_CUR); if (write_index_entries(masterfd, &idx, 1) != sizeof(struct index_entry)) { logf("masterfd writeback fail #1"); close(masterfd); return false; }
logf("Entry resurrected"); } }
/* Restore the master index position. */ lseek(masterfd, masterfd_pos, SEEK_SET);
/* Commit the data to the index. */ for (i = 0; i < count && !USR_CANCEL; i++) { int loc = lseek(masterfd, 0, SEEK_CUR);
if (read_index_entries(masterfd, &idx, 1) != sizeof(struct index_entry)) { logf("read fail #3"); close(masterfd); return false; }
for (j = 0; j < TAG_COUNT; j++) { if (!TAGCACHE_IS_NUMERIC(j)) continue;
idx.tag_seek[j] = entrybuf[i].tag_offset[j]; } idx.flag = entrybuf[i].flag;
if (idx.tag_seek[tag_commitid]) { /* Data has been resurrected. */ idx.flag |= FLAG_DIRTYNUM; } else if (tc_stat.ready && current_tcmh.commitid > 0) { idx.tag_seek[tag_commitid] = current_tcmh.commitid; idx.flag |= FLAG_DIRTYNUM; }
/* Write back the updated index. */ lseek(masterfd, loc, SEEK_SET); if (write_index_entries(masterfd, &idx, 1) != sizeof(struct index_entry)) { logf("write fail"); close(masterfd); return false; } }
entries_processed += count; logf("%d/%" PRId32 " entries processed", entries_processed, h->entry_count); }
close(masterfd);
return true;}
/** * Return values: * > 0 success * == 0 temporary failure * < 0 fatal error */static int build_index(int index_type, struct tagcache_header *h, int tmpfd){ int i; struct tagcache_header tch; struct master_header tcmh; struct index_entry idxbuf[IDX_BUF_DEPTH]; int idxbuf_pos; int fd = -1, masterfd; bool error = false; int init; int masterfd_pos;
logf("Building index: %d", index_type);
/* Check the number of entries we need to allocate ram for. */ commit_entry_count = h->entry_count + 1;
masterfd = open_master_fd(&tcmh, false); if (masterfd >= 0) { commit_entry_count += tcmh.tch.entry_count; close(masterfd); } else remove_files(); /* Just to be sure we are clean. */
/* Open the index file, which contains the tag names. */ fd = open_tag_fd(&tch, index_type, true); if (fd >= 0) { logf("tch.datasize=%" PRId32, tch.datasize); lookup_buffer_depth = 1 + /* First part */ commit_entry_count + /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH); } else { lookup_buffer_depth = 1 + /* First part */ commit_entry_count + /* Second part */ 0; }
logf("lookup_buffer_depth=%ld", lookup_buffer_depth); logf("commit_entry_count=%ld", commit_entry_count);
/* Allocate buffer for all index entries from both old and new * tag files. */ tempbufidx = 0; tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
/* Allocate lookup buffer. The first portion of commit_entry_count * contains the new tags in the temporary file and the second * part for locating entries already in the db. * * New tags Old tags * +---------+---------------------------+ * | index | position/ENTRY_CHUNK_SIZE | lookup buffer * +---------+---------------------------+ * * Old tags are inserted to a temporary buffer with position: * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...); * And new tags with index: * tempbuf_insert(idx, ...); * * The buffer is sorted and written into tag file: * tempbuf_sort(...); * leaving master index locations messed up. * * That is fixed using the lookup buffer for old tags: * new_seek = tempbuf_find_location(old_seek, ...); * and for new tags: * new_seek = tempbuf_find_location(idx); */ lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos]; tempbuf_pos += lookup_buffer_depth * sizeof(void **); memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
/* And calculate the remaining data space used mainly for storing * tag data (strings). */ tempbuf_left = tempbuf_size - tempbuf_pos - 8; if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0) { logf("Buffer way too small!"); close(fd); return 0; }
if (fd >= 0) { /** * If tag file contains unique tags (sorted index), we will load * it entirely into memory so we can resort it later for use with * chunked browsing. */ if (TAGCACHE_IS_SORTED(index_type)) { logf("loading tags..."); for (i = 0; i < tch.entry_count && !USR_CANCEL; i++) { struct tagfile_entry entry; int loc = lseek(fd, 0, SEEK_CUR); bool ret; switch (read_tagfile_entry_and_tag(fd, &entry, build_idx_buf, build_idx_bufsz)) { case e_SUCCESS_LEN_ZERO: /* Skip deleted entries. */ continue; case e_SUCCESS: break; case e_ENTRY_SIZEMISMATCH: logf("read error #7"); close(fd); return -2; case e_TAG_TOOLONG: logf("too long tag #3"); close(fd); return -2; case e_TAG_SIZEMISMATCH: logf("read error #8"); close(fd); return -2; }
/** * Save the tag and tag id in the memory buffer. Tag id * is saved so we can later reindex the master lookup * table when the index gets resorted. */ ret = tempbuf_insert(build_idx_buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH + commit_entry_count, entry.idx_id, TAGCACHE_IS_UNIQUE(index_type)); if (!ret) { close(fd); return -3; } do_timed_yield(); } logf("done"); } else tempbufidx = tch.entry_count; } else { logf("Create New Index: %d", index_type); /** * Creating new index file to store the tags. No need to preload * anything whether the index type is sorted or not. * * Note: although we are creating a file under the db path, it must * already exist by this point so no mkdir is required. */ fd = open_pathfmt(build_idx_buf, build_idx_bufsz, O_WRONLY | O_CREAT | O_TRUNC, "%s/" TAGCACHE_FILE_INDEX, tc_stat.db_path, index_type); if (fd < 0) { logf(TAGCACHE_FILE_INDEX " open fail", index_type); return -2; }
tch.magic = TAGCACHE_MAGIC; tch.entry_count = 0; tch.datasize = 0;
if (write_tagcache_header(fd, &tch) != sizeof(struct tagcache_header)) { logf("header write failed"); close(fd); return -2; } }
/* Loading the tag lookup file as "master file". */ logf("Loading index file"); masterfd = open_db_fd(TAGCACHE_FILE_MASTER, O_RDWR);
if (masterfd < 0) { logf("Creating new DB"); masterfd = open_db_fd(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
if (masterfd < 0) { logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER); close(fd); return -2; }
/* Write the header (write real values later). */ memset(&tcmh, 0, sizeof(struct master_header)); tcmh.tch = *h; tcmh.tch.entry_count = 0; tcmh.tch.datasize = 0; tcmh.dirty = true; write_master_header(masterfd, &tcmh); init = true; masterfd_pos = lseek(masterfd, 0, SEEK_CUR); } else { /** * Master file already exists so we need to process the current * file first. */ init = false;
if (read_master_header(masterfd, &tcmh) != sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC) { logf("header error"); close(fd); close(masterfd); return -2; }
/** * If we reach end of the master file, we need to expand it to * hold new tags. If the current index is not sorted, we can * simply append new data to end of the file. * However, if the index is sorted, we need to update all tag * pointers in the master file for the current index. */ masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry), SEEK_CUR); if (masterfd_pos == ffilesize(masterfd)) { logf("appending..."); init = true; } }
/** * Load new unique tags in memory to be sorted later and added * to the master lookup file. */ if (TAGCACHE_IS_SORTED(index_type)) { lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET); /* h is the header of the temporary file containing new tags. */ logf("inserting new tags..."); for (i = 0; i < h->entry_count && !USR_CANCEL; i++) { struct temp_file_entry entry;
if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) != sizeof(struct temp_file_entry)) { logf("read fail #3"); error = true; goto error_exit; }
/* Read data. */ if (entry.tag_length[index_type] >= build_idx_bufsz) { logf("too long entry!"); error = true; goto error_exit; }
lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR); if (read(tmpfd, build_idx_buf, entry.tag_length[index_type]) != entry.tag_length[index_type]) { logf("read fail #4"); error = true; goto error_exit; } str_setlen(build_idx_buf, entry.tag_length[index_type]);
#if defined(PLUGIN) if (user_check_tag(index_type, build_idx_buf))#endif /*defined(PLUGIN)*/ { if (TAGCACHE_IS_UNIQUE(index_type)) error = !tempbuf_insert(build_idx_buf, i, -1, true); else error = !tempbuf_insert(build_idx_buf, i, tcmh.tch.entry_count + i, false);
if (error) { logf("insert error"); goto error_exit; } } /* Skip to next. */ lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] - entry.tag_length[index_type], SEEK_CUR); do_timed_yield(); } logf("done");
/* Sort the buffer data and write it to the index file. */ lseek(fd, sizeof(struct tagcache_header), SEEK_SET); /** * We need to truncate the index file now. There can be junk left * at the end of file (however, we _should_ always follow the * entry_count and don't crash with that). */ ftruncate(fd, lseek(fd, 0, SEEK_CUR));
i = tempbuf_sort(fd); if (i < 0) goto error_exit; logf("sorted %d tags", i);
/** * Now update all indexes in the master lookup file. */ logf("updating indices..."); lseek(masterfd, sizeof(struct master_header), SEEK_SET); for (i = 0; i < tcmh.tch.entry_count && !USR_CANCEL; i += idxbuf_pos) { int j; int loc = lseek(masterfd, 0, SEEK_CUR);
idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
if (read_index_entries(masterfd, idxbuf, idxbuf_pos) != (ssize_t)sizeof(struct index_entry) * idxbuf_pos) { logf("read fail #5"); error = true; goto error_exit ; } lseek(masterfd, loc, SEEK_SET);
for (j = 0; j < idxbuf_pos; j++) { if (idxbuf[j].flag & FLAG_DELETED) { /* We can just ignore deleted entries. */ // idxbuf[j].tag_seek[index_type] = 0; continue; }
idxbuf[j].tag_seek[index_type] = tempbuf_find_location( idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH + commit_entry_count);
if (idxbuf[j].tag_seek[index_type] < 0) { logf("update error: %" PRId32 "/%d/%" PRId32, idxbuf[j].flag, i+j, tcmh.tch.entry_count); error = true; goto error_exit; }
do_timed_yield(); }
/* Write back the updated index. */ if (write_index_entries(masterfd, idxbuf, idxbuf_pos) != (ssize_t)sizeof(struct index_entry) * idxbuf_pos) { logf("write fail"); error = true; goto error_exit; } } logf("done"); }
/** * Walk through the temporary file containing the new tags. */ // build_normal_index(h, tmpfd, masterfd, idx); logf("updating new indices..."); lseek(masterfd, masterfd_pos, SEEK_SET); lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET); lseek(fd, 0, SEEK_END); for (i = 0; i < h->entry_count && !USR_CANCEL; i += idxbuf_pos) { int j;
idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH); if (init) { memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH); } else { int loc = lseek(masterfd, 0, SEEK_CUR);
if (read_index_entries(masterfd, idxbuf, idxbuf_pos) != (ssize_t)sizeof(struct index_entry) * idxbuf_pos) { logf("read fail #6"); error = true; break ; } lseek(masterfd, loc, SEEK_SET); }
/* Read entry headers. */ for (j = 0; j < idxbuf_pos; j++) { if (!TAGCACHE_IS_SORTED(index_type)) { struct temp_file_entry entry; struct tagfile_entry fe;
if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) != sizeof(struct temp_file_entry)) { logf("read fail #7"); error = true; break ; }
/* Read data. */ if (entry.tag_length[index_type] >= build_idx_bufsz) { logf("too long entry!"); logf("length=%d", entry.tag_length[index_type]); logf("pos=0x%02lx", (unsigned long) lseek(tmpfd, 0, SEEK_CUR)); error = true; break ; }
lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR); if (read(tmpfd, build_idx_buf, entry.tag_length[index_type]) != entry.tag_length[index_type]) { logf("read fail #8"); logf("offset=0x%02" PRIx32, entry.tag_offset[index_type]); logf("length=0x%02x", entry.tag_length[index_type]); error = true; break ; }
/* Write to index file. */ idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR); fe.tag_length = entry.tag_length[index_type]; fe.idx_id = tcmh.tch.entry_count + i + j; write_tagfile_entry(fd, &fe); write(fd, build_idx_buf, fe.tag_length); tempbufidx++;
/* Skip to next. */ lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] - entry.tag_length[index_type], SEEK_CUR); } else { /* Locate the correct entry from the sorted array. */ idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j); if (idxbuf[j].tag_seek[index_type] < 0) { logf("entry not found (%d)", j); error = true; break ; } } }
/* Write index. */ if (write_index_entries(masterfd, idxbuf, idxbuf_pos) != (ssize_t)sizeof(struct index_entry) * idxbuf_pos) { logf("tagcache: write fail #4"); error = true; break ; }
do_timed_yield(); } logf("done");
/* Finally write the header. */ tch.magic = TAGCACHE_MAGIC; tch.entry_count = tempbufidx; tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header); lseek(fd, 0, SEEK_SET); write_tagcache_header(fd, &tch);
if (index_type != tag_filename) h->datasize += tch.datasize; logf("s:%d/%" PRId32 "/%" PRId32, index_type, tch.datasize, h->datasize); error_exit:
close(fd); close(masterfd);
if (error) return -2;
return 1;}
static bool commit(void){ struct tagcache_header tch; struct master_header tcmh; int i, len, rc; int tmpfd; int masterfd;#ifdef HAVE_DIRCACHE bool dircache_buffer_stolen = false;#endif#ifdef HAVE_TC_RAMCACHE bool ramcache_buffer_stolen = false;#endif logf("committing tagcache");
while (write_lock) sleep(1);
#if !defined(PLUGIN) int fd = open_db_fd(TAGCACHE_FILE_NOCOMMIT, O_RDONLY); if (fd >= 0) { logf("canceling commit"); tc_stat.commit_delayed = true; close(fd); tmpfd = -1; } else#endif /*!defined(PLUGIN)*/ { tmpfd = open_db_fd(TAGCACHE_FILE_TEMP, O_RDONLY); } if (tmpfd < 0) { logf("nothing to commit"); return true; }
/* Load the header. */ len = sizeof(struct tagcache_header); rc = read(tmpfd, &tch, len);
if (tch.magic != TAGCACHE_MAGIC || rc != len) { logf("incorrect tmpheader"); close(tmpfd); remove_db_file(TAGCACHE_FILE_TEMP); return false; }
if (tch.entry_count == 0) logf("nothing to commit");
/* Fully initialize existing headers (if any) before going further. */ tc_stat.ready = check_all_headers();
#ifdef HAVE_EEPROM_SETTINGS remove_db_file(TAGCACHE_STATEFILE);#endif
/* At first be sure to unload the ramcache! */#ifdef HAVE_TC_RAMCACHE tc_stat.ramcache = false;#endif
/* Beyond here, jump to commit_error to undo locks and restore dircache */ rc = false; read_lock++;
/* Try to steal every buffer we can :) */#ifdef HAVE_DIRCACHE if (tempbuf_size == 0) { /* Suspend dircache to free its allocation. */ dircache_free_buffer(); dircache_buffer_stolen = true;
allocate_tempbuf(); }#endif /* HAVE_DIRCACHE */
#ifdef HAVE_TC_RAMCACHE if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0) { tcrc_buffer_lock(); tempbuf = (char *)(tcramcache.hdr + 1); tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128; tempbuf_size &= ~0x03; ramcache_buffer_stolen = true; }#endif /* HAVE_TC_RAMCACHE */
#if defined(PLUGIN) if (tempbuf_size == 0) { tempbuf = rb->plugin_get_audio_buffer(&tempbuf_size); tempbuf_size &= ~0x03; }#endif /*defined(PLUGIN)*/
/* And finally fail if there are no buffers available. */ if (tempbuf_size == 0) { logf("delaying commit until next boot"); tc_stat.commit_delayed = true; close(tmpfd); goto commit_error; }
logf("commit %" PRId32 " entries...", tch.entry_count);
/* Mark DB dirty so it will stay disabled if commit fails. */ current_tcmh.dirty = true; update_master_header();
/* Now create the index files. */ tc_stat.commit_step = 0; tch.datasize = 0; tc_stat.commit_delayed = false;
for (i = 0; i < TAG_COUNT && !USR_CANCEL; i++) { int ret;
if (TAGCACHE_IS_NUMERIC(i)) continue;
tc_stat.commit_step++; ret = build_index(i, &tch, tmpfd); if (ret <= 0) { close(tmpfd); logf("tagcache failed init"); if (ret == 0) tc_stat.commit_delayed = true;
tc_stat.commit_step = 0; goto commit_error; } do_timed_yield(); }
if (!build_numeric_indices(&tch, tmpfd)) { logf("Failure to commit numeric indices"); close(tmpfd); tc_stat.commit_step = 0; goto commit_error; }
close(tmpfd);
tc_stat.commit_step = 0;
if (!USR_CANCEL) { /* Update the master index headers. */ if ( (masterfd = open_master_fd(&tcmh, true)) < 0) goto commit_error;
remove_db_file(TAGCACHE_FILE_TEMP);
tcmh.tch.entry_count += tch.entry_count; tcmh.tch.datasize = sizeof(struct master_header) + sizeof(struct index_entry) * tcmh.tch.entry_count + tch.datasize; tcmh.dirty = false; tcmh.commitid++;
lseek(masterfd, 0, SEEK_SET); write_master_header(masterfd, &tcmh); close(masterfd);
logf("tagcache committed"); tagcache_commit_finalize();
#if defined(HAVE_TC_RAMCACHE) if (ramcache_buffer_stolen) { tempbuf = NULL; tempbuf_size = 0; ramcache_buffer_stolen = false; tcrc_buffer_unlock(); }
/* Reload tagcache only if we committed new entries */ if (tc_stat.ramcache_allocated > 0 && tch.entry_count > 0) tagcache_start_scan();#endif /* HAVE_TC_RAMCACHE */
rc = true; } /*!USR_CANCEL*/
commit_error:#ifdef HAVE_TC_RAMCACHE if (ramcache_buffer_stolen) { tempbuf = NULL; tempbuf_size = 0; tcrc_buffer_unlock(); }#endif /* HAVE_TC_RAMCACHE */
read_lock--;
#ifdef HAVE_DIRCACHE /* Resume the dircache, if we stole the buffer. */ if (dircache_buffer_stolen) { free_tempbuf(); dircache_resume(); }#endif /* HAVE_DIRCACHE */
return rc;}
void tagcache_commit_finalize(void){ tc_stat.ready = check_all_headers(); tc_stat.readyvalid = true;}
#if !defined(PLUGIN)#ifndef __PCTOOL__
static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data){ struct index_entry idx;
if (!tc_stat.ready) return false;
if (!TAGCACHE_IS_NUMERIC(tag)) return false;
if (!get_index(masterfd, idx_id, &idx, false)) return false;
idx.tag_seek[tag] = data; idx.flag |= FLAG_DIRTYNUM;
return write_index(masterfd, idx_id, &idx);}
#if 0bool tagcache_modify_numeric_entry(struct tagcache_search *tcs, int tag, long data){ struct master_header myhdr;
if (tcs->masterfd < 0) { if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0) return false; }
return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);}#endif
static bool command_queue_is_full(void){ int next;
next = command_queue_widx + 1; if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH) next = 0;
return (next == command_queue_ridx);}
static void command_queue_sync_callback(void){ struct master_header myhdr; int masterfd;
mutex_lock(&command_queue_mutex);
if ( (masterfd = open_master_fd(&myhdr, true)) < 0) return;
while (command_queue_ridx != command_queue_widx) { struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
switch (ce->command) { case CMD_UPDATE_MASTER_HEADER: { close(masterfd); update_master_header();
/* Re-open the masterfd. */ if ( (masterfd = open_master_fd(&myhdr, true)) < 0) return;
break; } case CMD_UPDATE_NUMERIC: { modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data); break; } }
if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH) command_queue_ridx = 0; }
close(masterfd);
tc_stat.queue_length = 0; mutex_unlock(&command_queue_mutex);}
static void run_command_queue(bool force){ if (COMMAND_QUEUE_IS_EMPTY) return;
if (force || command_queue_is_full()) command_queue_sync_callback(); else register_storage_idle_func(command_queue_sync_callback);}
static void queue_command(int cmd, long idx_id, int tag, long data){ while (1) { int next;
mutex_lock(&command_queue_mutex); next = command_queue_widx + 1; if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH) next = 0;
/* Make sure queue is not full. */ if (next != command_queue_ridx) { struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
ce->command = cmd; ce->idx_id = idx_id; ce->tag = tag; ce->data = data;
command_queue_widx = next;
tc_stat.queue_length++;
mutex_unlock(&command_queue_mutex); break; }
/* Queue is full, try again later... */ mutex_unlock(&command_queue_mutex); sleep(1); }}
long tagcache_increase_serial(void){ long old;
if (!tc_stat.ready) return -2;
while (read_lock) sleep(1);
old = current_tcmh.serial++; queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
return old;}
void tagcache_update_numeric(int idx_id, int tag, long data){ queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);}#endif /* !__PCTOOL__ */
static bool write_tag(int fd, const char *tagstr, const char *datastr){ char buf[512]; const int bufsz = sizeof(buf); int i;
snprintf(buf, bufsz, "%s=\"", tagstr);
for (i = strlen(buf); i < (long)sizeof(buf)-4; i++) { if (*datastr == '\0') break;
if (*datastr == '"' || *datastr == '\\') buf[i++] = '\\';
else if (*datastr == '\n') { buf[i++] = '\\'; buf[i] = 'n'; continue; }
buf[i] = *(datastr++); }
str_setlen(buf, bufsz - 1); strmemccpy(&buf[i], "\" ", (bufsz - i - 1));
write(fd, buf, i + 2);
return true;}
#ifndef __PCTOOL__
static bool read_tag(char *dest, long size, const char *src, const char *tagstr){ int pos; char current_tag[32];
while (*src != '\0') { /* Skip all whitespace */ while (*src == ' ') src++;
if (*src == '\0') break;
pos = 0; /* Read in tag name */ while (*src != '=' && *src != ' ') { current_tag[pos] = *src; src++; pos++;
if (*src == '\0' || pos >= (int) sizeof(current_tag)) return false; }
str_setlen(current_tag, pos);
/* Read in tag data */
/* Find the start. */ while (*src != '"' && *src != '\0') src++;
if (*src == '\0' || *(++src) == '\0') return false;
/* Read the data. */ for (pos = 0; pos < size; pos++) { if (*src == '\0') break;
if (*src == '\\') { src++; if (*src == 'n') dest[pos] = '\n'; else dest[pos] = *src;
src++; continue; }
if (*src == '\0') break;
if (*src == '"') { src++; break; }
dest[pos] = *(src++); }
str_setlen(dest, pos);
if (!strcasecmp(tagstr, current_tag)) return true; }
return false;}
static int parse_changelog_line(int line_n, char *buf, void *parameters){ struct index_entry idx; char tag_data[TAGCACHE_BUFSZ]; int idx_id; long masterfd = (long)(intptr_t)parameters; const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed, tag_commitid, tag_lastelapsed, tag_lastoffset }; int i; (void)line_n;
if (*buf == '#') return 0;
/* logf("%d/%s", line_n, buf); */ if (!read_tag(tag_data, sizeof tag_data, buf, "filename")) { logf("%d/filename missing", line_n); logf("-> %s", buf); return 0; }
idx_id = find_index(tag_data); if (idx_id < 0) { logf("%d/entry not found", line_n); return 0; }
if (!get_index(masterfd, idx_id, &idx, false)) { logf("%d/failed to retrieve index entry", line_n); return 0; }
/* Stop if tag has already been modified. */ if (idx.flag & FLAG_DIRTYNUM) return 0;
logf("%d/import: %s", line_n, tag_data);
idx.flag |= FLAG_DIRTYNUM; for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++) { int data;
if (!read_tag(tag_data, sizeof tag_data, buf, tagcache_tag_to_str(import_tags[i]))) { continue; }
data = atoi(tag_data); if (data < 0) continue;
idx.tag_seek[import_tags[i]] = data;
if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial) current_tcmh.serial = data + 1; else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid) current_tcmh.commitid = data + 1; }
return write_index(masterfd, idx_id, &idx) ? 0 : -5;}
bool tagcache_import_changelog(void){ struct master_header myhdr; struct tagcache_header tch; int clfd; long masterfd; char buf[2048]; const int bufsz = sizeof(buf);
if (!tc_stat.ready) return false;
while (read_lock) sleep(1);
clfd = open_db_fd(TAGCACHE_FILE_CHANGELOG, O_RDONLY); if (clfd < 0) { logf("failure to open changelog"); return false; }
if ( (masterfd = open_master_fd(&myhdr, true)) < 0) { close(clfd); return false; }
write_lock++;
filenametag_fd = open_tag_fd(&tch, tag_filename, false);
fast_readline(clfd, buf, bufsz, (void *)(intptr_t)masterfd, parse_changelog_line);
close(clfd); close(masterfd);
if (filenametag_fd >= 0) { close(filenametag_fd); filenametag_fd = -1; }
write_lock--;
update_master_header();
return true;}
#endif /* !__PCTOOL__ */
bool tagcache_create_changelog(struct tagcache_search *tcs){ struct master_header myhdr; struct index_entry idx; char buf[TAGCACHE_BUFSZ]; const int bufsz = sizeof(buf); char temp[32]; int clfd; int i, j;
if (!tc_stat.ready) return false;
if (!tagcache_search(tcs, tag_filename)) return false;
/* Initialize the changelog */ clfd = open_db_fd(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC); if (clfd < 0) { logf("failure to open changelog"); tagcache_search_finish(tcs); return false; }
if (tcs->masterfd < 0) { if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0) { close(clfd); tagcache_search_finish(tcs); return false; } } else { lseek(tcs->masterfd, 0, SEEK_SET); read_master_header(tcs->masterfd, &myhdr); }
write(clfd, "## Changelog version 1\n", 23);
for (i = 0; i < myhdr.tch.entry_count; i++) { if (read_index_entries(tcs->masterfd, &idx, 1) != sizeof(struct index_entry)) { logf("read error #9"); tagcache_search_finish(tcs); close(clfd); return false; }
/* Skip until the entry found has been modified. */ if (! (idx.flag & FLAG_DIRTYNUM) ) continue;
/* Skip deleted entries too. */ if (idx.flag & FLAG_DELETED) continue;
/* Now retrieve all tags. */ for (j = 0; j < TAG_COUNT; j++) { if (TAGCACHE_IS_NUMERIC(j)) { itoa_buf(temp, sizeof temp, (int)idx.tag_seek[j]); write_tag(clfd, tagcache_tag_to_str(j), temp); continue; }
tcs->type = j; tagcache_retrieve(tcs, i, tcs->type, buf, bufsz); write_tag(clfd, tagcache_tag_to_str(j), buf); }
write(clfd, "\n", 1); do_timed_yield(); }
close(clfd);
tagcache_search_finish(tcs);
return true;}
static bool delete_entry(long idx_id){ int fd = -1; int masterfd = -1; int tag, i; struct index_entry idx, myidx; struct master_header myhdr; int in_use[TAG_COUNT];
logf("delete_entry(): %ld", idx_id);
#ifdef HAVE_TC_RAMCACHE /* At first mark the entry removed from ram cache. */ if (tc_stat.ramcache) tcramcache.hdr->indices[idx_id].flag |= FLAG_DELETED;#endif
if ( (masterfd = open_master_fd(&myhdr, true) ) < 0) return false;
lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR); if (read_index_entries(masterfd, &myidx, 1) != sizeof(struct index_entry)) { logf("delete_entry(): read error"); goto cleanup; }
if (myidx.flag & FLAG_DELETED) { logf("delete_entry(): already deleted!"); goto cleanup; }
myidx.flag |= FLAG_DELETED; lseek(masterfd, -(off_t)sizeof(struct index_entry), SEEK_CUR); if (write_index_entries(masterfd, &myidx, 1) != sizeof(struct index_entry)) { logf("delete_entry(): write_error #1"); goto cleanup; }
/* Now check which tags are no longer in use (if any) */ for (tag = 0; tag < TAG_COUNT; tag++) in_use[tag] = 0;
lseek(masterfd, sizeof(struct master_header), SEEK_SET); for (i = 0; i < myhdr.tch.entry_count; i++) { struct index_entry *idxp;
#ifdef HAVE_TC_RAMCACHE /* Use RAM DB if available for greater speed */ if (tc_stat.ramcache) idxp = &tcramcache.hdr->indices[i]; else#endif { if (read_index_entries(masterfd, &idx, 1) != sizeof(struct index_entry)) { logf("delete_entry(): read error #2"); goto cleanup; } idxp = &idx; }
if (idxp->flag & FLAG_DELETED) continue;
for (tag = 0; tag < TAG_COUNT; tag++) { if (TAGCACHE_IS_NUMERIC(tag)) continue;
if (idxp->tag_seek[tag] == myidx.tag_seek[tag]) in_use[tag]++; } }
/* Now delete all tags no longer in use. */ for (tag = 0; tag < TAG_COUNT; tag++) { struct tagcache_header tch; int oldseek = myidx.tag_seek[tag];
if (TAGCACHE_IS_NUMERIC(tag)) continue;
/** * Replace tag seek with a hash value of the field string data. * That way runtime statistics of moved or altered files can be * resurrected. */#ifdef HAVE_TC_RAMCACHE if (tc_stat.ramcache && tag != tag_filename) { struct tagfile_entry *tfe; int32_t *seek = &tcramcache.hdr->indices[idx_id].tag_seek[tag];
/* crc_32 is assumed not to yield (why would it...?) */ tfe = (struct tagfile_entry *)&tcramcache.hdr->tags[tag][*seek]; *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff); myidx.tag_seek[tag] = *seek; } else#endif /* HAVE_TC_RAMCACHE */ { struct tagfile_entry tfe;
/* Open the index file, which contains the tag names. */ if ((fd = open_tag_fd(&tch, tag, true)) < 0) goto cleanup;
/* Skip the header block */ lseek(fd, myidx.tag_seek[tag], SEEK_SET);
switch (read_tagfile_entry_and_tag(fd, &tfe, build_idx_buf, build_idx_bufsz)) { case e_SUCCESS_LEN_ZERO: logf("deleted_entry(): SUCCESS"); /* FALL THROUGH */ case e_SUCCESS: break; case e_ENTRY_SIZEMISMATCH: logf("delete_entry(): read error #3"); goto cleanup; case e_TAG_TOOLONG: logf("too long tag #4"); goto cleanup; case e_TAG_SIZEMISMATCH: logf("delete_entry(): read error #3"); goto cleanup; }
myidx.tag_seek[tag] = crc_32(build_idx_buf, strlen(build_idx_buf), 0xffffffff); }
if (in_use[tag]) { logf("in use: %d/%d", tag, in_use[tag]); if (fd >= 0) { close(fd); fd = -1; } continue; }
#ifdef HAVE_TC_RAMCACHE /* Delete from ram. */ if (tc_stat.ramcache && tag != tag_filename) { struct tagfile_entry *tagentry = (struct tagfile_entry *)&tcramcache.hdr->tags[tag][oldseek]; str_setlen(tagentry->tag_data, 0); }#endif /* HAVE_TC_RAMCACHE */
/* Open the index file, which contains the tag names. */ if (fd < 0) { if ((fd = open_tag_fd(&tch, tag, true)) < 0) goto cleanup; }
/* Skip the header block */ lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
/* Debug, print 10 first characters of the tag read(fd, buf, 10); buf[10]='\0'; logf("TAG:%s", buf); lseek(fd, -10, SEEK_CUR); */
/* Write first data byte in tag as \0 */ write(fd, "", 1);
/* Now tag data has been removed */ close(fd); fd = -1; }
/* Write index entry back into master index. */ lseek(masterfd, sizeof(struct master_header) + (idx_id * sizeof(struct index_entry)), SEEK_SET); if (write_index_entries(masterfd, &myidx, 1) != sizeof(struct index_entry)) { logf("delete_entry(): write_error #2"); goto cleanup; }
close(masterfd);
return true;
cleanup: if (fd >= 0) close(fd); if (masterfd >= 0) close(masterfd);
return false;}
/** * Returns true if there is an event waiting in the queue * that requires the current operation to be aborted. */static bool check_event_queue(void){#ifndef __PCTOOL__ struct queue_event ev;
if(!queue_peek(&tagcache_queue, &ev)) return false;
switch (ev.id) { case Q_STOP_SCAN: case SYS_POWEROFF: case SYS_REBOOT: case SYS_USB_CONNECTED: return true; }#endif /* __PCTOOL__ */
return false;}
#ifdef HAVE_TC_RAMCACHE
static void fix_ramcache(void* old_addr, void* new_addr){ ptrdiff_t offpos = new_addr - old_addr; for (int i = 0; i < TAG_COUNT; i++) tcramcache.hdr->tags[i] += offpos;}
static int move_cb(int handle, void* current, void* new){ (void)handle; fix_ramcache(current, new); tcramcache.hdr = new; return BUFLIB_CB_OK;}
static struct buflib_callbacks ops = { .move_callback = move_cb, .shrink_callback = NULL,};
static bool allocate_tagcache(void){ tc_stat.ramcache_allocated = 0; tcramcache.handle = 0; tcramcache.hdr = NULL;
/* Load the header. */ struct master_header tcmh; int fd = open_master_fd(&tcmh, false); if (fd < 0) return false;
close(fd);
/** * Now calculate the required cache size plus * some extra space for alignment fixes. */ size_t alloc_size = tcmh.tch.datasize + 256 + TAGCACHE_RESERVE + sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);#ifdef HAVE_DIRCACHE alloc_size += tcmh.tch.entry_count*sizeof(struct dircache_fileref);#endif
int handle = core_alloc_ex(alloc_size, &ops); if (handle <= 0) return false;
tcramcache.handle = handle; tcramcache.hdr = core_get_data(handle); tc_stat.ramcache_allocated = alloc_size;
memset(tcramcache.hdr, 0, sizeof(struct ramcache_header)); memcpy(¤t_tcmh, &tcmh, sizeof current_tcmh); logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
return true;}
#ifdef HAVE_EEPROM_SETTINGSstatic bool tagcache_dumpload(void){ struct statefile_header shdr; int fd, rc, handle;
tcramcache.handle = 0; tcramcache.hdr = NULL;
fd = open_db_fd(TAGCACHE_STATEFILE, O_RDONLY); if (fd < 0) { logf("no tagcache statedump"); return false; }
/* Check the statefile memory placement */ rc = read(fd, &shdr, sizeof(struct statefile_header)); if (rc != sizeof(struct statefile_header) || shdr.magic != TAGCACHE_STATEFILE_MAGIC || shdr.mh.tch.magic != TAGCACHE_MAGIC) { logf("incorrect statefile"); close(fd); return false; }
/* Lets allocate real memory and load it */ handle = core_alloc_ex(shdr.tc_stat.ramcache_allocated, &ops); if (handle <= 0) { logf("alloc failure"); return false; }
tcramcache.handle = handle; tcrc_buffer_lock(); tcramcache.hdr = core_get_data(handle); rc = read(fd, tcramcache.hdr, shdr.tc_stat.ramcache_allocated); tcrc_buffer_unlock();
close(fd);
if (rc != shdr.tc_stat.ramcache_allocated) { logf("read failure!"); core_free(handle); return false; }
tc_stat = shdr.tc_stat;
/* Now fix the pointers */ fix_ramcache(shdr.hdr, tcramcache.hdr);
/* Load the tagcache master header (should match the actual DB file header). */ memcpy(¤t_tcmh, &shdr.mh, sizeof current_tcmh);
return true;}
static bool tagcache_dumpsave(void){ struct statefile_header shdr; int fd;
if (!tc_stat.ramcache) return false;
fd = open_db_fd(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC); if (fd < 0) { logf("failed to create a statedump"); return false; }
/* Create the header */ shdr.magic = TAGCACHE_STATEFILE_MAGIC; shdr.hdr = tcramcache.hdr; memcpy(&shdr.mh, ¤t_tcmh, sizeof current_tcmh); memcpy(&shdr.tc_stat, &tc_stat, sizeof tc_stat); write(fd, &shdr, sizeof shdr);
/* And dump the data too */ tcrc_buffer_lock(); write(fd, tcramcache.hdr, tc_stat.ramcache_allocated); tcrc_buffer_unlock(); close(fd);
return true;}#endif /* HAVE_EEPROM_SETTINGS */
static bool load_tagcache(void){ /* DEBUG: After tagcache commit and dircache rebuild, hdr-sturcture * may become corrupt. */
bool ok = false; ssize_t bytesleft = tc_stat.ramcache_allocated - sizeof(struct ramcache_header); int fd;
#ifdef HAVE_DIRCACHE /* Wait for any in-progress dircache build to complete */ dircache_wait();#endif /* HAVE_DIRCACHE */
logf("loading tagcache to ram...");
tcrc_buffer_lock(); /* lock for the rest of the scan, simpler to handle */
fd = open_db_fd(TAGCACHE_FILE_MASTER, O_RDONLY); if (fd < 0) { logf("tagcache open failed"); goto failure; }
struct master_header tcmh; if (read_master_header(fd, &tcmh) != sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC) { logf("incorrect header"); goto failure; }
/* Master header copy should already match, this can be redundant to do. */ current_tcmh = tcmh;
/* Load the master index table. */ for (int i = 0; i < tcmh.tch.entry_count; i++) { bytesleft -= sizeof(struct index_entry); if (bytesleft < 0) { logf("too big tagcache."); goto failure; }
int rc = read_index_entries(fd, &tcramcache.hdr->indices[i], 1); if (rc != sizeof (struct index_entry)) { logf("read error #10"); goto failure; } }
close(fd); fd = -1;
/* Load the tags right after the index entries */ char *p = (char *)&tcramcache.hdr->indices[tcmh.tch.entry_count];
for (int tag = 0; tag < TAG_COUNT; tag++) { ssize_t rc;
if (TAGCACHE_IS_NUMERIC(tag)) continue;
p = TC_ALIGN_PTR(p, struct tagcache_header, &rc); bytesleft -= rc; if (bytesleft < (ssize_t)sizeof(struct tagcache_header)) { logf("Too big tagcache #10.5"); goto failure; }
tcramcache.hdr->tags[tag] = p;
/* Load the header */ struct tagcache_header *tch = (struct tagcache_header *)p; p += sizeof(struct tagcache_header); bytesleft -= sizeof (struct tagcache_header);
fd = open_tag_fd(tch, tag, false); if (rc < 0) goto failure;
/* Load the entries for this tag */ for (tcramcache.hdr->entry_count[tag] = 0; tcramcache.hdr->entry_count[tag] < tch->entry_count; tcramcache.hdr->entry_count[tag]++) { /* Abort if we got a critical event in queue */ if (do_timed_yield() && check_event_queue()) goto failure;
p = TC_ALIGN_PTR(p, struct tagfile_entry, &rc); bytesleft -= rc; if (bytesleft < (ssize_t)sizeof(struct tagfile_entry)) { logf("Too big tagcache #10.75"); goto failure; }
struct tagfile_entry *fe = (struct tagfile_entry *)p; off_t pos = lseek(fd, 0, SEEK_CUR);
/* Load the header for the tag itself */ if (read_tagfile_entry(fd, fe) != sizeof(struct tagfile_entry)) { /* End of lookup table. */ logf("read error #11"); goto failure; }
int idx_id = fe->idx_id; /* dircache reference clobbers *fe */ struct index_entry *idx = &tcramcache.hdr->indices[idx_id];
if (idx_id != -1 || tag == tag_filename) /* filename NOT optional */ { if (idx_id < 0 || idx_id >= tcmh.tch.entry_count) { logf("corrupt tagfile entry:tag=%d:idxid=%d", tag, idx_id); goto failure; }
if (idx->tag_seek[tag] != pos) { logf("corrupt data structures!:"); logf(" tag_seek[%d]=%" PRId32 ":pos=%ld", tag, idx->tag_seek[tag], (long) pos); goto failure; } }
/* We have a special handling for the filename tags; neither the paths nor the entry headers are stored; only the tagcache header and dircache references are. */ if (tag == tag_filename) { #ifdef HAVE_DIRCACHE if (idx->flag & FLAG_DIRCACHE) { /* This flag must not be used yet. */ logf("internal error!"); goto failure; }
p += sizeof (struct dircache_fileref); bytesleft -= sizeof (struct dircache_fileref); #endif /* HAVE_DIRCACHE */
char filename[TAGCACHE_BUFSZ]; if (fe->tag_length >= (long)sizeof(filename)-1) { read(fd, filename, 10); str_setlen(filename, 10); logf("TAG:%s", filename); logf("too long filename"); goto failure; }
if ((idx->flag & FLAG_DELETED) IFN_DIRCACHE( || !global_settings.tagcache_autoupdate )) { /* seek over tag data instead of reading */ if (lseek(fd, fe->tag_length, SEEK_CUR) < 0) { logf("read error #11.5"); goto failure; }
continue; }
if (read(fd, filename, fe->tag_length) != fe->tag_length) { logf("read error #12"); goto failure; } continue; }
bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length; if (bytesleft < 0) { logf("too big tagcache #2"); logf("tl: %" PRId32, fe->tag_length); logf("bl: %ld", (long) bytesleft); goto failure; }
p = fe->tag_data; rc = read(fd, p, fe->tag_length); p += rc;
if (rc != fe->tag_length) { logf("read error #13"); logf("rc=0x%04x", (unsigned int)rc); // 0x431 logf("len=0x%04" PRIx32, fe->tag_length); // 0x4000 logf("pos=0x%04lx", (unsigned long) lseek(fd, 0, SEEK_CUR)); // 0x433 logf("tag=0x%02x", tag); // 0x00 goto failure; } }
#ifdef HAVE_DIRCACHE if (tag == tag_filename) p = (char *)&tcrc_dcfrefs[tcmh.tch.entry_count]; #endif /* HAVE_DIRCACHE */
close(fd); }
tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft; logf("tagcache loaded into ram!"); logf("utilization: %d%%", 100*tc_stat.ramcache_used / tc_stat.ramcache_allocated);
ok = true;
failure: if (fd >= 0) close(fd);
tcrc_buffer_unlock(); return ok;}#endif /* HAVE_TC_RAMCACHE */
static bool check_deleted_file_refs(bool auto_update){ int fd; bool ret = true; char buf[TAGCACHE_BUFSZ]; const int bufsz = sizeof(buf); struct tagfile_entry tfe; struct tagcache_header hdr;
logf("reverse scan...");
#ifdef HAVE_DIRCACHE if (tcramcache.handle > 0) tcrc_buffer_lock(); else return false; /* Wait for any in-progress dircache build to complete */ dircache_wait();#else if (!auto_update) return false;#endif
fd = open_tag_fd(&hdr, tag_filename, false); /* open read only*/
if (fd < 0) { logf(TAGCACHE_FILE_INDEX " open fail", tag_filename); return false; }
processed_dir_count = 0;
while (!check_event_queue()) { int res = read_tagfile_entry_and_tag(fd, &tfe, buf, bufsz); processed_dir_count++;
switch (res) { case e_ENTRY_SIZEMISMATCH: logf("size mismatch entry EOF?"); /* likely EOF */ ret = false; goto wend_finished; case e_TAG_TOOLONG: logf("too long tag"); ret = false; goto wend_finished; case e_TAG_SIZEMISMATCH: logf("size mismatch tag - read error #14"); ret = false; goto wend_finished; case e_SUCCESS: break; case e_SUCCESS_LEN_ZERO: continue; }
int idx_id = tfe.idx_id; /* dircache reference clobbers *tfe */#ifdef HAVE_DIRCACHE struct index_entry *idx = &tcramcache.hdr->indices[idx_id]; unsigned int searchflag; if (!auto_update) { if(idx->flag & FLAG_DIRCACHE) /* already found */ { continue; } searchflag = DCS_CACHED_PATH; /* attempt to load cache references */ } else /* If auto updating, check storage too */ { searchflag = DCS_STORAGE_PATH; }
int rc_cache = dircache_search(searchflag | DCS_UPDATE_FILEREF, &tcrc_dcfrefs[idx_id], buf);
if (rc_cache > 0) /* in cache and we have fileref */ { idx->flag |= FLAG_DIRCACHE; } else if (rc_cache == 0) /* not in cache but okay */ {;} else if (auto_update && rc_cache == ENOENT)#else if (!file_exists(buf))#endif /* HAVE_DIRCACHE */ { logf("Entry no longer valid."); logf("-> %s / %" PRId32, buf, tfe.tag_length); delete_entry(idx_id); }
do_timed_yield(); }
wend_finished:
#ifdef HAVE_DIRCACHE if (tcramcache.handle > 0) tcrc_buffer_unlock();#endif close(fd); logf("done");
return ret;}
/* Note that this function must not be inlined, otherwise the whole point * of having the code in a separate function is lost. */static void NO_INLINE check_ignore(const char *dirname, int *ignore, int *unignore){ char newpath[MAX_PATH]; const int bufsz = sizeof(newpath);
/* check for a database.ignore file */ snprintf(newpath, bufsz, "%s/database.ignore", dirname); *ignore = file_exists(newpath); /* check for a database.unignore file */ snprintf(newpath, bufsz, "%s/database.unignore", dirname); *unignore = file_exists(newpath);}
/* max roots on native. on application more can be added via malloc() */#define MAX_STATIC_ROOTS 12
static struct search_roots_ll { const char *path; struct search_roots_ll * next;} roots_ll[MAX_STATIC_ROOTS];
/* check if the path is already included in the search roots, by the * means that the path itself or one of its parents folders is in the list */static bool search_root_exists(const char *path){ struct search_roots_ll *this; for(this = &roots_ll[0]; this; this = this->next) { size_t root_len = strlen(this->path); /* check if the link target is inside of an existing search root * don't add if target is inside, we'll scan it later */ if (!strncmp(this->path, path, root_len)) return true; } return false;}
#ifdef APPLICATION/* * This adds a path to the search roots, possibly during traveling through * the filesystem. It only adds if the path is not inside an already existing * search root. * * Returns true if it added the path to the search roots * * Windows 2000 and greater supports symlinks, but they don't provide * realpath() or readlink(), and symlinks are rarely used on them so * ignore this for windows for now **/static bool add_search_root(const char *name){ (void)name;#ifndef WIN32 struct search_roots_ll *this, *prev = NULL; char target[MAX_PATH]; const int target_bufsz = sizeof(target); /* Okay, realpath() is almost completely broken on android * * It doesn't accept NULL for resolved_name to dynamically allocate * the resulting path; and it assumes resolved_name to be PATH_MAX * (actually MAXPATHLEN, but it's the same [as of 2.3]) long * and blindly writes to the end if it * * therefore use sufficiently large static storage here * Note that PATH_MAX != MAX_PATH **/ static char abs_target[PATH_MAX]; ssize_t len;
len = readlink(name, target, target_bufsz-1); if (len < 0) return false;
str_setlen(target, len); if (realpath(target, abs_target) == NULL) return false;
if (search_root_exists(abs_target)) return false;
/* get the end of the list */ for(this = &roots_ll[0]; this; prev = this, this = this->next);
if (prev) { size_t len = strlen(abs_target) + 1; /* count \0 */ this = malloc(sizeof(struct search_roots_ll) + len ); if (!this || len > MIN(PATH_MAX, MAX_PATH)) { logf("Error at adding a search root: %s", this ? "path too long":"OOM"); free(this); prev->next = NULL; return false; } this->path = ((char*)this) + sizeof(struct search_roots_ll); strcpy((char*)this->path, abs_target); /* ok to cast const away here */ this->next = NULL; prev->next = this; logf("Added %s to the search roots\n", abs_target); return true; }#endif return false;}
static int free_search_root_single(struct search_roots_ll * start){ if (start < &roots_ll[0] && start >= &roots_ll[MAX_STATIC_ROOTS]) { free(start->next); return sizeof(struct search_roots_ll); } return 0;}
static int free_search_roots(struct search_roots_ll * start){ int ret = 0; if (start->next) { ret += free_search_root_single(start->next); } return ret;}#else /* native, simulator */#define add_search_root(a) do {} while(0)#define free_search_roots(a) do {} while(0)#endif
static bool check_dir(const char *dirname, int add_files){ int success = false;
DIR *dir = opendir(dirname); if (!dir) { logf("tagcache: opendir(%s) failed", dirname); return false; }
/* check for a database.ignore and database.unignore */ int ignore, unignore; check_ignore(dirname, &ignore, &unignore);
/* don't do anything if both ignore and unignore are there */ if (ignore != unignore) add_files = unignore;
/* Recursively scan the dir. */ while (!check_event_queue()) { struct dirent *entry = readdir(dir); if (entry == NULL) { success = true; break; }
if (is_dotdir_name(entry->d_name)) continue;
struct dirinfo info = dir_get_info(dir, entry); size_t len = strlen(curpath); path_append(&curpath[len-1], PA_SEP_HARD, entry->d_name, sizeof (curpath) - len);
processed_dir_count++; if (info.attribute & ATTR_DIRECTORY) {#ifndef SIMULATOR /* don't follow symlinks to dirs, but try to add it as a search root * this makes able to avoid looping in recursive symlinks */ if (info.attribute & ATTR_LINK) add_search_root(curpath); else#endif /* SIMULATOR */ check_dir(curpath, add_files); } else if (add_files) { tc_stat.curentry = curpath;
/* Add a new entry to the temporary db file. */ add_tagcache(curpath, info.mtime);
/* Wait until current path for debug screen is read and unset. */ while (tc_stat.syncscreen && tc_stat.curentry != NULL) yield();
tc_stat.curentry = NULL; }
str_setlen(curpath, len); }
closedir(dir);
return success;}
void tagcache_screensync_event(void){ tc_stat.curentry = NULL;}
void tagcache_screensync_enable(bool state){ tc_stat.syncscreen = state;}
#ifndef __PCTOOL__/* this is called by the database tool to not pull in global_settings */static#endifvoid do_tagcache_build(const char *path[]){ struct tagcache_header header; bool ret;
str_setlen(curpath, 0); data_size = 0; total_entry_count = 0; processed_dir_count = 0;
#ifdef HAVE_DIRCACHE dircache_wait();#endif
logf("updating tagcache");
cachefd = open_db_fd(TAGCACHE_FILE_TEMP, O_RDONLY); if (cachefd >= 0) { logf("skipping, cache already waiting for commit"); close(cachefd); return ; }
cachefd = open_db_fd(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC); if (cachefd < 0) { logf("master file open failed: %s", TAGCACHE_FILE_TEMP); return ; }
filenametag_fd = open_tag_fd(&header, tag_filename, false);
cpu_boost(true);
logf("Scanning files..."); /* Scan for new files. */ memset(&header, 0, sizeof(struct tagcache_header)); write(cachefd, &header, sizeof(struct tagcache_header));
ret = true;
if (path[0] == NULL) { logf("tagcache path empty, skipping..."); goto commit; }
roots_ll[0].path = path[0]; roots_ll[0].next = NULL;
#if defined(HAVE_MULTIVOLUME) && !defined(SIMULATOR) && !defined(__PCTOOL__) && !defined(APPLICATION) extern bool ns_volume_is_visible(int volume); /*rb_namespace.c*/ /* i is for the path vector, j for the roots_ll array */ int i = 1, j = 1; bool added = false; char volnamebuf[NUM_VOLUMES][VOL_MAX_LEN + 1]; /* we can just parse the root directory ('/') and get to any mounted * volume but we can also enumerate a volume in the root directory * when this occurs it leads to multiple entries since the files can * be reached through multiple paths ex, /Foo could also be /SD1/Foo * we used to hide the volume that was mapped but then when you switch * from the sd to the internal the paths don't map to the right volume * instead we will attempt to rewrite the root with any non-hidden volumes * failing that just leave the paths alone */ if (!strcmp(PATH_ROOTSTR, path[0])) { i = 0; j = 0; } /* path can be skipped , but root_ll entries can't */ for(; path[i] && j < MAX_STATIC_ROOTS; i++) { /* check if the link target is inside of an existing search root * don't add if target is inside, we'll scan it later */ if (!added && !strcmp(PATH_ROOTSTR, path[i])) { for (int v = 0; v < NUM_VOLUMES; v++) { if (ns_volume_is_visible(v)) { make_volume_root(v, volnamebuf[v]); roots_ll[j].path = volnamebuf[v]; if (j > 0) roots_ll[j-1].next = &roots_ll[j]; j++; added = true; } } if(!added) j = 1; added = true; continue; }#else /* i is for the path vector, j for the roots_ll array * path can be skipped , but root_ll entries can't */ for(int i = 1, j = 1; path[i] && j < MAX_STATIC_ROOTS; i++) {#endif /*def HAVE_MULTIVOLUME*/ if (search_root_exists(path[i])) /* skip this path */ continue;
roots_ll[j].path = path[i]; roots_ll[j-1].next = &roots_ll[j]; j++; }
struct search_roots_ll * this; /* check_dir might add new roots */ for(this = &roots_ll[0]; this; this = this->next) { logf("Search root %s", this->path); strmemccpy(curpath, this->path, sizeof(curpath));
if (ret) { if (dir_exists(this->path)) ret = check_dir(this->path, true); else logf("Dir not found %s", this->path); } } free_search_roots(&roots_ll[0]);
commit: /* Write the header. */ header.magic = TAGCACHE_MAGIC; header.datasize = data_size; header.entry_count = total_entry_count; lseek(cachefd, 0, SEEK_SET); write(cachefd, &header, sizeof(struct tagcache_header)); close(cachefd);
if (filenametag_fd >= 0) { close(filenametag_fd); filenametag_fd = -1; }
if (!ret) { logf("Aborted."); cpu_boost(false); return ; }
/* Commit changes to the database. */#ifdef __PCTOOL__ allocate_tempbuf();#endif if (commit()) { logf("tagcache built!"); }#ifdef __PCTOOL__ free_tempbuf();#endif
#ifdef HAVE_TC_RAMCACHE if (tcramcache.hdr) { /* Import runtime statistics if we just initialized the db. */ if (current_tcmh.serial == 0) queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0); }#endif
cpu_boost(false);}
#ifndef __PCTOOL__void tagcache_build(void){ char *vect[MAX_STATIC_ROOTS + 1]; /* +1 to ensure NULL sentinel */ char str[sizeof(global_settings.tagcache_scan_paths)]; strmemccpy(str, global_settings.tagcache_scan_paths, sizeof(str));
int res = split_string(str, ':', vect, MAX_STATIC_ROOTS); vect[res] = NULL;
do_tagcache_build((const char**)vect);}#endif /* __PCTOOL__ */
#ifdef HAVE_TC_RAMCACHEstatic void load_ramcache(void){ if (!tcramcache.hdr) return ;
cpu_boost(true);
/* At first we should load the cache (if exists). */ tc_stat.ramcache = load_tagcache();
if (!tc_stat.ramcache) { /* If loading failed, it must indicate some problem with the db * so disable it entirely to prevent further issues. */ tc_stat.ready = false; tcramcache.hdr = NULL; int handle = tcramcache.handle; tcramcache.handle = 0; core_free(handle); }
cpu_boost(false);}
void tagcache_unload_ramcache(void){ tc_stat.ramcache = false; /* Just to make sure there is no statefile present. */ // remove_db_file(TAGCACHE_STATEFILE);}#endif /* HAVE_TC_RAMCACHE */
#ifndef __PCTOOL__
/* * db_file_exists is noinline to minimize stack usage */static bool NO_INLINE db_file_exists(const char* filename){ char buf[MAX_PATH];
snprintf(buf, sizeof(buf), "%s/%s", tc_stat.db_path, filename);
return file_exists(buf);}
static void tagcache_thread(void){ struct queue_event ev; bool check_done = false; cpu_boost(true); /* If the previous cache build/update was interrupted, commit * the changes first in foreground. */ if (db_file_exists(TAGCACHE_FILE_TEMP)) {#if !(defined(__APPLE__) && (CONFIG_PLATFORM & PLATFORM_SDL)) static const char *lines[] = {ID2P(LANG_TAGCACHE_BUSY), ID2P(LANG_TAGCACHE_UPDATE)}; static const struct text_message message = {lines, 2};
if (gui_syncyesno_run_w_tmo(HZ * 5, YESNO_YES, str(LANG_TAGCACHE), &message, NULL, NULL) == YESNO_YES)#endif { allocate_tempbuf(); commit(); free_tempbuf(); } }
#ifdef HAVE_TC_RAMCACHE#ifdef HAVE_EEPROM_SETTINGS if (firmware_settings.initialized && firmware_settings.disk_clean && global_settings.tagcache_ram) { check_done = tagcache_dumpload(); }
remove_db_file(TAGCACHE_STATEFILE);#endif /* HAVE_EEPROM_SETTINGS */
/* Allocate space for the tagcache if found on disk. */ if (global_settings.tagcache_ram && !tc_stat.ramcache) allocate_tagcache();#endif /* HAVE_TC_RAMCACHE */
cpu_boost(false); tc_stat.initialized = true;
/* Don't delay bootup with the header check but do it on background. */ if (!tc_stat.ready) { sleep(HZ); tagcache_commit_finalize(); }
while (1) { run_command_queue(false);
queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
switch (ev.id) { case Q_IMPORT_CHANGELOG: tagcache_import_changelog(); break;
case Q_REBUILD: remove_files(); remove_db_file(TAGCACHE_FILE_TEMP); tagcache_build(); break;
case Q_UPDATE: tagcache_build();#ifdef HAVE_TC_RAMCACHE load_ramcache();#endif check_deleted_file_refs(true); break ;
case Q_START_SCAN: check_done = false; /* fallthrough */ case SYS_TIMEOUT: if (check_done || !tc_stat.ready) break ;
#ifdef HAVE_TC_RAMCACHE if (!tc_stat.ramcache && global_settings.tagcache_ram) { load_ramcache(); if (global_settings.tagcache_ram == TAGCACHE_RAM_ON) check_deleted_file_refs(global_settings.tagcache_autoupdate); if (tc_stat.ramcache && global_settings.tagcache_autoupdate) tagcache_build(); } else#endif /* HAVE_RC_RAMCACHE */ if (global_settings.tagcache_autoupdate) { tagcache_build();
/* This will be very slow unless dircache is enabled or target is flash based, but do it anyway for consistency. */ check_deleted_file_refs(true); }
logf("tagcache check done");
check_done = true; break ;
case Q_STOP_SCAN: break ;
case SYS_POWEROFF: case SYS_REBOOT: break ;
case SYS_USB_CONNECTED: logf("USB: TagCache"); usb_acknowledge(SYS_USB_CONNECTED_ACK, ev.data); usb_wait_for_disconnect(&tagcache_queue); break ; } }}
bool tagcache_prepare_shutdown(void){ if (tagcache_get_commit_step() > 0) return false;
tagcache_stop_scan(); while (read_lock || write_lock) sleep(1);
return true;}
void tagcache_shutdown(void){ /* Flush the command queue. */ run_command_queue(true);
#if defined(HAVE_EEPROM_SETTINGS) && defined(HAVE_TC_RAMCACHE) if (tc_stat.ramcache) tagcache_dumpsave();#endif}
void tagcache_remove_statefile(void){ remove_db_file(TAGCACHE_STATEFILE);}
static int get_progress(void){ int total_count = -1;
#ifdef HAVE_DIRCACHE struct dircache_info dcinfo; dircache_get_info(&dcinfo); if (dcinfo.status != DIRCACHE_IDLE && ((!tc_stat.ramcache) || current_tcmh.tch.entry_count == 0)) { total_count = dcinfo.entry_count; } else#endif /* HAVE_DIRCACHE */#ifdef HAVE_TC_RAMCACHE { if (tcramcache.hdr && tc_stat.ramcache) total_count = current_tcmh.tch.entry_count; }#endif /* HAVE_TC_RAMCACHE */
if (total_count < 0) return -1; else if (total_count == 0) return 0; else if (processed_dir_count > total_count) return 100; else return processed_dir_count * 100 / total_count;}
struct tagcache_stat* tagcache_get_stat(void){ tc_stat.total_entries = current_tcmh.tch.entry_count; tc_stat.progress = get_progress(); tc_stat.processed_entries = processed_dir_count;
return &tc_stat;}
void tagcache_start_scan(void){ queue_post(&tagcache_queue, Q_START_SCAN, 0);}
bool tagcache_update(void){ if (!tc_stat.ready) return false;
queue_post(&tagcache_queue, Q_UPDATE, 0); return false;}
bool tagcache_rebuild(void){ queue_post(&tagcache_queue, Q_REBUILD, 0); return false;}
void tagcache_stop_scan(void){ queue_post(&tagcache_queue, Q_STOP_SCAN, 0);}
#endif /* !__PCTOOL__ */
void tagcache_init(void){ memset(&tc_stat, 0, sizeof(struct tagcache_stat)); memset(¤t_tcmh, 0, sizeof(struct master_header)); filenametag_fd = -1; write_lock = read_lock = 0;
#ifndef __PCTOOL__ strmemccpy(tc_stat.db_path, global_settings.tagcache_db_path, sizeof(tc_stat.db_path)); mutex_init(&command_queue_mutex); queue_init(&tagcache_queue, true); create_thread(tagcache_thread, tagcache_stack, sizeof(tagcache_stack), 0, tagcache_thread_name IF_PRIO(, PRIORITY_BACKGROUND) IF_COP(, CPU));#else /* use default DB path */ strcpy(tc_stat.db_path, ROCKBOX_DIR); tc_stat.initialized = true; allocate_tempbuf(); commit(); free_tempbuf(); tc_stat.ready = check_all_headers();#endif}
#ifdef __PCTOOL__void tagcache_reverse_scan(void){ logf("Checking for deleted files"); check_deleted_file_refs(true);}#endif
bool tagcache_is_initialized(void){ return tc_stat.initialized;}bool tagcache_is_fully_initialized(void){ return tc_stat.readyvalid;}bool tagcache_is_usable(void){ return tc_stat.initialized && tc_stat.ready;}#ifdef HAVE_TC_RAMCACHEbool tagcache_is_in_ram(void){ return tc_stat.ramcache;}#endifint tagcache_get_commit_step(void){ return tc_stat.commit_step;}int tagcache_get_max_commit_step(void){ return (int)(SORTED_TAGS_COUNT)+1;}#endif /*!defined(PLUGIN)*/