jetstream v2 in zig stream.waow.tech
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788{ "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.4.0" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } ], "annotations": { "list": [ { "builtIn": 1, "name": "Annotations & Alerts", "type": "dashboard", "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)" }, { "name": "Phase transitions", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "enable": true, "hide": false, "iconColor": "#4a3aa7", "expr": "changes(max(jetstream_orchestrator_phase{job=~\"$job\",instance=~\"$instance\"})[$__interval:]) > 0", "titleFormat": "phase transition", "textFormat": "orchestrator phase changed" } ] }, "description": "Stream operational dashboard adapted from the checksum-pinned Jetstream V2 dashboard. Protocol and lifecycle rows are unchanged; the runtime row uses Stream process metrics, and final Stream-only rows expose backfill work over time and scheduler health.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [ { "title": "jetstream repo", "url": "https://github.com/bluesky-social/jetstream", "type": "link", "icon": "external link", "targetBlank": true, "tags": [], "asDropdown": false, "includeVars": false, "keepTime": false } ], "panels": [ { "id": 9, "type": "row", "title": "Overview", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "panels": [] }, { "id": 1, "type": "stat", "title": "Phase", "description": "Ingestion lifecycle phase. 0 is reserved for a process that never reached the dispatch loop.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 0, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "type": "value", "options": { "0": { "text": "unset", "color": "#898781", "index": 0 }, "1": { "text": "bootstrap", "color": "#2a78d6", "index": 1 }, "2": { "text": "merging", "color": "#eda100", "index": 2 }, "3": { "text": "steady state", "color": "#008300", "index": 3 } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "jetstream_orchestrator_phase{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 2, "type": "stat", "title": "Build", "description": "Version label from jetstream_build_info.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 3, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "name", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "jetstream_build_info{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "{{version}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 3, "type": "stat", "title": "Uptime", "description": "Uptime of the most recently started selected instance — shortest uptime surfaces recent restarts.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 6, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "s" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "time() - max(process_start_time_seconds{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 4, "type": "stat", "title": "Firehose events/s", "description": "Upstream subscribeRepos events decoded per second.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 9, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "cps", "decimals": 1 }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_events_received_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 5, "type": "stat", "title": "Appended events/s", "description": "Events durably appended to the active segment per second. During backfill this is dominated by repo downloads.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 12, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "cps", "decimals": 1 }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_ingest_events_appended_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 6, "type": "stat", "title": "Upstream freshness", "description": "Seconds since steady-state live ingest last saw an upstream event. No data until steady state.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 15, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#eda100", "value": 60 }, { "color": "#e34948", "value": 300 } ] }, "unit": "s", "noValue": "pre-steady-state" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max((time() - jetstream_livestream_last_seen_upstream_event_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"}) and (jetstream_livestream_last_seen_upstream_event_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"} > 0))", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 7, "type": "stat", "title": "Data dir free", "description": "Free bytes on the filesystem holding the data directory.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 18, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#e34948", "value": null }, { "color": "#eda100", "value": 50000000000.0 }, { "color": "green", "value": 250000000000.0 } ] }, "unit": "bytes" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "min(jetstream_data_dir_free_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 8, "type": "stat", "title": "Subscribers", "description": "Connected /subscribe websocket clients.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 21, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_subscribe_subscribers{job=~\"$job\",instance=~\"$instance\"}) or vector(0)", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 15, "type": "row", "title": "Live firehose", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "panels": [] }, { "id": 10, "type": "timeseries", "title": "Firehose intake", "description": "Upstream subscribeRepos decode results. Unknown events are archival holes a jetstream upgrade would fix.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "events received/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "decode errors/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "unknown events/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eb6834" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_events_received_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "events received/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_decode_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "decode errors/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_unknown_events_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "unknown events/s", "refId": "C", "range": true, "instant": false } ] }, { "id": 11, "type": "timeseries", "title": "Relay sequence gaps", "description": "Forward gaps in the relay's seq stream: upstream data loss, not local decode trouble. Should be flat at zero.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "gaps/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "missed seqs/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eb6834" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_sequence_gaps_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "gaps/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_sequence_gap_missed_seqs_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "missed seqs/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 12, "type": "timeseries", "title": "Connection health", "description": "Websocket reconnects and op=-1 error frames from the relay. Error-frame series appear only after the first occurrence.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 14 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "reconnects/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eda100" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_reconnects_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "reconnects/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (code) (rate(jetstream_livestream_stream_error_frames_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "error frame {{code}}/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 13, "type": "timeseries", "title": "Replay & verify drops", "description": "Deliberate steady-state drops (relay seq replays, stale resyncs) vs verify-queue overflow, which is permanent archival loss.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 14 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "verify queue drops/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_replayed_account_events_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "replayed #account/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_replayed_identity_events_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "replayed #identity/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_stale_resyncs_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "stale resyncs/s", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_verify_queue_dropped_events_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "verify queue drops/s", "refId": "D", "range": true, "instant": false } ] }, { "id": 14, "type": "timeseries", "title": "Verifier failures", "description": "Verifier rejections by error class. A sustained 'other' rate means a new class needs categorizing.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 14 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind) (rate(jetstream_verifier_failures_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{kind}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 22, "type": "row", "title": "Ingest writer & segments", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "panels": [] }, { "id": 16, "type": "timeseries", "title": "Append throughput", "description": "Writer.Append successes and failures. Append errors indicate writer trouble and should be flat at zero.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 23 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "events appended/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "append errors/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_ingest_events_appended_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "events appended/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_ingest_append_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "append errors/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 17, "type": "timeseries", "title": "Dropped at validation gate", "description": "Upstream records dropped at the ingest validation gate. Spec-invalid input vs unrepresentable vs upstream omissions.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 23 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (source, reason) (rate(jetstream_ingest_dropped_events_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{source}}/{{reason}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 18, "type": "timeseries", "title": "Blocks & rotations", "description": "zstd block fsync rate and segment rotations (rotations shown as count per interval — they are rare).", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 23 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_ingest_blocks_flushed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "blocks flushed/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(jetstream_ingest_segments_rotated_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "segments rotated", "refId": "B", "range": true, "instant": false } ] }, { "id": 19, "type": "timeseries", "title": "Active segment size", "description": "Compressed bytes written since the active segment's header. Saw-tooth resets on rotation.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 31 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "active segment bytes" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_ingest_active_segment_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "active segment bytes", "refId": "A", "range": true, "instant": false } ] }, { "id": 20, "type": "timeseries", "title": "Segment seal latency", "description": "End-to-end Writer.Seal duration (flush + footer + header pwrite + fsyncs), successful seals only.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 31 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_segment_seal_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_segment_seal_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "B", "range": true, "instant": false } ] }, { "id": 21, "type": "timeseries", "title": "Readable log", "description": "In-memory readable log serving hot /subscribe reads. Pinned overrun means retention budget exceeded by undurable bytes.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 31 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "pinned overrun bytes" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_ingest_readable_log_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "resident bytes", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_ingest_readable_log_pinned_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "pinned bytes", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_ingest_readable_log_pinned_overrun_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "pinned overrun bytes", "refId": "C", "range": true, "instant": false } ] }, { "id": 31, "type": "row", "title": "Backfill & retry", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }, "panels": [] }, { "id": 23, "type": "stat", "title": "Backfill progress", "description": "Repos completed this Run vs DIDs discovered via listRepos. Approximate: discovery keeps running during backfill.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 40 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#2a78d6", "value": null } ] }, "unit": "percentunit", "decimals": 1 }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_backfill_progress_completed{job=~\"$job\",instance=~\"$instance\"}) / clamp_min(sum(jetstream_backfill_discovered_total{job=~\"$job\",instance=~\"$instance\"}), 1)", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 24, "type": "stat", "title": "Repos completed", "description": "Repos the engine reported complete in the current Run.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 40 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_backfill_progress_completed{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 25, "type": "stat", "title": "Repos failed (run)", "description": "DIDs that exhausted their retry budget within the Run.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 40 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "#eda100", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_backfill_failed_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 26, "type": "stat", "title": "Completion queue depth", "description": "Repo completions waiting for a writer durable metadata batch.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 40 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "#eda100", "value": 1000 }, { "color": "#e34948", "value": 10000 } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_backfill_completion_queue_depth{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 27, "type": "timeseries", "title": "Repo throughput", "description": "listRepos discovery vs getRepo completion vs terminal failure.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "completed/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "failed/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "discovered/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_discovered_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "discovered/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_completed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "completed/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_failed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "failed/s", "refId": "C", "range": true, "instant": false } ] }, { "id": 28, "type": "timeseries", "title": "HandleRepo latency", "description": "Per-repo download+append duration, successful repos only.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_backfill_handle_repo_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(jetstream_backfill_handle_repo_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_backfill_handle_repo_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "C", "range": true, "instant": false } ] }, { "id": 29, "type": "timeseries", "title": "Completion durability wait", "description": "Time a repo completion waited between being queued and becoming durable. A rising tail is a durability backlog.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_backfill_completion_queue_wait_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_backfill_completion_queue_wait_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "B", "range": true, "instant": false } ] }, { "id": 30, "type": "timeseries", "title": "Steady-state failed-repo retry", "description": "Retry loop that drains repos which failed initial backfill.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 52 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "succeeded/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "failed/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "skipped (host parked)/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eda100" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_failed_repo_retry_attempts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "attempts/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_failed_repo_retry_succeeded_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "succeeded/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_failed_repo_retry_failed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "failed/s", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_failed_repo_retry_skipped_host_parked_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "skipped (host parked)/s", "refId": "D", "range": true, "instant": false } ] }, { "id": 41, "type": "row", "title": "Serving: /subscribe & archive API", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 60 }, "panels": [] }, { "id": 32, "type": "timeseries", "title": "Subscribers by compression", "description": "Connected /subscribe clients by negotiated wire compression.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 61 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (compression) (jetstream_subscribe_subscribers{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "{{compression}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 33, "type": "timeseries", "title": "Events sent to subscribers", "description": "JSON frames written to clients, plus events removed by per-subscriber filters before encoding.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 61 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (compression) (rate(jetstream_subscribe_events_sent_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{compression}}/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_events_filtered_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "filtered/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 34, "type": "timeseries", "title": "Egress bytes", "description": "Bytes written to /subscribe clients (post-compression for zstd) and archive getBlock downloads. sent/encoded ratio approximates the compression ratio.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 61 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (compression) (rate(jetstream_subscribe_bytes_sent_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "subscribe {{compression}}/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_bytes_encoded_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "pre-compression encoded/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_getblock_served_bytes_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "getBlock served/s", "refId": "C", "range": true, "instant": false } ] }, { "id": 35, "type": "timeseries", "title": "Subscriber drops & errors", "description": "Per-subscriber trouble. Encode errors and adversarial drops should be flat at zero.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 69 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "encode errors/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "adversarial drops/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(jetstream_subscribe_events_skipped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "skipped {{reason}}/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_events_oversize_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "oversize/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_encode_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "encode errors/s", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_adversarial_drops_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "adversarial drops/s", "refId": "D", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(jetstream_subscribe_options_update_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "options error {{reason}}/s", "refId": "E", "range": true, "instant": false } ] }, { "id": 36, "type": "timeseries", "title": "Cursor requests by mode", "description": "New /subscribe connections by cursor resolution outcome.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 69 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (mode) (rate(jetstream_subscribe_cursor_requests_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{mode}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 37, "type": "timeseries", "title": "Hot vs cold reads", "description": "ReadFrom calls served from the in-memory readable log vs falling through to the cold disk reader.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 69 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "hot (readable log)/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "cold (disk)/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eb6834" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_hot_reads_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "hot (readable log)/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_cold_reads_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "cold (disk)/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 38, "type": "timeseries", "title": "HTTP requests", "description": "Public-listener HTTP request rate by handler and status.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 77 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (handler, code) (rate(jetstream_http_request_duration_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{handler}} {{code}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 39, "type": "timeseries", "title": "API latency (p99)", "description": "99th percentile latency of the public API surfaces.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 77 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le,handler) (rate(jetstream_http_request_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "http {{handler}}", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_getblock_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "getBlock", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_subscribe_cursor_resolve_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "cursor resolve", "refId": "C", "range": true, "instant": false } ] }, { "id": 40, "type": "timeseries", "title": "getBlock requests", "description": "Archive getBlock requests by outcome. Series appear after the first request.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 77 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "ok/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "error/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (result) (rate(jetstream_getblock_requests_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{result}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 51, "type": "row", "title": "Storage: compaction, tombstones & manifest", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 85 }, "panels": [] }, { "id": 42, "type": "timeseries", "title": "Compaction passes", "description": "Delete/update compaction passes by result, plus passes triggered early by the tombstone cap.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 86 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "ok/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "error/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "early (tombstone cap)/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eda100" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (result) (rate(jetstream_compaction_passes_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{result}}/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_compaction_passes_early_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "early (tombstone cap)/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 43, "type": "timeseries", "title": "Compaction pass duration", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 86 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_compaction_pass_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_compaction_pass_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "B", "range": true, "instant": false } ] }, { "id": 44, "type": "timeseries", "title": "Rows dropped & tombstones collected", "description": "Compaction work: rows physically dropped from rewritten segments and tombstones collected by kind.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 86 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(jetstream_compaction_rows_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "dropped {{reason}}/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind) (rate(jetstream_compaction_tombstones_collected_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "collected {{kind}}/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 45, "type": "timeseries", "title": "Tombstone set", "description": "Entries in the live in-memory tombstone set. Drops to zero after each compaction pass.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 94 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_compaction_tombstone_set_entries{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "entries", "refId": "A", "range": true, "instant": false } ] }, { "id": 46, "type": "timeseries", "title": "Tombstone set memory", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 94 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_compaction_tombstone_set_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "bytes", "refId": "A", "range": true, "instant": false } ] }, { "id": 47, "type": "timeseries", "title": "Compaction watermark lag", "description": "witnessed_at lag between the sealed segment tip and the compaction watermark (header granularity).", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 94 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_compaction_watermark_lag_seconds{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "lag", "refId": "A", "range": true, "instant": false } ] }, { "id": 48, "type": "timeseries", "title": "Compaction rewrite I/O", "description": "Segment-file bytes rewritten by compaction.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 94 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_compaction_bytes_rewritten_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "bytes/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 49, "type": "timeseries", "title": "Sealed segments", "description": "Sealed segments tracked by the manifest, and compaction rewrite activity per interval.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 102 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_manifest_segments_loaded{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "segments loaded", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(jetstream_compaction_segments_rewritten_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "rewritten (per interval)", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(jetstream_compaction_segments_skipped_clean_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "skipped clean (per interval)", "refId": "C", "range": true, "instant": false } ] }, { "id": 50, "type": "timeseries", "title": "Block index cache", "description": "Manifest BlockIndex lookups served from resident metadata. No data while there are no cold lookups.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 102 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_manifest_block_index_cache_hits_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(jetstream_manifest_block_index_cache_hits_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) + sum(rate(jetstream_manifest_block_index_cache_misses_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])), 1e-9)", "legendFormat": "hit ratio", "refId": "A", "range": true, "instant": false } ] }, { "id": 55, "type": "row", "title": "Store & integrity", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 110 }, "panels": [] }, { "id": 52, "type": "timeseries", "title": "Store ops", "description": "Pebble metadata-store operations by op and outcome. Any error series is a crash-loud-worthy signal.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 111 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (op, status) (rate(jetstream_store_op_duration_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{op}} {{status}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 53, "type": "timeseries", "title": "Store op latency (p99)", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 111 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le,op) (rate(jetstream_store_op_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "{{op}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 54, "type": "timeseries", "title": "Integrity signals (expect zero)", "description": "Data-integrity error counters from every subsystem. All of these should be flat at zero; investigate any movement.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 111 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "ingest append errors/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_ingest_append_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "ingest append errors/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_on_fail_store_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "backfill OnFail store errors/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_completion_stage_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "completion stage errors/s", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_import_rows_corrupt_offset_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "import corrupt offsets/s", "refId": "D", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (op) (rate(jetstream_store_op_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",status=\"error\"}[$__rate_interval]))", "legendFormat": "store {{op}} errors/s", "refId": "E", "range": true, "instant": false } ] }, { "id": 61, "type": "row", "title": "Cutover & merge", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 119 }, "panels": [ { "id": 56, "type": "timeseries", "title": "Phase timeline", "description": "1=bootstrap, 2=merging, 3=steady state. Transitions are also marked as dashboard annotations.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 120 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "phase" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#4a3aa7" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_orchestrator_phase{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "phase", "refId": "A", "range": true, "instant": false } ] }, { "id": 57, "type": "timeseries", "title": "Merge rev filter", "description": "live_segments/ events surviving the rev filter into the steady-state segments vs already covered by backfill.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 120 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "kept/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "dropped (rev covered)/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#898781" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_orchestrator_merge_events_kept_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "kept/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_orchestrator_merge_events_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "dropped (rev covered)/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 58, "type": "timeseries", "title": "Merge progress", "description": "Run-once merge counters, shown as raw totals.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 120 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_orchestrator_merge_segments_consumed_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "segments consumed", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_orchestrator_merge_did_lookups_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "DID lookups", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_orchestrator_merge_repo_revs_updated_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "repo revs updated", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_orchestrator_merge_dids_discovered_post_bootstrap_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "DIDs discovered post-bootstrap", "refId": "D", "range": true, "instant": false } ] }, { "id": 59, "type": "bargauge", "title": "Cutover state durations", "description": "Total wall-clock seconds spent in each cutover state across this process lifetime.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 128 }, "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "#4a3aa7" }, "mappings": [], "unit": "s", "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#4a3aa7", "value": null } ] } }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "horizontal", "displayMode": "basic", "valueMode": "color", "showUnfilled": true, "sizing": "auto", "namePlacement": "auto", "minVizWidth": 8, "minVizHeight": 16, "maxVizHeight": 300, "legend": { "showLegend": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (state) (jetstream_orchestrator_state_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "{{state}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 60, "type": "timeseries", "title": "Phase transitions", "description": "Phase transitions per interval; normally each fires exactly once per data-dir lifetime.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 16, "x": 8, "y": 128 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (from, to) (increase(jetstream_orchestrator_phase_transitions_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "{{from}} → {{to}}", "refId": "A", "range": true, "instant": false } ] } ] }, { "id": 68, "type": "row", "title": "Timestamp import", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 136 }, "panels": [ { "id": 62, "type": "stat", "title": "Import phase", "description": "Current timestamp-import phase.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 4, "x": 0, "y": 137 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "type": "value", "options": { "0": { "text": "idle", "color": "text", "index": 0 }, "1": { "text": "parse + bucket", "color": "#2a78d6", "index": 1 }, "2": { "text": "apply", "color": "#eb6834", "index": 2 } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_import_phase{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 63, "type": "timeseries", "title": "Import rows", "description": "Timestamp-import pipeline throughput across phases.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 10, "x": 4, "y": 137 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_import_rows_parsed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "parsed/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (scope) (rate(jetstream_import_rows_matched_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "matched {{scope}}/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_import_rows_mutated_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "mutated/s", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_import_dids_matched_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "DIDs matched/s", "refId": "D", "range": true, "instant": false } ] }, { "id": 64, "type": "timeseries", "title": "Import rows rejected", "description": "CSV rows rejected during parse, by reason.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 10, "x": 14, "y": 137 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(jetstream_import_rows_rejected_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{reason}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 65, "type": "timeseries", "title": "Import rewrite activity", "description": "Phase C segment rewrites per interval.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 10, "x": 0, "y": 145 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(jetstream_import_segments_examined_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "segments examined", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(jetstream_import_segments_patched_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "segments patched", "refId": "B", "range": true, "instant": false } ] }, { "id": 66, "type": "timeseries", "title": "Import rewrite I/O", "description": "On-disk segment bytes rewritten by import.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 10, "x": 10, "y": 145 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_import_bytes_rewritten_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "bytes/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 67, "type": "timeseries", "title": "Import jobs", "description": "Terminal import jobs per interval by result.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 4, "x": 20, "y": 145 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "ok" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (result) (increase(jetstream_import_jobs_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "{{result}}", "refId": "A", "range": true, "instant": false } ] } ] }, { "id": 75, "type": "row", "title": "Process runtime", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 153 }, "panels": [] }, { "id": 69, "type": "timeseries", "title": "CPU", "description": "Process CPU usage in cores vs the scheduler limit.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 154 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "cores used" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "GOMAXPROCS" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#898781" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "cores used", "refId": "A", "range": true, "instant": false } ] }, { "id": 70, "type": "timeseries", "title": "Memory", "description": "Resident set vs Go heap. GOMEMLIMIT is hidden when unset.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 154 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "RSS" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "heap in use" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#1baf7a" } } ] }, { "matcher": { "id": "byName", "options": "GC goal" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#898781" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "RSS", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(process_virtual_memory_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "virtual", "refId": "B", "range": true, "instant": false } ] }, { "id": 71, "type": "timeseries", "title": "Threads", "description": "A monotonic goroutine climb is a leak.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 154 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "goroutines" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "OS threads" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#898781" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(stream_process_threads{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "OS threads", "refId": "A", "range": true, "instant": false } ] }, { "id": 72, "type": "timeseries", "title": "File descriptors", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 162 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "open" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "max" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#898781" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(process_open_fds{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "open", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(process_max_fds{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "max", "refId": "B", "range": true, "instant": false } ] }, { "id": 73, "type": "timeseries", "title": "Page faults", "description": "Stop-the-world pause durations from the Go GC summary.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 162 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(process_minor_page_faults_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "minor/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(process_major_page_faults_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "major/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 74, "type": "timeseries", "title": "Context switches", "description": "Process-level network throughput (procfs; Linux only).", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 162 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "receive/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "transmit/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#1baf7a" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(process_voluntary_context_switches_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "voluntary/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(process_involuntary_context_switches_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "involuntary/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 80, "type": "row", "title": "Backfill progress over time", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 170 }, "panels": [], "description": "The final network denominator is unknown until listRepos discovery finishes." }, { "id": 81, "type": "timeseries", "title": "Backfill repositories over time", "description": "Absolute repository counts across the current backfill run. listRepos is fetched in pages of 1,000; Stream follows upstream Jetstream V2 by accumulating about 100,000 entries before shuffling and dispatching a batch. Durably committed advances only after that batch checkpoint. These lines are not a percentage of the full network.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 171 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "events received/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "decode errors/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "unknown events/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eb6834" } } ] } ] }, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_backfill_discovered_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "discovered", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_backfill_completion_queued_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "processed", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(stream_backfill_repos_durable{status=\"complete\",job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "durably committed", "refId": "C", "range": true, "instant": false } ] }, { "id": 76, "type": "row", "title": "Stream pipeline diagnostics", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 175 }, "panels": [], "description": "Live during bootstrap and steady state; gaps and terminal failures are never inferred from archive throughput." }, { "id": 77, "type": "timeseries", "title": "Pipeline tickets", "description": "Cumulative scheduler boundaries. Emitted is the historical name for every settled ticket, including explicit drops and verifier rejections.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 176 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(stream_pipeline_ticket{job=~\"$job\",instance=~\"$instance\"}) by (stage)", "legendFormat": "{{stage}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 78, "type": "timeseries", "title": "Pipeline stage gaps", "description": "Current unsettled work split between the unclaimed verifier queue and claimed work awaiting settlement. Their sum is submitted-to-emitted.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 176 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(stream_pipeline_stage_gap{job=~\"$job\",instance=~\"$instance\"}) by (boundary)", "legendFormat": "{{boundary}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 79, "type": "timeseries", "title": "Pipeline terminal failures", "description": "Any non-zero value is deployment-significant. The gauge marks the current pipeline fatal; the counters retain its first failing stage.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 176 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(stream_pipeline_fatal{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "fatal", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(stream_pipeline_failures_total{job=~\"$job\",instance=~\"$instance\"}) by (stage)", "legendFormat": "{{stage}} failures", "refId": "B", "range": true, "instant": false } ] } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "stream", "jetstream", "atproto" ], "templating": { "list": [ { "name": "datasource", "label": "Data source", "type": "datasource", "query": "prometheus", "current": {}, "hide": 0, "includeAll": false, "multi": false, "options": [], "refresh": 1, "regex": "", "skipUrlSync": false }, { "name": "job", "label": "Job", "type": "query", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "query": { "query": "label_values(jetstream_build_info, job)", "refId": "job" }, "definition": "label_values(jetstream_build_info, job)", "current": {}, "hide": 0, "includeAll": true, "allValue": ".*", "multi": true, "options": [], "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1 }, { "name": "instance", "label": "Instance", "type": "query", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "query": { "query": "label_values(jetstream_build_info{job=~\"$job\"}, instance)", "refId": "instance" }, "definition": "label_values(jetstream_build_info{job=~\"$job\"}, instance)", "current": {}, "hide": 0, "includeAll": true, "allValue": ".*", "multi": true, "options": [], "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1 } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Stream / Jetstream V2", "uid": "stream-jetstream-v2", "version": 1, "weekStart": ""}