Fork of daniellemaywood.uk/gleam — Wasm codegen work
Something went wrong. Try again.
197 kB · 5367 lines
Rust
at wasm
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368// SPDX-License-Identifier: Apache-2.0// SPDX-FileCopyrightText: 2020 The Gleam contributors
// Gleam Parser//// Terminology:// Expression Unit:// Essentially a thing that goes between operators.// Int, Bool, function call, "{" expression-sequence "}", case x {}, ..etc//// Expression:// One or more Expression Units separated by an operator//// Binding:// (let|let assert|use) name (:TypeAnnotation)? = Expression//// Expression Sequence:// * One or more Expressions// * A Binding followed by at least one more Expression Sequences//// Naming Conventions:// parse_x// Parse a specific part of the grammar, not erroring if it cannot.// Generally returns `Result<Option<A>, ParseError>`, note the inner Option//// expect_x// Parse a generic or specific part of the grammar, erroring if it cannot.// Generally returns `Result<A, ParseError>`, note no inner Option//// maybe_x// Parse a generic part of the grammar. Returning `None` if it cannot.// Returns `Some(x)` and advances the token stream if it can.//// Operator Precedence Parsing:// Needs to take place in expressions and in clause guards.// It is accomplished using the Simple Precedence Parser algorithm.// See: https://en.wikipedia.org/wiki/Simple_precedence_parser//// It relies or the operator grammar being in the general form:// e ::= expr op expr | expr// Which just means that exprs and operators always alternate, starting with an expr//// The gist of the algorithm is:// Create 2 stacks, one to hold expressions, and one to hold un-reduced operators.// While consuming the input stream, if an expression is encountered add it to the top// of the expression stack. If an operator is encountered, compare its precedence to the// top of the operator stack and perform the appropriate action, which is either using an// operator to reduce 2 expressions on the top of the expression stack or put it on the top// of the operator stack. When the end of the input is reached, attempt to reduce all of the// expressions down to a single expression(or no expression) using the remaining operators// on the operator stack. If there are any operators left, or more than 1 expression left// this is a syntax error. But the implementation here shouldn't need to handle that case// as the outer parser ensures the correct structure.//pub mod error;pub mod extra;pub mod lexer;mod token;
use crate::Warning;use crate::analyse::Inferred;use crate::ast::{ Arg, ArgNames, Assert, AssignName, Assignment, AssignmentKind, BinOp, BitArrayOption, BitArraySegment, BitArraySize, CAPTURE_VARIABLE, CallArg, Clause, ClauseGuard, Constant, CustomType, Definition, Function, FunctionLiteralKind, HasLocation, Import, IntOperator, Module, ModuleConstant, Pattern, Publicity, RecordBeingUpdated, RecordConstructor, RecordConstructorArg, RecordUpdateArg, SrcSpan, Statement, TailPattern, TargetedDefinition, TodoKind, TypeAlias, TypeAst, TypeAstConstructor, TypeAstConstructorName, TypeAstFn, TypeAstHole, TypeAstTuple, TypeAstVar, UnqualifiedImport, UntypedArg, UntypedClause, UntypedClauseGuard, UntypedConstant, UntypedDefinition, UntypedExpr, UntypedModule, UntypedPattern, UntypedRecordUpdateArg, UntypedStatement, UntypedUseAssignment, Use, UseAssignment,};use crate::build::Target;use crate::error::wrap;use crate::exhaustiveness::CompiledCase;use crate::parse::error::IncorrectNamePosition;use crate::parse::extra::ModuleExtra;use crate::type_::Deprecation;use crate::type_::error::{VariableDeclaration, VariableOrigin, VariableSyntax};use crate::type_::expression::{Implementations, Purity};use crate::type_::printer::Names;use crate::warning::{DeprecatedSyntaxWarning, WarningEmitter};use camino::Utf8PathBuf;use ecow::EcoString;use error::{LexicalError, ParseError, ParseErrorType};use lexer::{LexResult, Spanned};use num_bigint::BigInt;use serde::{Deserialize, Serialize};use std::cmp::Ordering;use std::collections::{HashSet, VecDeque};use std::hash::{Hash, Hasher};use std::str::FromStr;pub use token::Token;use vec1::{Vec1, vec1};
#[cfg(test)]mod tests;
#[derive(Debug)]pub struct Parsed { pub module: UntypedModule, pub extra: ModuleExtra,}
/// We use this to keep track of the `@internal` annotation for top level/// definitions. Instead of using just a boolean we want to keep track of the/// source position of the annotation in case it is present. This way we can/// report a better error message highlighting the annotation in case it is/// used on a private definition (it doesn't make sense to mark something/// private as internal):////// ```txt/// @internal/// ^^^^^^^^^ we first get to the annotation/// fn wibble() {}/// ^^ and only later discover it's applied on a private definition/// so we have to keep track of the attribute's position to highlight it/// in the resulting error message./// ```#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]enum InternalAttribute { #[default] Missing, Present(SrcSpan),}
#[derive(Debug, Default)]struct Attributes { target: Option<Target>, deprecated: Deprecation, external_erlang: Option<(EcoString, EcoString, SrcSpan)>, external_javascript: Option<(EcoString, EcoString, SrcSpan)>, external_wasm: Option<(EcoString, EcoString, SrcSpan)>, internal: InternalAttribute,}
impl Attributes { fn has_function_only(&self) -> bool { self.external_erlang.is_some() || self.external_javascript.is_some() || self.external_wasm.is_some() }
fn has_external_for(&self, target: Target) -> bool { match target { Target::Erlang => self.external_erlang.is_some(), Target::JavaScript => self.external_javascript.is_some(), Target::Wasm => self.external_wasm.is_some(), } }
fn set_external_for(&mut self, target: Target, ext: Option<(EcoString, EcoString, SrcSpan)>) { match target { Target::Erlang => self.external_erlang = ext, Target::JavaScript => self.external_javascript = ext, Target::Wasm => self.external_wasm = ext, } }}
//// Public Interface//
pub type SpannedString = (SrcSpan, EcoString);
pub fn parse_module( path: Utf8PathBuf, src: &str, warnings: &WarningEmitter,) -> Result<Parsed, ParseError> { let lex = lexer::make_tokenizer(src); let mut parser = Parser::new(lex); let mut parsed = parser.parse_module()?; parsed.extra = parser.extra;
let src = EcoString::from(src); for warning in parser.warnings { warnings.emit(Warning::DeprecatedSyntax { path: path.clone(), src: src.clone(), warning, }); }
for detached in parser.detached_doc_comments { warnings.emit(Warning::DetachedDocComment { path: path.clone(), src: src.clone(), location: detached, }); }
Ok(parsed)}
//// Test Interface//#[cfg(test)]pub fn parse_statement_sequence(src: &str) -> Result<Vec1<UntypedStatement>, ParseError> { let lex = lexer::make_tokenizer(src); let mut parser = Parser::new(lex); let expr = parser.parse_statement_seq(); let expr = parser.ensure_no_errors_or_remaining_input(expr)?; match expr { Some((e, _)) => Ok(e), _ => parse_error(ParseErrorType::ExpectedExpr, SrcSpan { start: 0, end: 0 }), }}
//// Test Interface//#[cfg(test)]pub fn parse_const_value(src: &str) -> Result<UntypedConstant, ParseError> { let lex = lexer::make_tokenizer(src); let mut parser = Parser::new(lex); let expr = parser.parse_const_value(); let expr = parser.ensure_no_errors_or_remaining_input(expr)?; match expr { Some(e) => Ok(e), _ => parse_error(ParseErrorType::ExpectedExpr, SrcSpan { start: 0, end: 0 }), }}
//// Parser//#[derive(Debug)]pub struct Parser<T: Iterator<Item = LexResult>> { tokens: T, lex_errors: Vec<LexicalError>, warnings: Vec<DeprecatedSyntaxWarning>, tok0: Option<Spanned>, tok1: Option<Spanned>, extra: ModuleExtra, doc_comments: VecDeque<(u32, EcoString)>, detached_doc_comments: Vec<SrcSpan>,}impl<T> Parser<T>where T: Iterator<Item = LexResult>,{ pub fn new(input: T) -> Self { let mut parser = Parser { tokens: input, lex_errors: vec![], warnings: vec![], tok0: None, tok1: None, extra: ModuleExtra::new(), doc_comments: VecDeque::new(), detached_doc_comments: Vec::new(), }; parser.advance(); parser.advance(); parser }
fn parse_module(&mut self) -> Result<Parsed, ParseError> { let definitions = self.series_of(&Parser::parse_definition, None); let definitions = self.ensure_no_errors_or_remaining_input(definitions)?; let module = Module { name: "".into(), documentation: vec![], type_info: (), definitions, names: Names::new(), unused_definition_positions: HashSet::new(), }; Ok(Parsed { module, extra: ModuleExtra::new(), }) }
// The way the parser is currently implemented, it cannot exit immediately while advancing // the token stream upon seeing a LexError. That is to avoid having to put `?` all over the // place and instead we collect LexErrors in `self.lex_errors` and attempt to continue parsing. // Once parsing has returned we want to surface an error in the order: // 1) LexError, 2) ParseError, 3) More Tokens Left fn ensure_no_errors_or_remaining_input<A>( &mut self, parse_result: Result<A, ParseError>, ) -> Result<A, ParseError> { let parse_result = self.ensure_no_errors(parse_result)?; if let Some((start, token, end)) = self.next_tok() { // there are still more tokens let expected = vec!["An import, const, type, or function.".into()]; return parse_error( ParseErrorType::UnexpectedToken { token, expected, hint: None, }, SrcSpan { start, end }, ); } // no errors Ok(parse_result) }
// The way the parser is currently implemented, it cannot exit immediately // while advancing the token stream upon seeing a LexError. That is to avoid // having to put `?` all over the place and instead we collect LexErrors in // `self.lex_errors` and attempt to continue parsing. // Once parsing has returned we want to surface an error in the order: // 1) LexError, 2) ParseError fn ensure_no_errors<A>( &mut self, parse_result: Result<A, ParseError>, ) -> Result<A, ParseError> { if let Some(error) = self.lex_errors.first() { // Lex errors first let location = error.location; let error = *error; parse_error(ParseErrorType::LexError { error }, location) } else { // Return any existing parse error parse_result } }
fn parse_definition(&mut self) -> Result<Option<TargetedDefinition>, ParseError> { let mut attributes = Attributes::default(); let location = self.parse_attributes(&mut attributes)?;
let def = match (self.tok0.take(), self.tok1.as_ref()) { // Imports (Some((start, Token::Import, _)), _) => { self.advance(); self.parse_import(start) } // Module Constants (Some((start, Token::Const, _)), _) => { self.advance(); self.parse_module_const(start, false, &attributes) } (Some((start, Token::Pub, _)), Some((_, Token::Const, _))) => { self.advance(); self.advance(); self.parse_module_const(start, true, &attributes) }
// Function (Some((start, Token::Fn, _)), _) => { self.advance(); self.parse_function(start, false, false, &mut attributes) } (Some((start, Token::Pub, _)), Some((_, Token::Fn, _))) => { self.advance(); self.advance(); self.parse_function(start, true, false, &mut attributes) }
// Custom Types, and Type Aliases (Some((start, Token::Type, _)), _) => { self.advance(); self.parse_custom_type(start, false, false, &mut attributes) } (Some((start, Token::Pub, _)), Some((_, Token::Opaque, _))) => { self.advance(); self.advance(); let _ = self.expect_one(&Token::Type)?; self.parse_custom_type(start, true, true, &mut attributes) } (Some((start, Token::Pub, _)), Some((_, Token::Type, _))) => { self.advance(); self.advance(); self.parse_custom_type(start, true, false, &mut attributes) } (Some((start, Token::Opaque, _)), Some((_, Token::Type, _))) => { // A private opaque type makes no sense! We still want to parse it // and return an error later during the analysis phase. self.advance(); self.advance(); self.parse_custom_type(start, false, true, &mut attributes) }
(t0, _) => { self.tok0 = t0; Ok(None) } }?;
match (def, location) { (Some(definition), _) if definition.is_function() || definition.is_custom_type() => { Ok(Some(TargetedDefinition { definition, target: attributes.target, })) }
(Some(definition), None) => Ok(Some(TargetedDefinition { definition, target: attributes.target, })),
(_, Some(location)) if attributes.has_function_only() => { parse_error(ParseErrorType::ExpectedFunctionDefinition, location) }
(Some(definition), _) => Ok(Some(TargetedDefinition { definition, target: attributes.target, })),
(_, Some(location)) => parse_error(ParseErrorType::ExpectedDefinition, location),
(None, None) => Ok(None), } }
// // Parse Expressions //
// examples: // unit // unit op unit // unit op unit pipe unit(call) // unit op unit pipe unit(call) pipe unit(call) fn parse_expression(&mut self) -> Result<Option<UntypedExpr>, ParseError> { self.parse_expression_inner(false) }
fn parse_expression_inner( &mut self, is_let_binding: bool, ) -> Result<Option<UntypedExpr>, ParseError> { // uses the simple operator parser algorithm let mut opstack = vec![]; let mut estack = vec![]; let mut last_op_start = 0; let mut last_op_end = 0;
// This is used to keep track if we've just ran into a `|>` operator in // order to properly parse an echo based on its position: if it is in a // pipeline then it isn't expected to be followed by an expression. // Otherwise, it's expected to be followed by an expression. let mut expression_unit_context = ExpressionUnitContext::Other;
loop { match self.parse_expression_unit(expression_unit_context)? { Some(unit) => { self.post_process_expression_unit(&unit, is_let_binding)?; estack.push(unit); } _ if estack.is_empty() => return Ok(None), _ => { return parse_error( ParseErrorType::OpNakedRight, SrcSpan { start: last_op_start, end: last_op_end, }, ); } }
let Some((op_s, t, op_e)) = self.tok0.take() else { break; };
let Some(p) = precedence(&t) else { self.tok0 = Some((op_s, t, op_e)); break; };
expression_unit_context = if t == Token::Pipe { ExpressionUnitContext::FollowingPipe } else { ExpressionUnitContext::Other };
// Is Op self.advance(); last_op_start = op_s; last_op_end = op_e; let _ = handle_op( Some(((op_s, t, op_e), p)), &mut opstack, &mut estack, &do_reduce_expression, ); }
Ok(handle_op( None, &mut opstack, &mut estack, &do_reduce_expression, )) }
fn post_process_expression_unit( &mut self, unit: &UntypedExpr, is_let_binding: bool, ) -> Result<(), ParseError> { // Produce better error message for `[x] = [1]` outside // of `let` statement. if !is_let_binding && let UntypedExpr::List { .. } = unit && let Some((start, Token::Equal, end)) = self.tok0 { return parse_error(ParseErrorType::NoLetBinding, SrcSpan { start, end }); } Ok(()) }
// examples: // 1 // "one" // True // fn() { "hi" } // unit().unit().unit() // A(a.., label: tuple(1)) // { expression_sequence } fn parse_expression_unit( &mut self, context: ExpressionUnitContext, ) -> Result<Option<UntypedExpr>, ParseError> { let mut expr = match self.tok0.take() { Some((start, Token::String { value }, end)) => { self.advance(); UntypedExpr::String { location: SrcSpan { start, end }, value, } } Some((start, Token::Int { value, int_value }, end)) => { self.advance(); UntypedExpr::Int { location: SrcSpan { start, end }, value, int_value, } }
Some((start, Token::Float { value, float_value }, end)) => { self.advance(); UntypedExpr::Float { location: SrcSpan { start, end }, value, float_value, } }
// var lower_name and UpName Some((start, Token::Name { name } | Token::UpName { name }, end)) => { self.advance(); UntypedExpr::Var { location: SrcSpan { start, end }, name, } }
Some((start, Token::Todo, end)) => { self.advance(); let message = self.maybe_parse_as_message()?; let end = message.as_ref().map_or(end, |m| m.location().end); UntypedExpr::Todo { location: SrcSpan { start, end }, kind: TodoKind::Keyword, message, } }
Some((start, Token::Panic, end)) => { self.advance(); let message = self.maybe_parse_as_message()?; let end = message.as_ref().map_or(end, |m| m.location().end); UntypedExpr::Panic { location: SrcSpan { start, end }, message, } }
Some((start, Token::Echo, echo_end)) => { self.advance(); if context == ExpressionUnitContext::FollowingPipe { // If an echo is used as a step in a pipeline (`|> echo`) // then it cannot be followed by an expression. let message = self.maybe_parse_as_message()?; let end = message.as_ref().map_or(echo_end, |m| m.location().end); UntypedExpr::Echo { location: SrcSpan { start, end }, keyword_end: echo_end, expression: None, message, } } else { // Otherwise it must be followed by an expression. // However, you might have noticed we're not erroring if the // expression is not there. Instead we move this error to // the analysis phase so that a wrong usage of echo won't // stop analysis from happening everywhere and be fault // tolerant like everything else. let expression = self.parse_expression()?; let end = expression.as_ref().map_or(echo_end, |e| e.location().end);
let message = self.maybe_parse_as_message()?; let end = message.as_ref().map_or(end, |m| m.location().end);
UntypedExpr::Echo { location: SrcSpan { start, end }, keyword_end: echo_end, expression: expression.map(Box::new), message, } } }
Some((start, Token::Hash, _)) => { self.advance(); let _ = self .expect_one(&Token::LeftParen) .map_err(|error| self.add_comment_style_hint(error))?; let elements = self.series_of(&Parser::parse_expression, Some(&Token::Comma))?; let (_, end) = self.expect_one_following_series(&Token::RightParen, "an expression")?; UntypedExpr::Tuple { location: SrcSpan { start, end }, elements, } }
// list Some((start, Token::LeftSquare, _)) => { self.advance(); let (elements, elements_end_with_comma) = self.series_of_has_trailing_separator( &Parser::parse_expression, Some(&Token::Comma), )?;
// Parse an optional tail let mut tail = None; let mut elements_after_tail = None; let mut dot_dot_location = None;
if let Some((start, end)) = self.maybe_one(&Token::DotDot) { dot_dot_location = Some((start, end)); tail = self.parse_expression()?.map(Box::new); if self.maybe_one(&Token::Comma).is_some() { // See if there's a list of items after the tail, // like `[..wibble, wobble, wabble]` let elements = self.series_of(&Parser::parse_expression, Some(&Token::Comma)); match elements { Err(_) => {} Ok(elements) => { elements_after_tail = Some(elements); } } }
if tail.is_some() { if !elements_end_with_comma { self.warnings .push(DeprecatedSyntaxWarning::DeprecatedListPrepend { location: SrcSpan { start, end }, }); }
// Give a better error when there is two consecutive spreads // like `[..wibble, ..wabble, woo]`. However, if there's other // elements after the tail of the list if let Some((second_start, second_end)) = self.maybe_one(&Token::DotDot) { let _second_tail = self.parse_expression();
if elements_after_tail.is_none() || elements_after_tail .as_ref() .is_some_and(|vec| vec.is_empty()) { return parse_error( ParseErrorType::ListSpreadWithAnotherSpread { first_spread_location: SrcSpan { start, end }, }, SrcSpan { start: second_start, end: second_end, }, ); } } } }
let (_, end) = self.expect_one(&Token::RightSquare)?;
// Return errors for malformed lists match dot_dot_location { Some((start, end)) if tail.is_none() => { return parse_error( ParseErrorType::ListSpreadWithoutTail, SrcSpan { start, end }, ); } _ => {} } if tail.is_some() && elements.is_empty() && elements_after_tail.as_ref().is_none_or(|e| e.is_empty()) { return parse_error( ParseErrorType::ListSpreadWithoutElements, SrcSpan { start, end }, ); }
match elements_after_tail { Some(elements) if !elements.is_empty() => { let (start, end) = match (dot_dot_location, tail) { (Some((start, _)), Some(tail)) => (start, tail.location().end), (_, _) => (start, end), }; return parse_error( ParseErrorType::ListSpreadFollowedByElements, SrcSpan { start, end }, ); } _ => {} }
UntypedExpr::List { location: SrcSpan { start, end }, elements, tail, } }
// BitArray Some((start, Token::LtLt, _)) => { self.advance(); let segments = self.series_of( &|this| { Parser::parse_bit_array_segment( this, &(|this| this.parse_expression_unit(ExpressionUnitContext::Other)), &Parser::expect_expression, &bit_array_expr_int, ) }, Some(&Token::Comma), )?; let (_, end) = self.expect_one_following_series(&Token::GtGt, "a bit array segment")?; UntypedExpr::BitArray { location: SrcSpan { start, end }, segments, } } Some((start, Token::Fn, _)) => { self.advance(); let mut attributes = Attributes::default(); match self.parse_function(start, false, true, &mut attributes)? { Some(Definition::Function(Function { location, arguments, body, return_annotation, end_position, .. })) => { let Ok(body) = Vec1::try_from_vec(body) else { return parse_error(ParseErrorType::ExpectedFunctionBody, location); };
UntypedExpr::Fn { location: SrcSpan::new(location.start, end_position), end_of_head_byte_index: location.end, kind: FunctionLiteralKind::Anonymous { head: location }, arguments, body, return_annotation, } }
_ => { // this isn't just none, it could also be Some(UntypedExpr::..) return self.next_tok_unexpected(vec!["An opening parenthesis.".into()]); } } }
// expression block "{" "}" Some((start, Token::LeftBrace, _)) => { self.advance(); self.parse_block(start)? }
// case Some((start, Token::Case, case_e)) => { self.advance(); let subjects = self.series_of(&Parser::parse_expression, Some(&Token::Comma))?; if self.maybe_one(&Token::LeftBrace).is_some() { let clauses = self.series_of(&Parser::parse_case_clause, None)?; let (_, end) = self.expect_one_following_series(&Token::RightBrace, "a case clause")?; if subjects.is_empty() { return parse_error( ParseErrorType::ExpectedExpr, SrcSpan { start, end: case_e }, ); } else { UntypedExpr::Case { location: SrcSpan { start, end }, subjects, clauses: Some(clauses), } } } else { UntypedExpr::Case { location: SrcSpan::new( start, subjects .last() .map(|subject| subject.location().end) .unwrap_or(case_e), ), subjects, clauses: None, } } }
// Helpful error if trying to write an if expression instead of a // case. Some((start, Token::If, end)) => { return parse_error(ParseErrorType::IfExpression, SrcSpan { start, end }); }
// Helpful error on possibly trying to group with "(". Some((start, Token::LeftParen, _)) => { return parse_error(ParseErrorType::ExprLparStart, SrcSpan { start, end: start }); }
// Boolean negation Some((start, Token::Bang, _end)) => { self.advance(); match self.parse_expression_unit(ExpressionUnitContext::Other)? { Some(value) => UntypedExpr::NegateBool { location: SrcSpan { start, end: value.location().end, }, value: Box::from(value), }, None => { return parse_error( ParseErrorType::ExpectedExpr, SrcSpan { start, end: start }, ); } } }
// Int negation Some((start, Token::Minus, _end)) => { self.advance(); match self.parse_expression_unit(ExpressionUnitContext::Other)? { Some(value) => UntypedExpr::NegateInt { location: SrcSpan { start, end: value.location().end, }, value: Box::from(value), }, None => { return parse_error( ParseErrorType::ExpectedExpr, SrcSpan { start, end: start }, ); } } }
t0 => { self.tok0 = t0; return Ok(None); } };
// field access and call can stack up loop { match self.maybe_one(&Token::Dot) { Some((dot_start, _)) => { let start = expr.location().start; // field access match self.tok0.take() { // tuple access Some(( _, Token::Int { value, int_value: _, }, end, )) => { self.advance(); let v = value.replace("_", ""); match u64::from_str(&v) { Ok(index) => { expr = UntypedExpr::TupleIndex { location: SrcSpan { start, end }, index, tuple: Box::new(expr), } } _ => { return parse_error( ParseErrorType::InvalidTupleAccess, SrcSpan { start, end }, ); } } }
Some((label_start, Token::Name { name: label }, end)) => { self.advance(); expr = UntypedExpr::FieldAccess { location: SrcSpan { start, end }, label_location: SrcSpan { start: label_start, end, }, label, container: Box::new(expr), } }
Some((label_start, Token::UpName { name: label }, end)) => { self.advance(); expr = UntypedExpr::FieldAccess { location: SrcSpan { start, end }, label_location: SrcSpan { start: label_start, end, }, label, container: Box::new(expr), } }
t0 => { // parse a field access with no label self.tok0 = t0; let end = dot_start + 1; expr = UntypedExpr::FieldAccess { location: SrcSpan { start, end }, label_location: SrcSpan { start: dot_start, end, }, label: "".into(), container: Box::new(expr), }; return Ok(Some(expr)); } } } _ => { if let Some((left_paren, _)) = self.maybe_one(&Token::LeftParen) { let start = expr.location().start; match self.maybe_one(&Token::DotDot) { Some((dot_s, _)) => { // Record update let base = self.expect_expression()?; let base_e = base.location().end; let record = RecordBeingUpdated { base: Box::new(base), location: SrcSpan { start: dot_s, end: base_e, }, }; let mut arguments = vec![]; if self.maybe_one(&Token::Comma).is_some() { arguments = self.series_of( &Parser::parse_record_update_arg, Some(&Token::Comma), )?; } let (_, end) = self.expect_one(&Token::RightParen)?;
expr = UntypedExpr::RecordUpdate { location: SrcSpan { start, end }, spread_start: dot_s, constructor: Box::new(expr), record, arguments, }; } _ => { // Call let arguments = self.parse_fn_arguments()?; let (_, end) = self.expect_one(&Token::RightParen)?; expr = make_call(expr, arguments, start, end, left_paren)?; } } } else { // done break; } } } }
Ok(Some(expr)) }
fn add_comment_style_hint(&self, mut err: ParseError) -> ParseError { if let ParseErrorType::UnexpectedToken { ref mut hint, .. } = err.error { let text = "Maybe you meant to create a comment?\nComments in Gleam start with `//`, not `#`"; *hint = Some(text.into()); } err }
// A `use` expression // use <- function // use <- function() // use <- function(a, b) // use <- module.function(a, b) // use a, b, c <- function(a, b) // use a, b, c, <- function(a, b) fn parse_use(&mut self, start: u32, end: u32) -> Result<UntypedStatement, ParseError> { let assignments = match self.tok0 { Some((_, Token::LArrow, _)) => { vec![] } _ => self.series_of(&Parser::parse_use_assignment, Some(&Token::Comma))?, };
_ = self.expect_one_following_series(&Token::LArrow, "a use variable assignment")?; let call = self.expect_expression()?;
let assignments_location = match (assignments.first(), assignments.last()) { (Some(first), Some(last)) => SrcSpan { start: first.location.start, end: last.location.end, }, (_, _) => SrcSpan { start, end }, };
Ok(Statement::Use(Use { location: SrcSpan::new(start, call.location().end), assignments_location, right_hand_side_location: call.location(), assignments, call: Box::new(call), })) }
fn parse_use_assignment(&mut self) -> Result<Option<UntypedUseAssignment>, ParseError> { let start = self.tok0.as_ref().map(|t| t.0).unwrap_or(0);
let pattern = self .parse_pattern(PatternPosition::UsePattern)? .ok_or_else(|| ParseError { error: ParseErrorType::ExpectedPattern, location: SrcSpan { start, end: start }, })?;
let annotation = self.parse_type_annotation(&Token::Colon)?; let end = match annotation { Some(ref a) => a.location().end, None => pattern.location().end, };
Ok(Some(UseAssignment { location: SrcSpan { start, end }, pattern, annotation, })) }
fn maybe_parse_as_message(&mut self) -> Result<Option<Box<UntypedExpr>>, ParseError> { let message = if self.maybe_one(&Token::As).is_some() { let expression = self.expect_expression_unit(ExpressionUnitContext::Other)?; Some(Box::new(expression)) } else { None };
Ok(message) }
fn maybe_parse_constant_as_message( &mut self, ) -> Result<Option<Box<UntypedConstant>>, ParseError> { let message = if let Some((as_start, as_end)) = self.maybe_one(&Token::As) { match self.parse_const_value_unit()? { Some(constant) => Some(Box::new(constant)), None => { return Err(ParseError { error: ParseErrorType::MissingConstantAsMessage, location: SrcSpan::new(as_start, as_end), }); } } } else { None };
Ok(message) }
// An assignment, with `Let` already consumed fn parse_assignment(&mut self, start: u32) -> Result<UntypedStatement, ParseError> { let mut kind = match self.tok0 { Some((assert_keyword_start, Token::Assert, assert_end)) => { _ = self.next_tok(); AssignmentKind::Assert { location: SrcSpan::new(start, assert_end), assert_keyword_start, message: None, } } _ => AssignmentKind::Let, }; let pattern = match self.parse_pattern(PatternPosition::LetAssignment)? { Some(p) => p, _ => { // DUPE: 62884 return self.next_tok_unexpected(vec!["A pattern".into()])?; } }; let annotation = self.parse_type_annotation(&Token::Colon)?; let (eq_s, eq_e) = self.maybe_one(&Token::Equal).ok_or(ParseError { error: ParseErrorType::ExpectedEqual, location: SrcSpan { start: pattern.location().start, end: pattern.location().end, }, })?; let value = self.parse_expression_inner(true)?.ok_or(match self.tok0 { Some((start, Token::DiscardName { .. }, end)) => ParseError { error: ParseErrorType::IncorrectName { kind: IncorrectNamePosition::Variable, }, location: SrcSpan { start, end }, },
_ => ParseError { error: ParseErrorType::ExpectedValue, location: SrcSpan { start: eq_s, end: eq_e, }, }, })?;
let mut end = value.location().end;
match &mut kind { AssignmentKind::Let | AssignmentKind::Generated => {} AssignmentKind::Assert { message, .. } => { if self.maybe_one(&Token::As).is_some() { let message_expression = self.expect_expression_unit(ExpressionUnitContext::Other)?; end = message_expression.location().end; *message = Some(message_expression); } } }
Ok(Statement::Assignment(Box::new(Assignment { location: SrcSpan { start, end }, value, compiled_case: CompiledCase::failure(), pattern, annotation, kind, }))) }
// An assert statement, with `Assert` already consumed fn parse_assert(&mut self, start: u32) -> Result<UntypedStatement, ParseError> { let value = self.expect_expression()?; let mut end = value.location().end;
let message = if self.maybe_one(&Token::As).is_some() { let message_expression = self.expect_expression_unit(ExpressionUnitContext::Other)?; end = message_expression.location().end; Some(message_expression) } else { None };
Ok(Statement::Assert(Assert { location: SrcSpan { start, end }, value, message, })) }
// examples: // expr // expr expr.. // expr assignment.. // assignment // assignment expr.. // assignment assignment.. fn parse_statement_seq(&mut self) -> Result<Option<(Vec1<UntypedStatement>, u32)>, ParseError> { let mut statements = vec![]; let mut start = None; let mut end = 0;
// Try and parse as many expressions as possible while let Some(statement) = self.parse_statement()? { if start.is_none() { start = Some(statement.location().start); } end = statement.location().end; statements.push(statement); }
match Vec1::try_from_vec(statements) { Ok(statements) => Ok(Some((statements, end))), Err(_) => Ok(None), } }
fn parse_statement(&mut self) -> Result<Option<UntypedStatement>, ParseError> { match self.tok0.take() { Some((start, Token::Use, end)) => { self.advance(); Ok(Some(self.parse_use(start, end)?)) }
Some((start, Token::Let, _)) => { self.advance(); Ok(Some(self.parse_assignment(start)?)) }
Some((start, Token::Assert, _)) => { self.advance(); Ok(Some(self.parse_assert(start)?)) }
// Helpful error when trying to define a constant inside a function. Some((start, Token::Const, end)) => parse_error( ParseErrorType::ConstantInsideFunction, SrcSpan { start, end }, ),
token => { self.tok0 = token; self.parse_statement_errors()?; let expression = self.parse_expression()?.map(Statement::Expression); Ok(expression) } } }
fn parse_statement_errors(&mut self) -> Result<(), ParseError> { // Better error: name definitions must start with `let` if let Some((_, Token::Name { .. }, _)) = self.tok0.as_ref() && let Some((start, Token::Equal | Token::Colon, end)) = self.tok1 { return parse_error(ParseErrorType::NoLetBinding, SrcSpan { start, end }); } Ok(()) }
fn parse_block(&mut self, start: u32) -> Result<UntypedExpr, ParseError> { let body = self.parse_statement_seq()?; let (_, end) = self.expect_one(&Token::RightBrace)?; let location = SrcSpan { start, end }; let statements = match body { Some((statements, _)) => statements, None => vec1![Statement::Expression(UntypedExpr::Todo { kind: TodoKind::EmptyBlock, location, message: None })], };
Ok(UntypedExpr::Block { location, statements, }) }
// The left side of an "=" or a "->" fn parse_pattern( &mut self, position: PatternPosition, ) -> Result<Option<UntypedPattern>, ParseError> { let pattern = match self.tok0.take() { // Pattern::Var or Pattern::Constructor start Some((start, Token::Name { name }, end)) => { self.advance();
// A variable is not permitted on the left hand side of a `<>` if let Some((_, Token::Concatenate, _)) = self.tok0.as_ref() { return concat_pattern_variable_left_hand_side_error(start, end); }
if self.maybe_one(&Token::Dot).is_some() { // We're doing this to get a better error message instead of a generic // `I was expecting a type`, you can have a look at this issue to get // a better idea: https://github.com/gleam-lang/gleam/issues/2841. match self.expect_constructor_pattern(Some((start, name, end)), position) { Ok(result) => result, Err(ParseError { location: SrcSpan { end, .. }, .. }) => { return parse_error( ParseErrorType::InvalidModuleTypePattern, SrcSpan { start, end }, ); } } } else { Pattern::Variable { origin: VariableOrigin { syntax: VariableSyntax::Variable(name.clone()), declaration: position.to_declaration(), }, location: SrcSpan { start, end }, name, type_: (), } } } // Constructor Some((start, tok @ Token::UpName { .. }, end)) => { self.tok0 = Some((start, tok, end)); self.expect_constructor_pattern(None, position)? }
Some((start, Token::DiscardName { name }, end)) => { self.advance();
// A discard is not permitted on the left hand side of a `<>` if let Some((_, Token::Concatenate, _)) = self.tok0.as_ref() { return concat_pattern_variable_left_hand_side_error(start, end); }
Pattern::Discard { location: SrcSpan { start, end }, name, type_: (), } }
Some((start, Token::String { value }, end)) => { self.advance();
match self.tok0 { // String matching with assignment, it could either be a // String prefix matching: "Hello, " as greeting <> name -> ... // or a full string matching: "Hello, World!" as greeting -> ... Some((_, Token::As, _)) => { self.advance(); let (name_start, name, name_end) = self.expect_name(IncorrectNamePosition::AsPattern)?; let name_span = SrcSpan { start: name_start, end: name_end, };
match self.tok0 { // String prefix matching with assignment // "Hello, " as greeting <> name -> ... Some((_, Token::Concatenate, _)) => { self.advance(); let (r_start, right, r_end) = self.expect_assign_name()?;
// Can't match on suffix literal if let Some(( second_concat_start, Token::Concatenate, second_concat_end, )) = self.tok0 { let suffix_end = match &self.tok1 { Some((_start, Token::String { .. }, end)) => *end, _ => second_concat_end, }; return concat_pattern_variable_with_suffix( second_concat_start, right.name().clone(), suffix_end, ); }
Pattern::StringPrefix { location: SrcSpan { start, end: r_end }, left_location: SrcSpan { start, end: name_end, }, right_location: SrcSpan { start: r_start, end: r_end, }, left_side_string: value, left_side_assignment: Some((name, name_span)), right_side_assignment: right, } } // Full string matching with assignment _ => { return Ok(Some(Pattern::Assign { name, location: name_span, pattern: Box::new(Pattern::String { location: SrcSpan { start, end }, value, }), })); } } }
// String prefix matching with no left side assignment // "Hello, " <> name -> ... Some((_, Token::Concatenate, _)) => { self.advance(); let (r_start, right, r_end) = self.expect_assign_name()?;
// Can't match on suffix literal if let Some((second_concat_start, Token::Concatenate, second_concat_end)) = self.tok0 { let suffix_end = match &self.tok1 { Some((_start, Token::String { .. }, end)) => *end, _ => second_concat_end, }; return concat_pattern_variable_with_suffix( second_concat_start, right.name().clone(), suffix_end, ); }
Pattern::StringPrefix { location: SrcSpan { start, end: r_end }, left_location: SrcSpan { start, end }, right_location: SrcSpan { start: r_start, end: r_end, }, left_side_string: value, left_side_assignment: None, right_side_assignment: right, } }
// Full string matching // "Hello, World!" -> ... _ => Pattern::String { location: SrcSpan { start, end }, value, }, } } Some((start, Token::Int { value, int_value }, end)) => { self.advance(); Pattern::Int { location: SrcSpan { start, end }, value, int_value, } } Some((start, Token::Float { value, float_value }, end)) => { self.advance(); Pattern::Float { location: SrcSpan { start, end }, value, float_value, } } Some((start, Token::Hash, _)) => { self.advance(); let _ = self.expect_one(&Token::LeftParen)?; let elements = self.series_of(&|this| this.parse_pattern(position), Some(&Token::Comma))?; let (_, end) = self.expect_one_following_series(&Token::RightParen, "a pattern")?; Pattern::Tuple { location: SrcSpan { start, end }, elements, } } // BitArray Some((start, Token::LtLt, _)) => { self.advance(); let segments = self.series_of( &|this| { this.parse_bit_array_segment( &|this| match this.parse_pattern(position) { Ok(Some(Pattern::BitArray { location, .. })) => { parse_error(ParseErrorType::NestedBitArrayPattern, location) } x => x, }, &Parser::expect_bit_array_pattern_segment_arg, &bit_array_size_int, ) }, Some(&Token::Comma), )?; let (_, end) = self.expect_one_following_series(&Token::GtGt, "a bit array segment pattern")?; Pattern::BitArray { location: SrcSpan { start, end }, segments, } }
// List Some((start, Token::LeftSquare, _)) => { self.advance(); let (elements, elements_end_with_comma) = self.series_of_has_trailing_separator( &|this| this.parse_pattern(position), Some(&Token::Comma), )?;
let mut elements_after_tail = None; let mut dot_dot_location = None; let tail = match self.tok0 { Some((dot_dot_start, Token::DotDot, dot_dot_end)) => { dot_dot_location = Some((dot_dot_start, dot_dot_end)); if !elements.is_empty() && !elements_end_with_comma { self.warnings .push(DeprecatedSyntaxWarning::DeprecatedListPattern { location: SrcSpan { start: dot_dot_start, end: dot_dot_end, }, }); }
self.advance(); let tail = self.parse_pattern(position)?; if self.maybe_one(&Token::Comma).is_some() { // See if there's a list of items after the tail, // like `[..wibble, wobble, wabble]` let elements = self.series_of( &|this| this.parse_pattern(position), Some(&Token::Comma), ); match elements { Err(_) => {} Ok(elements) => { elements_after_tail = Some(elements); } } } Some(tail) } _ => None, };
let (end, closing_square_bracket_end) = self.expect_one_following_series(&Token::RightSquare, "a pattern")?;
// If there are elements after the tail, return an error match elements_after_tail { Some(elements) if !elements.is_empty() => { let (start, end) = match (dot_dot_location, tail) { (Some((start, _)), Some(Some(tail))) => (start, tail.location().end), (Some((start, end)), Some(None)) => (start, end), (_, _) => (start, end), }; return parse_error( ParseErrorType::ListPatternSpreadFollowedByElements, SrcSpan { start, end }, ); } _ => {} }
let tail = match tail { // There is a tail and it has a Pattern::Var or Pattern::Discard Some(Some(pattern @ (Pattern::Variable { .. } | Pattern::Discard { .. }))) => { Some(pattern) } // There is a tail and but it has no content, implicit discard Some(Some(pattern)) => { return parse_error(ParseErrorType::InvalidTailPattern, pattern.location()); } Some(None) => Some(Pattern::Discard { location: SrcSpan { start: closing_square_bracket_end - 1, end: closing_square_bracket_end, }, name: "_".into(), type_: (), }), // No tail specified None => None, };
if elements.is_empty() && tail.as_ref().is_some_and(|pattern| pattern.is_discard()) { self.warnings .push(DeprecatedSyntaxWarning::DeprecatedListCatchAllPattern { location: SrcSpan { start, end: closing_square_bracket_end, }, }); }
Pattern::List { location: SrcSpan { start, end: closing_square_bracket_end, }, elements, tail: tail.map(|tail_pattern| { let dot_dot_start = dot_dot_location .expect("parsed tail with no preceding `..`") .0;
Box::new(TailPattern { location: SrcSpan::new(dot_dot_start, tail_pattern.location().end), pattern: tail_pattern, }) }), type_: (), } }
// No pattern t0 => { self.tok0 = t0; return Ok(None); } };
match self.tok0 { Some((_, Token::As, _)) => { self.advance(); let (start, name, end) = self.expect_name(IncorrectNamePosition::AsPattern)?; Ok(Some(Pattern::Assign { name, location: SrcSpan { start, end }, pattern: Box::new(pattern), })) } _ => Ok(Some(pattern)), } }
fn add_multi_line_clause_hint(&self, mut err: ParseError) -> ParseError { if let ParseErrorType::UnexpectedToken { ref mut hint, .. } = err.error { *hint = Some("Did you mean to wrap a multi line clause in curly braces?".into()); } err }
// examples: // pattern -> expr // pattern, pattern if -> expr // pattern, pattern | pattern, pattern if -> expr fn parse_case_clause(&mut self) -> Result<Option<UntypedClause>, ParseError> { let patterns = self.parse_patterns(PatternPosition::CaseClause)?; match &patterns.first() { Some(lead) => { let mut alternative_patterns = vec![]; while let Some((vbar_start, vbar_end)) = self.maybe_one(&Token::Vbar) { let patterns = self.parse_patterns(PatternPosition::CaseClause)?; if patterns.is_empty() { return parse_error( ParseErrorType::ExpectedPattern, SrcSpan { start: vbar_start, end: vbar_end, }, ); } alternative_patterns.push(patterns); } let guard = self.parse_case_clause_guard()?; let (arr_s, arr_e) = self .expect_one(&Token::RArrow) .map_err(|error| self.add_multi_line_clause_hint(error))?; let then = self.parse_expression()?; match then { Some(then) => Ok(Some(Clause { location: SrcSpan { start: lead.location().start, end: then.location().end, }, pattern: patterns, alternative_patterns, guard, then, })), None => match self.tok0 { Some((start, Token::DiscardName { .. }, end)) => parse_error( ParseErrorType::IncorrectName { kind: IncorrectNamePosition::Variable, }, SrcSpan { start, end }, ), _ => parse_error( ParseErrorType::ExpectedExpr, SrcSpan { start: arr_s, end: arr_e, }, ), }, } } _ => Ok(None), } } fn parse_patterns( &mut self, position: PatternPosition, ) -> Result<Vec<UntypedPattern>, ParseError> { self.series_of(&|this| this.parse_pattern(position), Some(&Token::Comma)) }
// examples: // if a // if a < b // if a < b || b < c fn parse_case_clause_guard(&mut self) -> Result<Option<UntypedClauseGuard>, ParseError> { let Some((start, end)) = self.maybe_one(&Token::If) else { return Ok(None); }; let clause_guard_result = self.parse_clause_guard_inner(); // If inner clause is none, a warning should be shown to let the user // know that empty clauses in guards are deprecated. if let Ok(None) = clause_guard_result { self.warnings .push(DeprecatedSyntaxWarning::DeprecatedEmptyClauseGuard { location: SrcSpan { start, end }, }); } clause_guard_result }
fn parse_clause_guard_inner(&mut self) -> Result<Option<UntypedClauseGuard>, ParseError> { let mut opstack = vec![]; let mut estack = vec![]; let mut last_op_start = 0; let mut last_op_end = 0; loop { match self.parse_case_clause_guard_unit()? { Some(unit) => estack.push(unit), _ => { if estack.is_empty() { return Ok(None); } else { return parse_error( ParseErrorType::OpNakedRight, SrcSpan { start: last_op_start, end: last_op_end, }, ); } } }
let Some((op_s, t, op_e)) = self.tok0.take() else { break; };
let Some(precedence) = t.guard_precedence() else { // Is not Op self.tok0 = Some((op_s, t, op_e)); break; };
// Is Op self.advance(); last_op_start = op_s; last_op_end = op_e; let _ = handle_op( Some(((op_s, t, op_e), precedence)), &mut opstack, &mut estack, &do_reduce_clause_guard, ); }
Ok(handle_op( None, &mut opstack, &mut estack, &do_reduce_clause_guard, )) }
/// Checks if we have an unexpected left parenthesis and returns appropriate /// error if it is a function call. fn parse_function_call_in_clause_guard(&mut self, start: u32) -> Result<(), ParseError> { if let Some((l_paren_start, l_paren_end)) = self.maybe_one(&Token::LeftParen) { if let Ok((_, end)) = self .parse_fn_arguments() .and(self.expect_one(&Token::RightParen)) { return parse_error(ParseErrorType::CallInClauseGuard, SrcSpan { start, end }); }
return parse_error( ParseErrorType::UnexpectedToken { token: Token::LeftParen, expected: vec![Token::RArrow.to_string().into()], hint: None, }, SrcSpan { start: l_paren_start, end: l_paren_end, }, ) .map_err(|error| self.add_multi_line_clause_hint(error)); }
Ok(()) }
// examples // a // 1 // a.1 // { a } // a || b // a < b || b < c fn parse_case_clause_guard_unit(&mut self) -> Result<Option<UntypedClauseGuard>, ParseError> { match self.tok0.take() { Some((start, Token::Bang, _)) => { self.advance(); match self.parse_case_clause_guard_unit()? { Some(unit) => Ok(Some(ClauseGuard::Not { location: SrcSpan { start, end: unit.location().end, }, expression: Box::new(unit), })), None => { parse_error(ParseErrorType::ExpectedValue, SrcSpan { start, end: start }) } } }
Some((start, Token::Name { name }, end)) => { self.advance();
self.parse_function_call_in_clause_guard(start)?;
let mut unit = match self.parse_record_in_clause_guard(&name, SrcSpan { start, end })? { Some(record) => record, _ => ClauseGuard::Var { location: SrcSpan { start, end }, type_: (), name, definition_location: SrcSpan::default(), // We don't know the origin until type analysis, so // we just put `Generated` here as a placeholder. origin: VariableOrigin { syntax: VariableSyntax::Generated, declaration: VariableDeclaration::Generated, }, }, };
loop { let dot_s = match self.maybe_one(&Token::Dot) { Some((dot_s, _)) => dot_s, None => return Ok(Some(unit)), };
match self.next_tok() { Some(( _, Token::Int { value, int_value: _, }, int_e, )) => { let v = value.replace("_", ""); match u64::from_str(&v) { Ok(index) => { unit = ClauseGuard::TupleIndex { location: SrcSpan { start: dot_s, end: int_e, }, index, type_: (), tuple: Box::new(unit), }; } _ => { return parse_error( ParseErrorType::InvalidTupleAccess, SrcSpan { start, end }, ); } } }
Some((name_start, Token::Name { name: label }, name_end)) => { self.parse_function_call_in_clause_guard(start)?;
unit = ClauseGuard::FieldAccess { label_location: SrcSpan { start: name_start, end: name_end, }, index: None, label, type_: (), container: Box::new(unit), }; }
Some((start, _, end)) => { return parse_error( ParseErrorType::IncorrectName { kind: IncorrectNamePosition::Variable, }, SrcSpan { start, end }, ); }
_ => return self.next_tok_unexpected(vec!["A positive integer".into()]), } } }
Some((start, Token::LeftBrace, _)) => { self.advance(); Ok(Some(self.parse_case_clause_guard_block(start)?)) }
t0 => { self.tok0 = t0; match self.parse_const_value()? { Some(const_val) => { // Constant Ok(Some(ClauseGuard::Constant(const_val))) } _ => Ok(None), } } } }
fn parse_case_clause_guard_block( &mut self, start: u32, ) -> Result<UntypedClauseGuard, ParseError> { let body = self.parse_clause_guard_inner()?;
let Some(body) = body else { let location = match self.next_tok() { Some((_, Token::RightBrace, end)) => SrcSpan { start, end }, Some((_, _, _)) | None => SrcSpan { start, end: start + 1, }, };
return parse_error(ParseErrorType::EmptyGuardBlock, location); };
let (_, end) = self.expect_one(&Token::RightBrace)?; Ok(ClauseGuard::Block { location: SrcSpan { start, end }, value: Box::new(body), }) }
fn parse_record_in_clause_guard( &mut self, module: &EcoString, module_location: SrcSpan, ) -> Result<Option<UntypedClauseGuard>, ParseError> { let (name, end) = match (self.tok0.take(), self.peek_tok1()) { (Some((_, Token::Dot, _)), Some(Token::UpName { .. })) => { self.advance(); // dot let Some((_, Token::UpName { name }, end)) = self.next_tok() else { return Ok(None); }; (name, end) } (tok0, _) => { self.tok0 = tok0; return Ok(None); } };
match self.parse_const_record_finish( module_location.start, Some((module.clone(), module_location)), name, end, )? { Some(record) => Ok(Some(ClauseGuard::Constant(record))), _ => Ok(None), } }
// examples: // UpName( args ) fn expect_constructor_pattern( &mut self, module: Option<(u32, EcoString, u32)>, position: PatternPosition, ) -> Result<UntypedPattern, ParseError> { let (name_start, name, name_end) = self.expect_upname()?; let mut start = name_start; let (arguments, spread, end) = self.parse_constructor_pattern_arguments(name_end, position)?; if let Some((s, _, _)) = module { start = s; } Ok(Pattern::Constructor { location: SrcSpan { start, end }, name_location: SrcSpan::new(name_start, name_end), arguments, module: module.map(|(start, n, end)| (n, SrcSpan { start, end })), name, spread, constructor: Inferred::Unknown, type_: (), }) }
// examples: // ( args ) #[allow(clippy::type_complexity)] fn parse_constructor_pattern_arguments( &mut self, upname_end: u32, position: PatternPosition, ) -> Result<(Vec<CallArg<UntypedPattern>>, Option<SrcSpan>, u32), ParseError> { if self.maybe_one(&Token::LeftParen).is_some() { let (arguments, arguments_end_with_comma) = self.series_of_has_trailing_separator( &|this| this.parse_constructor_pattern_arg(position), Some(&Token::Comma), )?;
let spread = self .maybe_one(&Token::DotDot) .map(|(start, end)| SrcSpan { start, end });
if let Some(spread_location) = spread { let _ = self.maybe_one(&Token::Comma); if !arguments.is_empty() && !arguments_end_with_comma { self.warnings .push(DeprecatedSyntaxWarning::DeprecatedRecordSpreadPattern { location: spread_location, }); } } let (_, end) = self.expect_one(&Token::RightParen)?; Ok((arguments, spread, end)) } else { Ok((vec![], None, upname_end)) } }
// examples: // a: <pattern> // a: // <pattern> fn parse_constructor_pattern_arg( &mut self, position: PatternPosition, ) -> Result<Option<CallArg<UntypedPattern>>, ParseError> { match (self.tok0.take(), self.tok1.take()) { // named arg (Some((start, Token::Name { name }, _)), Some((_, Token::Colon, end))) => { self.advance(); self.advance(); match self.parse_pattern(position)? { Some(value) => Ok(Some(CallArg { implicit: None, location: SrcSpan { start, end: value.location().end, }, label: Some(name), value, })), _ => { // Argument supplied with a label shorthand. Ok(Some(CallArg { implicit: None, location: SrcSpan { start, end }, label: Some(name.clone()), value: UntypedPattern::Variable { origin: VariableOrigin { syntax: VariableSyntax::LabelShorthand(name.clone()), declaration: position.to_declaration(), }, name, location: SrcSpan { start, end }, type_: (), }, })) } } } // unnamed arg (t0, t1) => { self.tok0 = t0; self.tok1 = t1; match self.parse_pattern(position)? { Some(value) => Ok(Some(CallArg { implicit: None, location: value.location(), label: None, value, })), _ => Ok(None), } } } }
// examples: // a: expr // a: fn parse_record_update_arg(&mut self) -> Result<Option<UntypedRecordUpdateArg>, ParseError> { match self.maybe_name() { Some((start, label, _)) => { let (_, end) = self.expect_one(&Token::Colon)?; let value = self.parse_expression()?; match value { Some(value) => Ok(Some(UntypedRecordUpdateArg { label, location: SrcSpan { start, end: value.location().end, }, value, })), _ => { // Argument supplied with a label shorthand. Ok(Some(UntypedRecordUpdateArg { label: label.clone(), location: SrcSpan { start, end }, value: UntypedExpr::Var { name: label, location: SrcSpan { start, end }, }, })) } } } _ => Ok(None), } }
// // Parse Functions //
// Starts after "fn" // // examples: // fn a(name: String) -> String { .. } // pub fn a(name name: String) -> String { .. } fn parse_function( &mut self, start: u32, public: bool, is_anon: bool, attributes: &mut Attributes, ) -> Result<Option<UntypedDefinition>, ParseError> { let documentation = if is_anon { None } else { self.take_documentation(start) }; let mut name = None; if !is_anon { let (name_start, n, name_end) = self.expect_name(IncorrectNamePosition::Function)?; name = Some(( SrcSpan { start: name_start, end: name_end, }, n, )); } if let Some((less_start, less_end)) = self.maybe_one(&Token::Less) { return Err(ParseError { error: ParseErrorType::FunctionDefinitionAngleGenerics, location: SrcSpan { start: less_start, end: less_end, }, }); } let _ = self .expect_one(&Token::LeftParen) .map_err(|error| self.add_anon_function_hint(error))?; let arguments = self.series_of(&|this| this.parse_fn_param(is_anon), Some(&Token::Comma))?; let (_, rpar_e) = self.expect_one_following_series(&Token::RightParen, "a function parameter")?;
// Check for TypeScript-style return type annotation (:) instead of arrow (->) if let Some((colon_start, colon_end)) = self.maybe_one(&Token::Colon) { return Err(ParseError { error: ParseErrorType::UnexpectedToken { token: Token::Colon, expected: vec!["`->`".into()], hint: Some("Return type annotations are written using `->`, not `:`".into()), }, location: SrcSpan { start: colon_start, end: colon_end, }, }); }
let return_annotation = self.parse_type_annotation(&Token::RArrow)?;
let (body_start, body, end, end_position) = match self.maybe_one(&Token::LeftBrace) { Some((left_brace_start, _)) => { let some_body = self.parse_statement_seq()?; let (_, right_brace_end) = self.expect_one(&Token::RightBrace)?; let end = return_annotation .as_ref() .map(|l| l.location().end) .unwrap_or(rpar_e); let body = match some_body { None => vec![Statement::Expression(UntypedExpr::Todo { kind: TodoKind::EmptyFunction { function_location: SrcSpan { start, end }, }, location: SrcSpan { start: left_brace_start + 1, end: right_brace_end, }, message: None, })], Some((body, _)) => body.to_vec(), };
(Some(left_brace_start), body, end, right_brace_end) }
None => (None, vec![], rpar_e, rpar_e), };
Ok(Some(Definition::Function(Function { documentation, location: SrcSpan { start, end }, end_position, body_start, publicity: self.publicity(public, attributes.internal)?, name, arguments, body, return_type: (), return_annotation, deprecation: std::mem::take(&mut attributes.deprecated), external_erlang: attributes.external_erlang.take(), external_javascript: attributes.external_javascript.take(), external_wasm: attributes.external_wasm.take(), implementations: Implementations { gleam: true, can_run_on_erlang: true, can_run_on_javascript: true, can_run_on_wasm: true, uses_erlang_externals: false, uses_javascript_externals: false, uses_wasm_externals: false, }, purity: Purity::Pure, }))) }
fn add_anon_function_hint(&self, mut err: ParseError) -> ParseError { if let ParseErrorType::UnexpectedToken { ref mut hint, token: Token::Name { .. }, .. } = err.error { *hint = Some("Only module-level functions can be named.".into()); } err }
fn publicity( &self, public: bool, internal: InternalAttribute, ) -> Result<Publicity, ParseError> { match (internal, public) { (InternalAttribute::Missing, true) => Ok(Publicity::Public), (InternalAttribute::Missing, false) => Ok(Publicity::Private), (InternalAttribute::Present(location), true) => Ok(Publicity::Internal { attribute_location: Some(location), }), (InternalAttribute::Present(location), false) => Err(ParseError { error: ParseErrorType::RedundantInternalAttribute, location, }), } }
// Parse a single function definition param // // examples: // _ // a // a a // a _ // a _:A // a a:A fn parse_fn_param(&mut self, is_anon: bool) -> Result<Option<UntypedArg>, ParseError> { let (start, names, mut end) = match (self.tok0.take(), self.tok1.take()) { // labeled discard ( Some((start, Token::Name { name: label }, tok0_end)), Some((name_start, Token::DiscardName { name }, end)), ) => { if is_anon { return parse_error( ParseErrorType::UnexpectedLabel, SrcSpan { start, end: tok0_end, }, ); }
self.advance(); self.advance(); ( start, ArgNames::LabelledDiscard { name, name_location: SrcSpan::new(name_start, end), label, label_location: SrcSpan::new(start, tok0_end), }, end, ) } // discard (Some((start, Token::DiscardName { name }, end)), t1) => { self.tok1 = t1; self.advance(); ( start, ArgNames::Discard { name, location: SrcSpan { start, end }, }, end, ) } // labeled name ( Some((start, Token::Name { name: label }, tok0_end)), Some((name_start, Token::Name { name }, end)), ) => { if is_anon { return parse_error( ParseErrorType::UnexpectedLabel, SrcSpan { start, end: tok0_end, }, ); }
self.advance(); self.advance(); ( start, ArgNames::NamedLabelled { name, name_location: SrcSpan::new(name_start, end), label, label_location: SrcSpan::new(start, tok0_end), }, end, ) } // name (Some((start, Token::Name { name }, end)), t1) => { self.tok1 = t1; self.advance(); ( start, ArgNames::Named { name, location: SrcSpan { start, end }, }, end, ) } (t0, t1) => { self.tok0 = t0; self.tok1 = t1; return Ok(None); } }; let annotation = match self.parse_type_annotation(&Token::Colon)? { Some(a) => { end = a.location().end; Some(a) } _ => None, }; Ok(Some(Arg { location: SrcSpan { start, end }, type_: (), names, annotation, })) }
// Parse function call arguments, no parens // // examples: // _ // expr, expr // a: _, expr // a: expr, _, b: _ fn parse_fn_arguments(&mut self) -> Result<Vec<ParserArg>, ParseError> { self.series_of(&Parser::parse_fn_argument, Some(&Token::Comma)) }
// Parse a single function call arg // // examples: // _ // expr // a: _ // a: expr fn parse_fn_argument(&mut self) -> Result<Option<ParserArg>, ParseError> { let label = match (self.tok0.take(), self.tok1.take()) { (Some((start, Token::Name { name }, _)), Some((_, Token::Colon, end))) => { self.advance(); self.advance(); Some((start, name, end)) } (t0, t1) => { self.tok0 = t0; self.tok1 = t1; None } };
match self.parse_expression()? { Some(value) => { let arg = match label { Some((start, label, _)) => CallArg { implicit: None, label: Some(label), location: SrcSpan { start, end: value.location().end, }, value, }, _ => CallArg { implicit: None, label: None, location: value.location(), value, }, }; Ok(Some(ParserArg::Arg(Box::new(arg)))) } _ => { match self.maybe_discard_name() { Some((name_start, name, name_end)) => { let arg = match label { Some((label_start, label, _)) => ParserArg::Hole { label: Some(label), arg_location: SrcSpan { start: label_start, end: name_end, }, discard_location: SrcSpan { start: name_start, end: name_end, }, name, }, _ => ParserArg::Hole { label: None, arg_location: SrcSpan { start: name_start, end: name_end, }, discard_location: SrcSpan { start: name_start, end: name_end, }, name, }, };
Ok(Some(arg)) } _ => { match label { Some((start, label, end)) => { // Argument supplied with a label shorthand. Ok(Some(ParserArg::Arg(Box::new(CallArg { implicit: None, label: Some(label.clone()), location: SrcSpan { start, end }, value: UntypedExpr::Var { name: label, location: SrcSpan { start, end }, }, })))) } _ => Ok(None), } } } } } }
// // Parse Custom Types //
// examples: // type A { A } // type A { A(String) } // type Box(inner_type) { Box(inner: inner_type) } // type NamedBox(inner_type) { Box(String, inner: inner_type) } fn parse_custom_type( &mut self, start: u32, public: bool, opaque: bool, attributes: &mut Attributes, ) -> Result<Option<UntypedDefinition>, ParseError> { let documentation = self.take_documentation(start); let (name_start, name, parameters, end, name_end) = self.expect_type_name()?; let name_location = SrcSpan::new(name_start, name_end);
let (constructors, end_position) = match self.tok0.take() { // If we see `type Wibble {`, then we know we're parsing a custom type. Some((_, Token::LeftBrace, _)) => { self.advance();
// If we see a lowercase name, rather than an uppercase one. We // know there's a syntax error! So now we can try and provide a // nice error message, based on what that wrong code looks like. if let Some((name_start, Token::Name { .. }, name_end)) = &self.tok0 { return Err(self.invalid_record_constructor_error( name, public, opaque, *name_start, *name_end, )); }
let constructors = self.series_of( &|this| this.parse_record_constructor(), // No separator None, )?; let close_end = self.expect_custom_type_close()?; (constructors, close_end) }
// If we see `type Wibble =` then we know we're parsing a type alias. Some((equal_start, Token::Equal, equal_end)) => { self.advance();
if opaque { return parse_error(ParseErrorType::OpaqueTypeAlias, SrcSpan { start, end }); }
if let Some(type_) = self.parse_type()? { return Ok(Some(Definition::TypeAlias(TypeAlias { documentation, location: SrcSpan::new(start, type_.location().end), publicity: self.publicity(public, attributes.internal)?, alias: name, name_location, parameters, type_ast: type_, type_: (), deprecation: std::mem::take(&mut attributes.deprecated), }))); } else { return parse_error( ParseErrorType::ExpectedType, SrcSpan::new(equal_start, equal_end), ); } }
token @ (Some(_) | None) => { self.tok0 = token; (vec![], end) } };
Ok(Some(Definition::CustomType(CustomType { documentation, location: SrcSpan { start, end }, end_position, publicity: self.publicity(public, attributes.internal)?, opaque, name, name_location, parameters, constructors, typed_parameters: vec![], deprecation: std::mem::take(&mut attributes.deprecated), external_erlang: std::mem::take(&mut attributes.external_erlang), external_javascript: std::mem::take(&mut attributes.external_javascript), }))) }
fn parse_record_constructor(&mut self) -> Result<Option<RecordConstructor<()>>, ParseError> { // The only attribute supported on constructors is @deprecated let mut attributes = Attributes::default(); let attr_loc = self.parse_attributes(&mut attributes)?;
if let Some(attr_span) = attr_loc { // Expecting all but the deprecated atterbutes to be default if attributes.external_erlang.is_some() || attributes.external_javascript.is_some() || attributes.target.is_some() || attributes.internal != InternalAttribute::Missing { return parse_error(ParseErrorType::UnknownAttributeRecordVariant, attr_span); } }
match self.maybe_upname() { Some((name_start, constructor_name, name_end)) => { let documentation = self.take_documentation(name_start); let (arguments, arguments_end) = self.parse_record_constructor_arguments()?;
Ok(Some(RecordConstructor { location: SrcSpan { start: name_start, end: arguments_end.max(name_end), }, name_location: SrcSpan { start: name_start, end: name_end, }, name: constructor_name, arguments, documentation, deprecation: attributes.deprecated, })) } _ => Ok(None), } }
/// This takes place when we find a lowercase name as a record constructor /// variant (that name is passed as an argument here). /// We want to look at the following tokens to produce a nice error message: /// /// ```gleam /// pub type Wibble { /// wibble /// //^^^^^^ Error, this should be uppercase! /// } /// ``` /// /// But if the thing looks like a record definition, we want a specialised /// error message: /// /// ```gleam /// pub type Wibble { /// wibble: Int, /// wobble: String /// } /// // Suggest wrapping this in a constructor. /// ``` /// fn invalid_record_constructor_error( &mut self, type_name: EcoString, public: bool, opaque: bool, name_start: u32, name_end: u32, ) -> ParseError { let fields = self.series_of( &|this| this.parse_record_constructor_field(), Some(&Token::Comma), );
match fields { // If there's a list of fields right inside the type that means the // developer might have forgotten to wrap the thing in a constructor. // Basically writing something like this: // // ```gleam // pub type Wibble { // String, // wibble: Int, // } // ``` // // So we want to produce a specialised error message pointing them // in the right direction. Ok(fields) if let Some((_, Token::RightBrace, _)) = self.tok0 => ParseError { location: SrcSpan { start: fields .first() .map_or(name_start, |field| field.location.start), end: fields.last().map_or(name_end, |field| field.location.end), }, error: ParseErrorType::ExpectedRecordConstructor { type_name, public, opaque, fields, }, },
// Otherwise we fall back to telling them the lowercase name should // be uppercased! Ok(_) | Err(_) => ParseError { error: ParseErrorType::IncorrectUpName, location: SrcSpan { start: name_start, end: name_end, }, }, } }
// examples: // *no args* // () // (a, b) fn parse_record_constructor_arguments( &mut self, ) -> Result<(Vec<RecordConstructorArg<()>>, u32), ParseError> { if self.maybe_one(&Token::LeftParen).is_some() { let arguments = self.series_of( &|this| this.parse_record_constructor_field(), Some(&Token::Comma), )?; let (_, end) = self .expect_one_following_series(&Token::RightParen, "a constructor argument name")?; Ok((arguments, end)) } else { Ok((vec![], 0)) } }
fn parse_record_constructor_field( &mut self, ) -> Result<Option<RecordConstructorArg<()>>, ParseError> { match (self.tok0.take(), self.tok1.take()) { (Some((start, Token::Name { name }, name_end)), Some((_, Token::Colon, end))) => { let _ = self.next_tok(); let _ = self.next_tok(); let doc = self.take_documentation(start); match self.parse_type()? { Some(type_ast) => { let end = type_ast.location().end; Ok(Some(RecordConstructorArg { label: Some((SrcSpan::new(start, name_end), name)), ast: type_ast, location: SrcSpan { start, end }, type_: (), doc, })) } None => parse_error(ParseErrorType::ExpectedType, SrcSpan { start, end }), } } (t0, t1) => { self.tok0 = t0; self.tok1 = t1; match self.parse_type()? { Some(type_ast) => { let doc = match &self.tok0 { Some((start, _, _)) => self.take_documentation(*start), None => None, }; let type_location = type_ast.location(); Ok(Some(RecordConstructorArg { label: None, ast: type_ast, location: type_location, type_: (), doc, })) } None => Ok(None), } } } }
// examples: // A // A(one, two) fn expect_type_name( &mut self, ) -> Result<(u32, EcoString, Vec<SpannedString>, u32, u32), ParseError> { let (start, upname, end) = self.expect_upname()?; if let Some((par_s, _)) = self.maybe_one(&Token::LeftParen) { let arguments = self.series_of(&|this| Ok(this.maybe_name()), Some(&Token::Comma))?; let (_, par_e) = self.expect_one_following_series(&Token::RightParen, "a name")?; if arguments.is_empty() { return parse_error( ParseErrorType::TypeDefinitionNoArguments, SrcSpan::new(par_s, par_e), ); } let arguments2 = arguments .into_iter() .map(|(start, name, end)| (SrcSpan { start, end }, name)) .collect(); Ok((start, upname, arguments2, par_e, end)) } else if let Some((less_start, less_end)) = self.maybe_one(&Token::Less) { let mut arguments = self.series_of( &|this| // Permit either names (`a`) or upnames (`A`) in this error-handling mode, // as upnames are common in other languages. Convert to lowercase so the // example is correct whichever was used. Ok(this.maybe_name() .or_else(|| this.maybe_upname()) .map(|(_, name, _)| name.to_lowercase())), Some(&Token::Comma), )?;
// If no type arguments were parsed, fall back to a dummy type argument as an example, // because `Type()` would be invalid if arguments.is_empty() { arguments = vec!["value".into()]; }
Err(ParseError { error: ParseErrorType::TypeDefinitionAngleGenerics { name: upname, arguments, }, location: SrcSpan { start: less_start, end: less_end, }, }) } else { Ok((start, upname, vec![], end, end)) } }
// // Parse Type Annotations //
// examples: // :a // :Int // :Result(a, _) // :Result(Result(a, e), #(_, String)) fn parse_type_annotation(&mut self, start_tok: &Token) -> Result<Option<TypeAst>, ParseError> { if let Some((start, end)) = self.maybe_one(start_tok) { match self.parse_type() { Ok(None) => parse_error(ParseErrorType::ExpectedType, SrcSpan { start, end }), other => other, } } else { Ok(None) } }
// Parse the type part of a type annotation, same as `parse_type_annotation` minus the ":" fn parse_type(&mut self) -> Result<Option<TypeAst>, ParseError> { match self.tok0.take() { // Type hole Some((start, Token::DiscardName { name }, end)) => { self.advance(); Ok(Some(TypeAst::Hole(TypeAstHole { location: SrcSpan { start, end }, name, }))) }
// Tuple Some((start, Token::Hash, _)) => { self.advance(); let _ = self.expect_one(&Token::LeftParen)?; let elements = self.parse_types()?; let (_, end) = self.expect_one(&Token::RightParen)?; Ok(Some(TypeAst::Tuple(TypeAstTuple { location: SrcSpan { start, end }, elements, }))) }
// Function Some((start, Token::Fn, _)) => { self.advance(); let _ = self.expect_one(&Token::LeftParen)?; let arguments = self.series_of(&|this| this.parse_type(), Some(&Token::Comma))?; let _ = self.expect_one_following_series(&Token::RightParen, "a type")?; let (arr_s, arr_e) = self.expect_one(&Token::RArrow)?; let return_ = self.parse_type()?; match return_ { Some(return_) => Ok(Some(TypeAst::Fn(TypeAstFn { location: SrcSpan { start, end: return_.location().end, }, return_: Box::new(return_), arguments, }))), _ => parse_error( ParseErrorType::ExpectedType, SrcSpan { start: arr_s, end: arr_e, }, ), } }
// Constructor function Some((start, Token::UpName { name }, end)) => { self.advance(); let name = TypeAstConstructorName::Unqualified { name, location: SrcSpan::new(start, end), }; self.parse_type_name_finish(start, end, name) }
// Constructor Module or type Variable Some((start, Token::Name { name: module }, end)) => { self.advance();
if let Some((_, dot_end)) = self.maybe_one(&Token::Dot) { let module_location = SrcSpan::new(start, end); match self.maybe_upname() { Some((name_start, name, name_end)) => { let name = TypeAstConstructorName::Qualified { module, module_location, dot_location: dot_end, name: Some((name, SrcSpan::new(name_start, name_end))), }; self.parse_type_name_finish(start, name_end, name) } None => { let name = TypeAstConstructorName::Qualified { module, module_location, dot_location: dot_end, name: None, }; self.parse_type_name_finish(start, dot_end, name) } } } else { Ok(Some(TypeAst::Var(TypeAstVar { location: SrcSpan { start, end }, name: module, }))) } }
t0 => { self.tok0 = t0; Ok(None) } } }
// Parse the '( ... )' of a type name fn parse_type_name_finish( &mut self, start: u32, end: u32, name: TypeAstConstructorName, ) -> Result<Option<TypeAst>, ParseError> { if let Some((left_paren_start, left_paren_end)) = self.maybe_one(&Token::LeftParen) { // In case the type is qualified and is missing a name, it doesn't // make sense to parse a types list: we don't want to accept // something like `wibble.(a, b)`. // Instead we want to say that `(` is unexpected and we were // expecting a type name instead: if name.name().is_none() { return Err(ParseError { error: ParseErrorType::ExpectedUpName, location: SrcSpan::new(left_paren_start, left_paren_end), }); }
let arguments = self.parse_types()?; let (_, right_paren_end) = self.expect_one(&Token::RightParen)?; Ok(Some(TypeAst::Constructor(TypeAstConstructor { location: SrcSpan::new(start, right_paren_end), name, arguments, start_parentheses: Some(left_paren_start), }))) } else if let Some((less_start, less_end)) = self.maybe_one(&Token::Less) { let location = SrcSpan::new(less_start, less_end); let (module, name) = match name { TypeAstConstructorName::Qualified { module, name: Some((name, _)), .. } => (Some(module), name), TypeAstConstructorName::Unqualified { name, .. } => (None, name),
// If we're here it means someone has typed something truly // wrong that looks like this: `wibble.<`. // In this case the hint about angle brackets wouldn't make much // sense, so we fallback to just reporting an invalid token // error saying we were expecting an uppercase name TypeAstConstructorName::Qualified { name: None, .. } => { return Err(ParseError { error: ParseErrorType::ExpectedUpName, location, }); } };
// Otherwise we try and report a nicer error, suggesting one should // use `(a, b)` instead of `<a, b>`. let arguments = self.parse_types()?; Err(ParseError { location, error: ParseErrorType::TypeUsageAngleGenerics { name, module, arguments, }, }) } else { Ok(Some(TypeAst::Constructor(TypeAstConstructor { location: SrcSpan { start, end }, name, arguments: vec![], start_parentheses: None, }))) } }
// For parsing a comma separated "list" of types, for tuple, constructor, and function fn parse_types(&mut self) -> Result<Vec<TypeAst>, ParseError> { let elements = self.series_of(&|this| this.parse_type(), Some(&Token::Comma))?; Ok(elements) }
// // Parse Imports //
// examples: // import a // import a/b // import a/b.{c} // import a/b.{c as d} as e fn parse_import(&mut self, import_start: u32) -> Result<Option<UntypedDefinition>, ParseError> { let mut start = 0; let mut end; let mut module = EcoString::new(); let mut last_segment_end;
// Gather module names loop { let (s, name, e) = self.expect_name(IncorrectNamePosition::Module)?; if module.is_empty() { start = s; } else { module.push('/'); } module.push_str(&name); end = e; last_segment_end = e;
// Useful error for : import a/.{b} if let Some((s, _)) = self.maybe_one(&Token::SlashDot) { return parse_error( ParseErrorType::ExpectedName, SrcSpan { start: s + 1, end: s + 1, }, ); }
// break if there's no trailing slash if self.maybe_one(&Token::Slash).is_none() { break; } }
let (_, documentation) = self.take_documentation(start).unzip();
// Gather imports let mut unqualified_values = vec![]; let mut unqualified_types = vec![];
if let Some((dot_start, dot_end)) = self.maybe_one(&Token::Dot) { if let Err(e) = self.expect_one(&Token::LeftBrace) { // If the module does contain a '/', then it's unlikely that the user // intended for the import to be pythonic, so skip this. if module.contains('/') { return Err(e); }
// Catch `import gleam.io` and provide a more helpful error... let ParseErrorType::UnexpectedToken { token: Token::Name { name } | Token::UpName { name }, .. } = &e.error else { return Err(e); };
return Err(ParseError { error: ParseErrorType::IncorrectImportModuleSeparator { module, item: name.clone(), }, location: SrcSpan::new(dot_start, dot_end), }); }
let parsed = self.parse_unqualified_imports()?; unqualified_types = parsed.types; unqualified_values = parsed.values; let (_, e) = self.expect_one(&Token::RightBrace)?; end = e; }
// Parse as_name let mut as_name = None; if let Some((as_start, _)) = self.maybe_one(&Token::As) { let (_, name, e) = self.expect_assign_name()?;
end = e; as_name = Some(( name, SrcSpan { start: as_start, end, }, )); }
Ok(Some(Definition::Import(Import { documentation, location: SrcSpan { start: import_start, end, }, module_location: SrcSpan { start, end: last_segment_end, }, unqualified_values, unqualified_types, module, as_name, package: (), }))) }
// [Name (as Name)? | UpName (as Name)? ](, [Name (as Name)? | UpName (as Name)?])*,? fn parse_unqualified_imports(&mut self) -> Result<ParsedUnqualifiedImports, ParseError> { let mut imports = ParsedUnqualifiedImports::default(); loop { // parse imports match self.tok0.take() { Some((start, Token::Name { name }, end)) => { self.advance(); let location = SrcSpan { start, end }; let mut import = UnqualifiedImport { name, location, name_position: location.start, as_name: None, }; if self.maybe_one(&Token::As).is_some() { let (_, as_name, end) = self.expect_name(IncorrectNamePosition::AsPattern)?; import.as_name = Some(as_name); import.location.end = end; } imports.values.push(import); }
Some((start, Token::UpName { name }, end)) => { self.advance(); let location = SrcSpan { start, end }; let mut import = UnqualifiedImport { name, location, name_position: location.start, as_name: None, }; if self.maybe_one(&Token::As).is_some() { let (_, as_name, end) = self.expect_upname()?; import.as_name = Some(as_name); import.location.end = end; } imports.values.push(import); }
Some((start, Token::Type, _)) => { self.advance(); let (name_start, name, end) = self.expect_upname()?; let location = SrcSpan { start, end }; let mut import = UnqualifiedImport { name, location, name_position: name_start, as_name: None, }; if self.maybe_one(&Token::As).is_some() { let (_, as_name, end) = self.expect_upname()?; import.as_name = Some(as_name); import.location.end = end; } imports.types.push(import); }
t0 => { self.tok0 = t0; break; } } // parse comma match self.tok0 { Some((_, Token::Comma, _)) => { self.advance(); } _ => break, } } Ok(imports) }
// // Parse Constants //
// examples: // const a = 1 // const a:Int = 1 // pub const a:Int = 1 fn parse_module_const( &mut self, start: u32, public: bool, attributes: &Attributes, ) -> Result<Option<UntypedDefinition>, ParseError> { let (name_start, name, name_end) = self.expect_name(IncorrectNamePosition::Constant)?; let documentation = self.take_documentation(name_start);
let annotation = self.parse_type_annotation(&Token::Colon)?;
let (eq_s, eq_e) = self.expect_one(&Token::Equal)?; match self.parse_const_value()? { Some(value) => { Ok(Some(Definition::ModuleConstant(ModuleConstant { documentation, location: SrcSpan { start,
// End after the type annotation if it's there, otherwise after the name end: annotation .as_ref() .map(|annotation| annotation.location().end) .unwrap_or(0) .max(name_end), }, publicity: self.publicity(public, attributes.internal)?, name, name_location: SrcSpan::new(name_start, name_end), annotation, value: Box::new(value), type_: (), deprecation: attributes.deprecated.clone(), implementations: Implementations { gleam: true, can_run_on_erlang: true, can_run_on_javascript: true, can_run_on_wasm: true, uses_erlang_externals: false, uses_javascript_externals: false, uses_wasm_externals: false, }, }))) } _ => parse_error( ParseErrorType::NoValueAfterEqual, SrcSpan { start: eq_s, end: eq_e, }, ), } }
// examples: // 1 // "hi" // True // [1,2,3] // wibble <> "wobble" fn parse_const_value(&mut self) -> Result<Option<UntypedConstant>, ParseError> { let constant_result = self.parse_const_value_unit(); match constant_result { Ok(Some(constant)) => self.parse_const_maybe_concatenation(constant), _ => constant_result, } }
fn parse_const_value_unit(&mut self) -> Result<Option<UntypedConstant>, ParseError> { match self.tok0.take() { Some((start, Token::Todo, end)) => { self.advance(); let message = self.maybe_parse_constant_as_message()?; let end = message .as_ref() .map_or(end, |message| message.location().end); Ok(Some(Constant::Todo { location: SrcSpan { start, end }, type_: (), message, })) }
Some((start, Token::String { value }, end)) => { self.advance(); Ok(Some(Constant::String { value, location: SrcSpan { start, end }, })) }
Some((start, Token::Float { value, float_value }, end)) => { self.advance(); Ok(Some(Constant::Float { value, location: SrcSpan { start, end }, float_value, })) }
Some((start, Token::Int { value, int_value }, end)) => { self.advance(); Ok(Some(Constant::Int { value, int_value, location: SrcSpan { start, end }, })) }
Some((start, Token::Hash, _)) => { self.advance(); let _ = self.expect_one(&Token::LeftParen)?; let elements = self.series_of(&Parser::parse_const_value, Some(&Token::Comma))?; let (_, end) = self.expect_one_following_series(&Token::RightParen, "a constant value")?; Ok(Some(Constant::Tuple { elements, location: SrcSpan { start, end }, type_: (), })) }
Some((start, Token::LeftSquare, _)) => { self.advance();
let (elements, elements_end_with_comma) = self.series_of_has_trailing_separator( &Parser::parse_const_value, Some(&Token::Comma), )?;
// Parse an optional tail let mut tail = None; let mut elements_after_tail = None; let mut dot_dot_location = None;
// If there are no elements, we still want to parse a tail so // that we can report a better error message. if (elements_end_with_comma || elements.is_empty()) && let Some((start, end)) = self.maybe_one(&Token::DotDot) { dot_dot_location = Some((start, end)); tail = self.parse_const_value()?.map(Box::new); if self.maybe_one(&Token::Comma).is_some() { // See if there's a list of items after the tail, // like `[..wibble, wobble, wabble]` let elements = self.series_of(&Parser::parse_const_value, Some(&Token::Comma)); match elements { Err(_) => {} Ok(elements) => { elements_after_tail = Some(elements); } } }
if tail.is_some() { // Give a better error when there are two lists being // concatenated like `[..wibble, ..wabble, woo]`, or if // there are elements after the tail. if let Some((second_start, second_end)) = self.maybe_one(&Token::DotDot) { let _second_tail = self.parse_const_value();
if elements_after_tail.is_none() || elements_after_tail .as_ref() .is_some_and(|vec| vec.is_empty()) { return parse_error( ParseErrorType::ListSpreadWithAnotherSpread { first_spread_location: SrcSpan { start, end }, }, SrcSpan { start: second_start, end: second_end, }, ); } } } }
let (_, end) = self.expect_one_following_series(&Token::RightSquare, "a constant value")?;
// Return errors for malformed lists match dot_dot_location { Some((start, end)) if tail.is_none() => { return parse_error( ParseErrorType::ListSpreadWithoutTail, SrcSpan { start, end }, ); } _ => {} } if tail.is_some() && elements.is_empty() && elements_after_tail.as_ref().is_none_or(|e| e.is_empty()) { return parse_error( ParseErrorType::ListSpreadWithoutElements, SrcSpan { start, end }, ); }
match elements_after_tail { Some(elements) if !elements.is_empty() => { let (start, end) = match (dot_dot_location, tail) { (Some((start, _)), Some(tail)) => (start, tail.location().end), (_, _) => (start, end), }; return parse_error( ParseErrorType::ListSpreadFollowedByElements, SrcSpan { start, end }, ); } _ => {} }
Ok(Some(Constant::List { elements, location: SrcSpan { start, end }, type_: (), tail, })) } // BitArray Some((start, Token::LtLt, _)) => { self.advance(); let segments = self.series_of( &|this| { this.parse_bit_array_segment( &Parser::parse_const_value, &Parser::expect_const_int, &bit_array_const_int, ) }, Some(&Token::Comma), )?; let (_, end) = self.expect_one_following_series(&Token::GtGt, "a bit array segment")?; Ok(Some(Constant::BitArray { location: SrcSpan { start, end }, segments, })) }
Some((start, Token::UpName { name }, end)) => { self.advance(); self.parse_const_record_finish(start, None, name, end) }
Some((start, Token::Name { name }, module_end)) if self.peek_tok1() == Some(&Token::Dot) => { self.advance(); // name self.advance(); // dot
match self.tok0.take() { Some((_, Token::UpName { name: upname }, end)) => { self.advance(); // upname self.parse_const_record_finish( start, Some((name, SrcSpan::new(start, module_end))), upname, end, ) } Some((_, Token::Name { name: end_name }, end)) => { self.advance(); // name
match self.tok0 { Some((_, Token::LeftParen, _)) => parse_error( ParseErrorType::UnexpectedFunction, SrcSpan { start, end: end + 1, }, ), _ => Ok(Some(Constant::Var { location: SrcSpan { start, end }, module: Some((name, SrcSpan::new(start, module_end))), name: end_name, constructor: None, type_: (), })), } } Some((start, token, end)) => parse_error( ParseErrorType::UnexpectedToken { token, expected: vec!["UpName".into(), "Name".into()], hint: None, }, SrcSpan { start, end }, ), None => { parse_error(ParseErrorType::UnexpectedEof, SrcSpan { start: 0, end: 0 }) } } }
Some((start, Token::Name { name }, end)) => { self.advance(); // name
match self.tok0 { Some((_, Token::LeftParen, _)) => parse_error( ParseErrorType::UnexpectedFunction, SrcSpan { start, end: end + 1, }, ), _ => Ok(Some(Constant::Var { location: SrcSpan { start, end }, module: None, name, constructor: None, type_: (), })), } }
// Helpful error for fn Some((start, Token::Fn, end)) => { parse_error(ParseErrorType::NotConstType, SrcSpan { start, end }) }
t0 => { self.tok0 = t0; Ok(None) } } }
fn parse_const_maybe_concatenation( &mut self, left: UntypedConstant, ) -> Result<Option<UntypedConstant>, ParseError> { match self.tok0.take() { Some((op_start, Token::Concatenate, op_end)) => { self.advance();
match self.parse_const_value() { Ok(Some(right_constant_value)) => Ok(Some(Constant::StringConcatenation { location: SrcSpan { start: left.location().start, end: right_constant_value.location().end, }, left: Box::new(left), right: Box::new(right_constant_value), })), _ => parse_error( ParseErrorType::OpNakedRight, SrcSpan { start: op_start, end: op_end, }, ), } } t0 => { self.tok0 = t0; Ok(Some(left)) } } }
// Parse the '( .. )' of a const type constructor fn parse_const_record_finish( &mut self, start: u32, module: Option<(EcoString, SrcSpan)>, name: EcoString, end: u32, ) -> Result<Option<UntypedConstant>, ParseError> { match self.maybe_one(&Token::LeftParen) { Some((par_s, _)) => { if let Some((dot_dot_start, _)) = self.maybe_one(&Token::DotDot) { let record = match self.parse_const_value()? { Some(value) => RecordBeingUpdated { location: SrcSpan::new(dot_dot_start, value.location().end), base: Box::new(value), }, None => { return parse_error( ParseErrorType::UnexpectedEof, SrcSpan::new(par_s, par_s + 2), ); } };
let mut update_arguments = vec![]; if self.maybe_one(&Token::Comma).is_some() { update_arguments = self.series_of( &Parser::parse_const_record_update_arg, Some(&Token::Comma), )?; }
let (_, par_e) = self.expect_one_following_series( &Token::RightParen, "a constant record update argument", )?;
let constructor_location = SrcSpan { start, end };
Ok(Some(Constant::RecordUpdate { location: SrcSpan { start, end: par_e }, constructor_location, module, name, record, arguments: update_arguments, type_: (), field_map: Inferred::Unknown, })) } else { let arguments = self.series_of(&Parser::parse_const_record_arg, Some(&Token::Comma))?;
let (_, par_e) = self.expect_one_following_series( &Token::RightParen, "a constant record argument", )?;
Ok(Some(Constant::Record { location: SrcSpan { start, end: par_e }, module, name, arguments: Some(arguments), type_: (), field_map: Inferred::Unknown, record_constructor: None, })) } } _ => Ok(Some(Constant::Record { location: SrcSpan { start, end }, module, name, arguments: None, type_: (), field_map: Inferred::Unknown, record_constructor: None, })), } }
// examples: // name: const // const // name: fn parse_const_record_arg(&mut self) -> Result<Option<CallArg<UntypedConstant>>, ParseError> { let label = match (self.tok0.take(), self.tok1.take()) { // Named arg (Some((start, Token::Name { name }, _)), Some((_, Token::Colon, end))) => { self.advance(); self.advance(); Some((start, name, end)) }
// Unnamed arg (t0, t1) => { self.tok0 = t0; self.tok1 = t1; None } };
match self.parse_const_value()? { Some(value) => match label { Some((start, label, _)) => Ok(Some(CallArg { implicit: None, location: SrcSpan { start, end: value.location().end, }, value, label: Some(label), })), _ => Ok(Some(CallArg { implicit: None, location: value.location(), value, label: None, })), }, _ => { match label { Some((start, label, end)) => { // Argument supplied with a label shorthand. Ok(Some(CallArg { implicit: None, location: SrcSpan { start, end }, label: Some(label.clone()), value: UntypedConstant::Var { location: SrcSpan { start, end }, constructor: None, module: None, name: label, type_: (), }, })) } _ => Ok(None), } } } }
fn parse_const_record_update_arg( &mut self, ) -> Result<Option<RecordUpdateArg<UntypedConstant>>, ParseError> { let (start, label, label_end) = match (self.tok0.take(), self.tok1.take()) { // Named arg - required for record updates (Some((start, Token::Name { name }, _)), Some((_, Token::Colon, end))) => { self.advance(); self.advance(); (start, name, end) }
// Unnamed arg or other - return error since record updates require labels (Some((start, Token::Name { name }, end)), t1) => { self.tok0 = Some((start, Token::Name { name: name.clone() }, end)); self.tok1 = t1;
// Check if this is label shorthand (name without colon) // In this case, use the name as both label and value match self.parse_const_value()? { Some(value) if value.location() == SrcSpan { start, end } => { return Ok(Some(RecordUpdateArg { label: name, location: SrcSpan { start, end }, value, })); } _ => { self.tok0 = Some((start, Token::Name { name }, end)); return parse_error(ParseErrorType::ExpectedName, SrcSpan { start, end }); } } }
(t0, t1) => { self.tok0 = t0; self.tok1 = t1; return Ok(None); } };
match self.parse_const_value()? { Some(value) => Ok(Some(RecordUpdateArg { label, location: SrcSpan { start, end: value.location().end, }, value, })), _ => { // Label shorthand: field without value means field: field Ok(Some(RecordUpdateArg { label: label.clone(), location: SrcSpan { start, end: label_end, }, value: UntypedConstant::Var { location: SrcSpan { start, end: label_end, }, constructor: None, module: None, name: label, type_: (), }, })) } } }
// // Bit String parsing //
// The structure is roughly the same for pattern, const, and expr // that's why these functions take functions // // pattern (: option)? fn parse_bit_array_segment<A>( &mut self, value_parser: &impl Fn(&mut Self) -> Result<Option<A>, ParseError>, arg_parser: &impl Fn(&mut Self) -> Result<A, ParseError>, to_int_segment: &impl Fn(EcoString, BigInt, u32, u32) -> A, ) -> Result<Option<BitArraySegment<A, ()>>, ParseError> where A: HasLocation + std::fmt::Debug, { match value_parser(self)? { Some(value) => { let options = if self.maybe_one(&Token::Colon).is_some() { self.series_of( &|this| this.parse_bit_array_option(&arg_parser, &to_int_segment), Some(&Token::Minus), )? } else { vec![] }; let end = options .last() .map(|option| option.location().end) .unwrap_or_else(|| value.location().end); Ok(Some(BitArraySegment { location: SrcSpan { start: value.location().start, end, }, value: Box::new(value), type_: (), options, })) } _ => Ok(None), } }
// examples: // 1 // size(1) // size(five) // utf8 fn parse_bit_array_option<A: std::fmt::Debug>( &mut self, arg_parser: &impl Fn(&mut Self) -> Result<A, ParseError>, to_int_segment: &impl Fn(EcoString, BigInt, u32, u32) -> A, ) -> Result<Option<BitArrayOption<A>>, ParseError> { match self.tok0.take() { // named segment Some((start, Token::Name { name }, end)) => { self.advance(); if self.maybe_one(&Token::LeftParen).is_some() { // named function segment match name.as_str() { "unit" => match self.tok0.take() { Some((int_s, Token::Int { value, .. }, int_e)) => { self.advance(); let (_, end) = self.expect_one(&Token::RightParen)?; let v = value.replace("_", ""); match u8::from_str(&v) { Ok(units) if units > 0 => Ok(Some(BitArrayOption::Unit { location: SrcSpan { start, end }, value: units, })),
_ => Err(ParseError { error: ParseErrorType::InvalidBitArrayUnit, location: SrcSpan { start: int_s, end: int_e, }, }), } } tok0 => { self.tok0 = tok0; self.next_tok_unexpected(vec!["A positive int".into()]) } },
"size" => { let value = arg_parser(self)?; let (_, end) = self.expect_one(&Token::RightParen)?; Ok(Some(BitArrayOption::Size { location: SrcSpan { start, end }, value: Box::new(value), short_form: false, })) } _ => parse_error( ParseErrorType::InvalidBitArraySegment, SrcSpan { start, end }, ), } } else { str_to_bit_array_option(&name, SrcSpan { start, end }) .ok_or(ParseError { error: ParseErrorType::InvalidBitArraySegment, location: SrcSpan { start, end }, }) .map(Some) } } // int segment Some((start, Token::Int { value, int_value }, end)) => { self.advance(); Ok(Some(BitArrayOption::Size { location: SrcSpan { start, end }, value: Box::new(to_int_segment(value, int_value, start, end)), short_form: true, })) } // invalid tok0 => { self.tok0 = tok0; self.next_tok_unexpected(vec![ "A valid bit array segment type".into(), "See: https://tour.gleam.run/data-types/bit-arrays/".into(), ]) } } }
fn expect_bit_array_pattern_segment_arg(&mut self) -> Result<UntypedPattern, ParseError> { Ok(Pattern::BitArraySize(self.expect_bit_array_size()?)) }
fn expect_bit_array_size(&mut self) -> Result<BitArraySize<()>, ParseError> { let mut opstack = vec![]; let mut estack: Vec<BitArraySize<()>> = vec![];
estack.push(self.parse_bit_array_size_unit()?);
loop { let Some((op_s, token, op_e)) = self.tok0.take() else { break; }; let Some(prec) = token_to_bit_array_size_operator(&token).map(|op| op.precedence()) else { self.tok0 = Some((op_s, token, op_e)); break; };
self.advance(); let _ = handle_op( Some(((op_s, token, op_e), prec)), &mut opstack, &mut estack, &reduce_bit_array_size, );
estack.push(self.parse_bit_array_size_unit()?); }
Ok( handle_op(None, &mut opstack, &mut estack, &reduce_bit_array_size) .expect("bit array size expression stack should not be empty"), ) }
fn parse_bit_array_size_unit(&mut self) -> Result<BitArraySize<()>, ParseError> { match self.tok0.take() { Some((start, Token::Name { name }, end)) => { self.advance(); Ok(BitArraySize::Variable { location: SrcSpan { start, end }, name, constructor: None, type_: (), }) } Some((start, Token::Int { value, int_value }, end)) => { self.advance(); Ok(BitArraySize::Int { location: SrcSpan { start, end }, value, int_value, }) } Some((start, Token::LeftBrace, _)) => { self.advance(); let inner = self.expect_bit_array_size()?; let (_, end) = self.expect_one(&Token::RightBrace)?;
Ok(BitArraySize::Block { location: SrcSpan { start, end }, inner: Box::new(inner), }) } tok0 => { self.tok0 = tok0; self.next_tok_unexpected(vec!["A variable name or an int".into()]) } } }
fn expect_const_int(&mut self) -> Result<UntypedConstant, ParseError> { match self.tok0.take() { Some((start, Token::Int { value, int_value }, end)) => { self.advance(); Ok(Constant::Int { location: SrcSpan { start, end }, value, int_value, }) } tok0 => { self.tok0 = tok0; self.next_tok_unexpected(vec!["An int".into()]) } } }
fn expect_expression(&mut self) -> Result<UntypedExpr, ParseError> { match self.parse_expression()? { Some(e) => Ok(e), _ => self.next_tok_unexpected(vec!["An expression".into()]), } }
fn expect_expression_unit( &mut self, context: ExpressionUnitContext, ) -> Result<UntypedExpr, ParseError> { if let Some(e) = self.parse_expression_unit(context)? { Ok(e) } else { self.next_tok_unexpected(vec!["An expression".into()]) } }
// // Parse Helpers //
/// Expect a particular token, advances the token stream fn expect_one(&mut self, wanted: &Token) -> Result<(u32, u32), ParseError> { match self.maybe_one(wanted) { Some((start, end)) => Ok((start, end)), None => self.next_tok_unexpected(vec![wanted.to_string().into()]), } }
// Expect a particular token after having parsed a series, advances the token stream // Used for giving a clearer error message in cases where the series item is what failed to parse fn expect_one_following_series( &mut self, wanted: &Token, series: &'static str, ) -> Result<(u32, u32), ParseError> { match self.maybe_one(wanted) { Some((start, end)) => Ok((start, end)), None => self.next_tok_unexpected(vec![wanted.to_string().into(), series.into()]), } }
/// Expect the end to a custom type definiton or handle an incorrect /// record constructor definition. fn expect_custom_type_close(&mut self) -> Result<u32, ParseError> { match self.maybe_one(&Token::RightBrace) { Some((_, end)) => Ok(end), None => match self.next_tok() { None => parse_error(ParseErrorType::UnexpectedEof, SrcSpan { start: 0, end: 0 }), Some((start, token, end)) => { let hint = match (&token, self.tok0.take()) { (&Token::Fn, _) | (&Token::Pub, Some((_, Token::Fn, _))) => { let text = "Gleam is not an object oriented programming language sofunctions are declared separately from types."; Some(wrap(text).into()) } (_, _) => None, };
parse_error( ParseErrorType::UnexpectedToken { token, expected: vec![ Token::RightBrace.to_string().into(), "a record constructor".into(), ], hint, }, SrcSpan { start, end }, ) } }, } }
// Expect a Name else a token dependent helpful error fn expect_name( &mut self, kind: IncorrectNamePosition, ) -> Result<(u32, EcoString, u32), ParseError> { let (start, token, end) = self.expect_assign_name()?; match token { AssignName::Variable(name) => Ok((start, name, end)), AssignName::Discard(_) => parse_error( ParseErrorType::IncorrectName { kind }, SrcSpan { start, end }, ), } }
fn expect_assign_name(&mut self) -> Result<(u32, AssignName, u32), ParseError> { let t = self.next_tok(); match t { Some((start, tok, end)) => match tok { Token::Name { name } => Ok((start, AssignName::Variable(name), end)), Token::DiscardName { name, .. } => Ok((start, AssignName::Discard(name), end)), Token::UpName { .. } => parse_error( ParseErrorType::IncorrectName { kind: IncorrectNamePosition::Variable, }, SrcSpan { start, end }, ), _ if tok.is_reserved_word() => parse_error( ParseErrorType::UnexpectedReservedWord, SrcSpan { start, end }, ), Token::Int { .. } | Token::Float { .. } | Token::String { .. } | Token::CommentDoc { .. } | Token::LeftParen | Token::RightParen | Token::LeftSquare | Token::RightSquare | Token::LeftBrace | Token::RightBrace | Token::Plus | Token::Minus | Token::Star | Token::Slash | Token::Less | Token::Greater | Token::LessEqual | Token::GreaterEqual | Token::Percent | Token::PlusDot | Token::MinusDot | Token::StarDot | Token::SlashDot | Token::LessDot | Token::GreaterDot | Token::LessEqualDot | Token::GreaterEqualDot | Token::Concatenate | Token::Colon | Token::Comma | Token::Hash | Token::Bang | Token::Equal | Token::EqualEqual | Token::NotEqual | Token::Vbar | Token::VbarVbar | Token::AmperAmper | Token::LtLt | Token::GtGt | Token::Pipe | Token::Dot | Token::RArrow | Token::LArrow | Token::DotDot | Token::At | Token::EndOfFile | Token::CommentNormal | Token::CommentModule | Token::NewLine | Token::As | Token::Assert | Token::Auto | Token::Case | Token::Const | Token::Delegate | Token::Derive | Token::Echo | Token::Else | Token::Fn | Token::If | Token::Implement | Token::Import | Token::Let | Token::Macro | Token::Opaque | Token::Panic | Token::Pub | Token::Test | Token::Todo | Token::Type | Token::Use => parse_error(ParseErrorType::ExpectedName, SrcSpan { start, end }), }, None => parse_error(ParseErrorType::UnexpectedEof, SrcSpan { start: 0, end: 0 }), } }
// Expect an UpName else a token dependent helpful error fn expect_upname(&mut self) -> Result<(u32, EcoString, u32), ParseError> { let t = self.next_tok(); match t { Some((start, tok, end)) => match tok { Token::Name { .. } | Token::DiscardName { .. } => { parse_error(ParseErrorType::IncorrectUpName, SrcSpan { start, end }) } Token::UpName { name } => Ok((start, name, end)), Token::Int { .. } | Token::Float { .. } | Token::String { .. } | Token::CommentDoc { .. } | Token::LeftParen | Token::RightParen | Token::LeftSquare | Token::RightSquare | Token::LeftBrace | Token::RightBrace | Token::Plus | Token::Minus | Token::Star | Token::Slash | Token::Less | Token::Greater | Token::LessEqual | Token::GreaterEqual | Token::Percent | Token::PlusDot | Token::MinusDot | Token::StarDot | Token::SlashDot | Token::LessDot | Token::GreaterDot | Token::LessEqualDot | Token::GreaterEqualDot | Token::Concatenate | Token::Colon | Token::Comma | Token::Hash | Token::Bang | Token::Equal | Token::EqualEqual | Token::NotEqual | Token::Vbar | Token::VbarVbar | Token::AmperAmper | Token::LtLt | Token::GtGt | Token::Pipe | Token::Dot | Token::RArrow | Token::LArrow | Token::DotDot | Token::At | Token::EndOfFile | Token::CommentNormal | Token::CommentModule | Token::NewLine | Token::As | Token::Assert | Token::Auto | Token::Case | Token::Const | Token::Delegate | Token::Derive | Token::Echo | Token::Else | Token::Fn | Token::If | Token::Implement | Token::Import | Token::Let | Token::Macro | Token::Opaque | Token::Panic | Token::Pub | Token::Test | Token::Todo | Token::Type | Token::Use => parse_error(ParseErrorType::ExpectedUpName, SrcSpan { start, end }), }, None => parse_error(ParseErrorType::UnexpectedEof, SrcSpan { start: 0, end: 0 }), } }
// Expect a target name. e.g. `javascript` or `erlang`. // The location of the preceding left parenthesis is required // to give the correct error span in case the target name is missing. fn expect_target(&mut self, paren_location: SrcSpan) -> Result<Target, ParseError> { let (start, t, end) = match self.next_tok() { Some(t) => t, None => { return parse_error(ParseErrorType::UnexpectedEof, SrcSpan { start: 0, end: 0 }); } }; if let Token::Name { name } = t { match name.as_str() { "javascript" => Ok(Target::JavaScript), "erlang" => Ok(Target::Erlang), "wasm" => Ok(Target::Wasm), "js" => { self.warnings .push(DeprecatedSyntaxWarning::DeprecatedTargetShorthand { location: SrcSpan::new(start, end), target: Target::JavaScript, }); Ok(Target::JavaScript) } "erl" => { self.warnings .push(DeprecatedSyntaxWarning::DeprecatedTargetShorthand { location: SrcSpan::new(start, end), target: Target::Erlang, }); Ok(Target::Erlang) } _ => parse_error(ParseErrorType::UnknownTarget, SrcSpan::new(start, end)), } } else { parse_error(ParseErrorType::ExpectedTargetName, paren_location) } }
// Expect a String else error fn expect_string(&mut self) -> Result<(u32, EcoString, u32), ParseError> { match self.tok0.take() { Some((start, Token::String { value }, end)) => { self.advance(); Ok((start, value, end)) } tok0 => { self.tok0 = tok0; self.next_tok_unexpected(vec!["a string".into()]) } } }
fn peek_tok1(&mut self) -> Option<&Token> { self.tok1.as_ref().map(|(_, token, _)| token) }
// If the next token matches the requested, consume it and return (start, end) fn maybe_one(&mut self, tok: &Token) -> Option<(u32, u32)> { match self.tok0.take() { Some((s, t, e)) if t == *tok => { self.advance(); Some((s, e)) }
t0 => { self.tok0 = t0; None } } }
// Parse a series by repeating a parser, and possibly a separator fn series_of<A>( &mut self, parser: &impl Fn(&mut Self) -> Result<Option<A>, ParseError>, sep: Option<&Token>, ) -> Result<Vec<A>, ParseError> { let (res, _) = self.series_of_has_trailing_separator(parser, sep)?; Ok(res) }
/// Parse a series by repeating a parser, and a separator. Returns true if /// the series ends with the trailing separator. fn series_of_has_trailing_separator<A>( &mut self, parser: &impl Fn(&mut Self) -> Result<Option<A>, ParseError>, sep: Option<&Token>, ) -> Result<(Vec<A>, bool), ParseError> { let mut results = vec![]; let mut final_separator = None; while let Some(result) = parser(self)? { results.push(result); if let Some(sep) = sep { if let Some(separator) = self.maybe_one(sep) { final_separator = Some(separator); } else { final_separator = None; break; }
// Helpful error if extra separator if let Some((start, end)) = self.maybe_one(sep) { return parse_error(ParseErrorType::ExtraSeparator, SrcSpan { start, end }); } } }
// If the sequence ends with a trailing comma we want to keep track of // its position. if let (Some(Token::Comma), Some((_, end))) = (sep, final_separator) { self.extra.trailing_commas.push(end); }
Ok((results, final_separator.is_some())) }
// If next token is a Name, consume it and return relevant info, otherwise, return none fn maybe_name(&mut self) -> Option<(u32, EcoString, u32)> { match self.tok0.take() { Some((s, Token::Name { name }, e)) => { self.advance(); Some((s, name, e)) } t0 => { self.tok0 = t0; None } } }
// if next token is an UpName, consume it and return relevant info, otherwise, return none fn maybe_upname(&mut self) -> Option<(u32, EcoString, u32)> { match self.tok0.take() { Some((s, Token::UpName { name }, e)) => { self.advance(); Some((s, name, e)) } t0 => { self.tok0 = t0; None } } }
// if next token is a DiscardName, consume it and return relevant info, otherwise, return none fn maybe_discard_name(&mut self) -> Option<(u32, EcoString, u32)> { match self.tok0.take() { Some((s, Token::DiscardName { name }, e)) => { self.advance(); Some((s, name, e)) } t0 => { self.tok0 = t0; None } } }
// Unexpected token error on the next token or EOF fn next_tok_unexpected<A>(&mut self, expected: Vec<EcoString>) -> Result<A, ParseError> { match self.next_tok() { None => parse_error(ParseErrorType::UnexpectedEof, SrcSpan { start: 0, end: 0 }), Some((start, token, end)) => parse_error( ParseErrorType::UnexpectedToken { token, expected, hint: None, }, SrcSpan { start, end }, ), } }
// Moves the token stream forward fn advance(&mut self) { let _ = self.next_tok(); }
// Moving the token stream forward // returns old tok0 fn next_tok(&mut self) -> Option<Spanned> { let t = self.tok0.take(); let mut previous_newline = None; let mut nxt; loop { match self.tokens.next() { // gather and skip extra Some(Ok((start, Token::CommentNormal, end))) => { self.extra.comments.push(SrcSpan { start, end }); previous_newline = None; } Some(Ok((start, Token::CommentDoc { content }, end))) => { self.extra.doc_comments.push(SrcSpan::new(start, end)); self.doc_comments.push_back((start, content)); previous_newline = None; } Some(Ok((start, Token::CommentModule, end))) => { self.extra.module_comments.push(SrcSpan { start, end }); previous_newline = None; } Some(Ok((start, Token::NewLine, _))) => { self.extra.new_lines.push(start); // If the previous token is a newline as well that means we // have run into an empty line. if let Some(start) = previous_newline { // We increase the byte position so that newline's start // doesn't overlap with the previous token's end. self.extra.empty_lines.push(start + 1); } previous_newline = Some(start); }
// die on lex error Some(Err(err)) => { nxt = None; self.lex_errors.push(err); break; }
Some(Ok(tok)) => { nxt = Some(tok); break; } None => { nxt = None; break; } } } self.tok0 = self.tok1.take(); self.tok1 = nxt.take(); t }
fn take_documentation(&mut self, until: u32) -> Option<(u32, EcoString)> { let mut content = String::new(); let mut doc_start = u32::MAX; while let Some((start, line)) = self.doc_comments.front() { if *start < doc_start { doc_start = *start; } if *start >= until { break; }
if self.extra.has_comment_between(*start, until) { // We ignore doc comments that come before a regular comment. let location = SrcSpan::new(*start, start + line.len() as u32); _ = self.doc_comments.pop_front(); self.detached_doc_comments.push(location); continue; }
content.push_str(line); content.push('\n'); _ = self.doc_comments.pop_front(); } if content.is_empty() { None } else { Some((doc_start, content.into())) } }
fn parse_attributes( &mut self, attributes: &mut Attributes, ) -> Result<Option<SrcSpan>, ParseError> { let mut attributes_span = None;
while let Some((start, end)) = self.maybe_one(&Token::At) { if attributes_span.is_none() { attributes_span = Some(SrcSpan { start, end }); }
let end = self.parse_attribute(start, attributes)?; attributes_span = attributes_span.map(|span| SrcSpan { start: span.start, end, }); }
Ok(attributes_span) }
fn parse_attribute( &mut self, start: u32, attributes: &mut Attributes, ) -> Result<u32, ParseError> { // Parse the name of the attribute.
let (_, name, end) = self.expect_name(IncorrectNamePosition::Attribute)?;
let end = match name.as_str() { "external" => { let _ = self.maybe_one(&Token::LeftParen).ok_or(ParseError { error: ParseErrorType::ExpectedExternalArguments, location: SrcSpan { start, end }, })?; self.parse_external_attribute(start, end, attributes) } "target" => self.parse_target_attribute(start, end, attributes), "deprecated" => self.parse_deprecated_attribute(start, end, attributes), "internal" => self.parse_internal_attribute(start, end, attributes), _ => parse_error(ParseErrorType::UnknownAttribute, SrcSpan { start, end }), }?;
Ok(end) }
fn parse_target_attribute( &mut self, start: u32, end: u32, attributes: &mut Attributes, ) -> Result<u32, ParseError> { let (paren_start, paren_end) = self.expect_one(&Token::LeftParen)?; let target = self.expect_target(SrcSpan::new(paren_start, paren_end))?; if attributes.target.is_some() { return parse_error(ParseErrorType::DuplicateAttribute, SrcSpan { start, end }); } let (_, end) = self.expect_one(&Token::RightParen)?; if attributes.target.is_some() { return parse_error(ParseErrorType::DuplicateAttribute, SrcSpan { start, end }); } attributes.target = Some(target); Ok(end) }
fn parse_external_attribute( &mut self, start: u32, end: u32, attributes: &mut Attributes, ) -> Result<u32, ParseError> { let (_, target, _) = self.expect_name(IncorrectNamePosition::Target)?;
let target = match target.as_str() { "erlang" => Target::Erlang, "javascript" => Target::JavaScript, "wasm" => Target::Wasm, _ => return parse_error(ParseErrorType::UnknownTarget, SrcSpan::new(start, end)), };
let _ = self.expect_one(&Token::Comma)?; let (_, module, _) = self.expect_string()?; let _ = self.expect_one(&Token::Comma)?; let (_, function, _) = self.expect_string()?; let _ = self.maybe_one(&Token::Comma); let (_, end) = self.expect_one(&Token::RightParen)?;
if attributes.has_external_for(target) { return parse_error(ParseErrorType::DuplicateAttribute, SrcSpan { start, end }); }
attributes.set_external_for(target, Some((module, function, SrcSpan { start, end }))); Ok(end) }
fn parse_deprecated_attribute( &mut self, start: u32, end: u32, attributes: &mut Attributes, ) -> Result<u32, ParseError> { let _ = self.expect_one(&Token::LeftParen).map_err(|_| ParseError { error: ParseErrorType::ExpectedDeprecationMessage, location: SrcSpan { start, end }, })?; if attributes.deprecated.is_deprecated() { return parse_error(ParseErrorType::DuplicateAttribute, SrcSpan::new(start, end)); } let (_, message, _) = self.expect_string().map_err(|_| ParseError { error: ParseErrorType::ExpectedDeprecationMessage, location: SrcSpan { start, end }, })?; let (_, end) = self.expect_one(&Token::RightParen)?; attributes.deprecated = Deprecation::Deprecated { message }; Ok(end) }
fn parse_internal_attribute( &mut self, start: u32, end: u32, attributes: &mut Attributes, ) -> Result<u32, ParseError> { match attributes.internal { // If `internal` is present that means that we have already run into // another `@internal` annotation, so it results in a `DuplicateAttribute` // error. InternalAttribute::Present(_) => { parse_error(ParseErrorType::DuplicateAttribute, SrcSpan::new(start, end)) } InternalAttribute::Missing => { attributes.internal = InternalAttribute::Present(SrcSpan::new(start, end)); Ok(end) } } }}
fn concat_pattern_variable_left_hand_side_error<T>(start: u32, end: u32) -> Result<T, ParseError> { Err(ParseError { error: ParseErrorType::ConcatPatternVariableLeftHandSide, location: SrcSpan::new(start, end), })}
fn concat_pattern_variable_with_suffix<T>( start: u32, name: EcoString, end: u32,) -> Result<T, ParseError> { Err(ParseError { error: ParseErrorType::ConcatPatternVariableWithSuffix { name }, location: SrcSpan::new(start, end), })}
// Operator Precedence Parsing//// Higher number means higher precedence.// All operators are left associative.
/// Simple-Precedence-Parser, handle seeing an operator or endfn handle_op<A>( next_op: Option<(Spanned, u8)>, opstack: &mut Vec<(Spanned, u8)>, estack: &mut Vec<A>, do_reduce: &impl Fn(Spanned, &mut Vec<A>),) -> Option<A> { let mut next_op = next_op; loop { match (opstack.pop(), next_op.take()) { (None, None) => match estack.pop() { Some(fin) => { if estack.is_empty() { return Some(fin); } else { panic!("Expression not fully reduced.") } } _ => { return None; } },
(None, Some(op)) => { opstack.push(op); break; }
(Some((op, _)), None) => do_reduce(op, estack),
(Some((opl, pl)), Some((opr, pr))) => { match pl.cmp(&pr) { // all ops are left associative Ordering::Greater | Ordering::Equal => { do_reduce(opl, estack); next_op = Some((opr, pr)); } Ordering::Less => { opstack.push((opl, pl)); opstack.push((opr, pr)); break; } } } } } None}
fn precedence(t: &Token) -> Option<u8> { if t == &Token::Pipe { return Some(6); } tok_to_binop(t).map(|op| op.precedence())}
fn tok_to_binop(t: &Token) -> Option<BinOp> { match t { Token::VbarVbar => Some(BinOp::Or), Token::AmperAmper => Some(BinOp::And), Token::EqualEqual => Some(BinOp::Eq), Token::NotEqual => Some(BinOp::NotEq), Token::Less => Some(BinOp::LtInt), Token::LessEqual => Some(BinOp::LtEqInt), Token::Greater => Some(BinOp::GtInt), Token::GreaterEqual => Some(BinOp::GtEqInt), Token::LessDot => Some(BinOp::LtFloat), Token::LessEqualDot => Some(BinOp::LtEqFloat), Token::GreaterDot => Some(BinOp::GtFloat), Token::GreaterEqualDot => Some(BinOp::GtEqFloat), Token::Plus => Some(BinOp::AddInt), Token::Minus => Some(BinOp::SubInt), Token::PlusDot => Some(BinOp::AddFloat), Token::MinusDot => Some(BinOp::SubFloat), Token::Percent => Some(BinOp::RemainderInt), Token::Star => Some(BinOp::MultInt), Token::StarDot => Some(BinOp::MultFloat), Token::Slash => Some(BinOp::DivInt), Token::SlashDot => Some(BinOp::DivFloat), Token::Concatenate => Some(BinOp::Concatenate), Token::Name { .. } | Token::UpName { .. } | Token::DiscardName { .. } | Token::Int { .. } | Token::Float { .. } | Token::String { .. } | Token::CommentDoc { .. } | Token::LeftParen | Token::RightParen | Token::LeftSquare | Token::RightSquare | Token::LeftBrace | Token::RightBrace | Token::Colon | Token::Comma | Token::Hash | Token::Bang | Token::Equal | Token::Vbar | Token::LtLt | Token::GtGt | Token::Pipe | Token::Dot | Token::RArrow | Token::LArrow | Token::DotDot | Token::At | Token::EndOfFile | Token::CommentNormal | Token::CommentModule | Token::NewLine | Token::As | Token::Assert | Token::Auto | Token::Case | Token::Const | Token::Delegate | Token::Derive | Token::Echo | Token::Else | Token::Fn | Token::If | Token::Implement | Token::Import | Token::Let | Token::Macro | Token::Opaque | Token::Panic | Token::Pub | Token::Test | Token::Todo | Token::Type | Token::Use => None, }}
fn token_to_bit_array_size_operator(t: &Token) -> Option<IntOperator> { match t { Token::Plus => Some(IntOperator::Add), Token::Minus => Some(IntOperator::Subtract), Token::Star => Some(IntOperator::Multiply), Token::Slash => Some(IntOperator::Divide), Token::Percent => Some(IntOperator::Remainder), Token::Name { .. } | Token::UpName { .. } | Token::DiscardName { .. } | Token::Int { .. } | Token::Float { .. } | Token::String { .. } | Token::CommentDoc { .. } | Token::LeftParen | Token::RightParen | Token::LeftSquare | Token::RightSquare | Token::LeftBrace | Token::RightBrace | Token::Less | Token::Greater | Token::LessEqual | Token::GreaterEqual | Token::PlusDot | Token::MinusDot | Token::StarDot | Token::SlashDot | Token::LessDot | Token::GreaterDot | Token::LessEqualDot | Token::GreaterEqualDot | Token::Concatenate | Token::Colon | Token::Comma | Token::Hash | Token::Bang | Token::Equal | Token::EqualEqual | Token::NotEqual | Token::Vbar | Token::VbarVbar | Token::AmperAmper | Token::LtLt | Token::GtGt | Token::Pipe | Token::Dot | Token::RArrow | Token::LArrow | Token::DotDot | Token::At | Token::EndOfFile | Token::CommentNormal | Token::CommentModule | Token::NewLine | Token::As | Token::Assert | Token::Auto | Token::Case | Token::Const | Token::Delegate | Token::Derive | Token::Echo | Token::Else | Token::Fn | Token::If | Token::Implement | Token::Import | Token::Let | Token::Macro | Token::Opaque | Token::Panic | Token::Pub | Token::Test | Token::Todo | Token::Type | Token::Use => None, }}
/// Simple-Precedence-Parser, perform reduction for expressionfn do_reduce_expression(op: Spanned, estack: &mut Vec<UntypedExpr>) { match (estack.pop(), estack.pop()) { (Some(er), Some(el)) => { let new_e = expr_op_reduction(op, el, er); estack.push(new_e); } _ => panic!("Tried to reduce without 2 expressions"), }}
/// Simple-Precedence-Parser, perform reduction for clause guardfn do_reduce_clause_guard(op: Spanned, estack: &mut Vec<UntypedClauseGuard>) { match (estack.pop(), estack.pop()) { (Some(er), Some(el)) => { let new_e = clause_guard_reduction(op, el, er); estack.push(new_e); } _ => panic!("Tried to reduce without 2 guards"), }}
/// Simple-Precedence-Parser, perform reduction for bit array size expressionsfn reduce_bit_array_size((_, token, _): Spanned, estack: &mut Vec<BitArraySize<()>>) { let operator = token_to_bit_array_size_operator(&token) .expect("only operator tokens are pushed onto the bit array size opstack"); match (estack.pop(), estack.pop()) { (Some(right), Some(left)) => { let location = SrcSpan { start: left.location().start, end: right.location().end, }; estack.push(BitArraySize::BinaryOperator { left: Box::new(left), right: Box::new(right), operator, location, }); } _ => panic!("Tried to reduce bit array size without 2 operands"), }}
fn expr_op_reduction( (token_start, token, _token_end): Spanned, left: UntypedExpr, right: UntypedExpr,) -> UntypedExpr { if token == Token::Pipe { let expressions = if let UntypedExpr::PipeLine { mut expressions } = left { expressions.push(right); expressions } else { vec1![left, right] }; UntypedExpr::PipeLine { expressions } } else { match tok_to_binop(&token) { Some(operator) => UntypedExpr::BinOp { location: SrcSpan { start: left.location().start, end: right.location().end, }, operator, operator_start: token_start, left: Box::new(left), right: Box::new(right), }, _ => { panic!("Token could not be converted to binop.") } } }}
fn clause_guard_reduction( (start, token, _end): Spanned, left: UntypedClauseGuard, right: UntypedClauseGuard,) -> UntypedClauseGuard { let location = SrcSpan { start: left.location().start, end: right.location().end, }; let left = Box::new(left); let right = Box::new(right); let operator = tok_to_binop(&token).expect("Token could not be converted to binop."); UntypedClauseGuard::BinaryOperator { location, operator, operator_start: start, left, right, }}
// BitArray Parse Helpers//// BitArrays in patterns, guards, and expressions have a very similar structure// but need specific types. These are helpers for that. There is probably a// rustier way to do this :)fn bit_array_size_int(value: EcoString, int_value: BigInt, start: u32, end: u32) -> UntypedPattern { Pattern::BitArraySize(BitArraySize::Int { location: SrcSpan { start, end }, value, int_value, })}
fn bit_array_expr_int(value: EcoString, int_value: BigInt, start: u32, end: u32) -> UntypedExpr { UntypedExpr::Int { location: SrcSpan { start, end }, value, int_value, }}
fn bit_array_const_int( value: EcoString, int_value: BigInt, start: u32, end: u32,) -> UntypedConstant { Constant::Int { location: SrcSpan { start, end }, value, int_value, }}
fn str_to_bit_array_option<A>(lit: &str, location: SrcSpan) -> Option<BitArrayOption<A>> { match lit { "bytes" => Some(BitArrayOption::Bytes { location }), "int" => Some(BitArrayOption::Int { location }), "float" => Some(BitArrayOption::Float { location }), "bits" => Some(BitArrayOption::Bits { location }), "utf8" => Some(BitArrayOption::Utf8 { location }), "utf16" => Some(BitArrayOption::Utf16 { location }), "utf32" => Some(BitArrayOption::Utf32 { location }), "utf8_codepoint" => Some(BitArrayOption::Utf8Codepoint { location }), "utf16_codepoint" => Some(BitArrayOption::Utf16Codepoint { location }), "utf32_codepoint" => Some(BitArrayOption::Utf32Codepoint { location }), "signed" => Some(BitArrayOption::Signed { location }), "unsigned" => Some(BitArrayOption::Unsigned { location }), "big" => Some(BitArrayOption::Big { location }), "little" => Some(BitArrayOption::Little { location }), "native" => Some(BitArrayOption::Native { location }), _ => None, }}
//// Error Helpers//fn parse_error<T>(error: ParseErrorType, location: SrcSpan) -> Result<T, ParseError> { Err(ParseError { error, location })}
//// Misc Helpers//
// Parsing a function call into the appropriate structure#[derive(Debug)]pub enum ParserArg { Arg(Box<CallArg<UntypedExpr>>), Hole { name: EcoString, /// The whole span of the argument. arg_location: SrcSpan, /// Just the span of the ignore name. discard_location: SrcSpan, label: Option<EcoString>, },}
pub fn make_call( fun: UntypedExpr, arguments: Vec<ParserArg>, start: u32, end: u32, open_parenthesis: u32,) -> Result<UntypedExpr, ParseError> { let mut hole_location = None;
let arguments = arguments .into_iter() .map(|argument| match argument { ParserArg::Arg(arg) => Ok(*arg), ParserArg::Hole { arg_location, discard_location, name, label, } => { if hole_location.is_some() { return parse_error(ParseErrorType::TooManyArgHoles, SrcSpan { start, end }); }
hole_location = Some(discard_location); if name != "_" { return parse_error( ParseErrorType::UnexpectedToken { token: Token::Name { name }, expected: vec!["An expression".into(), "An underscore".into()], hint: None, }, arg_location, ); }
Ok(CallArg { implicit: None, label, location: arg_location, value: UntypedExpr::Var { location: discard_location, name: CAPTURE_VARIABLE.into(), }, }) } }) .collect::<Result<_, _>>()?;
let call = UntypedExpr::Call { location: SrcSpan { start, end }, fun: Box::new(fun), arguments, open_parenthesis, };
match hole_location { // A normal call None => Ok(call),
// An anon function using the capture syntax run(_, 1, 2) Some(hole_location) => Ok(UntypedExpr::Fn { location: call.location(), end_of_head_byte_index: call.location().end, kind: FunctionLiteralKind::Capture { hole: hole_location, }, arguments: vec![Arg { location: hole_location, annotation: None, names: ArgNames::Named { name: CAPTURE_VARIABLE.into(), location: hole_location, }, type_: (), }], body: vec1![Statement::Expression(call)], return_annotation: None, }), }}
#[derive(Debug, Default)]struct ParsedUnqualifiedImports { types: Vec<UnqualifiedImport>, values: Vec<UnqualifiedImport>,}
/// Parses an Int value to a bigint.///pub fn parse_int_value(value: &str) -> Option<BigInt> { let (radix, value) = if let Some(value) = value.strip_prefix("0x") { (16, value) } else if let Some(value) = value.strip_prefix("0o") { (8, value) } else if let Some(value) = value.strip_prefix("0b") { (2, value) } else { (10, value) };
let value = value.trim_start_matches('_');
BigInt::parse_bytes(value.as_bytes(), radix)}
#[derive(Debug, PartialEq, Clone, Copy)]enum ExpressionUnitContext { FollowingPipe, Other,}
#[derive(Debug, Clone, Copy)]pub enum PatternPosition { LetAssignment, CaseClause, UsePattern,}
impl PatternPosition { pub fn to_declaration(&self) -> VariableDeclaration { match self { PatternPosition::LetAssignment => VariableDeclaration::LetPattern, PatternPosition::CaseClause => VariableDeclaration::ClausePattern, PatternPosition::UsePattern => VariableDeclaration::UsePattern, } }}
/// A thin f64 wrapper that does not permit NaN./// This allows us to implement `Eq`, which require reflexivity.////// Used for gleam float literals, which cannot be NaN.////// While there is no syntax for "infinity", float literals might be too big and/// overflow into infinity. This is still allowed so we can parse big literal/// numbers and the error will be raised during the analysis phase.#[derive(Clone, Copy, Debug, PartialEq)]pub struct LiteralFloatValue(f64);
impl LiteralFloatValue { pub const ONE: Self = LiteralFloatValue(1.0); pub const ZERO: Self = LiteralFloatValue(0.0);
/// Parse from a string, returning `None` if the string /// is not a valid f64 or the float is `NaN`` pub fn parse(value: &str) -> Option<Self> { value .replace("_", "") .parse::<f64>() .ok() .filter(|float| !float.is_nan()) .map(LiteralFloatValue) }
pub fn value(&self) -> f64 { self.0 }}
impl Eq for LiteralFloatValue {}
impl Ord for LiteralFloatValue { fn cmp(&self, other: &Self) -> Ordering { self.0 .partial_cmp(&other.0) .expect("Only NaN comparisons should fail") }}
impl PartialOrd for LiteralFloatValue { fn partial_cmp(&self, other: &Self) -> Option<Ordering> { Some(self.cmp(other)) }}
impl Hash for LiteralFloatValue { fn hash<H: Hasher>(&self, state: &mut H) { self.0.to_bits().hash(state); }}
impl Serialize for LiteralFloatValue { fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: serde::Serializer, { serializer.serialize_f64(self.0) }}
impl<'de> Deserialize<'de> for LiteralFloatValue { fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: serde::Deserializer<'de>, { let value = f64::deserialize(deserializer)?; if value.is_nan() { Err(serde::de::Error::custom("NaN is not allowed")) } else { Ok(LiteralFloatValue(value)) } }}