diff --git a/src/Common/Arg.fs b/src/Common/Arg.fs index fa592155..19d8e27c 100644 --- a/src/Common/Arg.fs +++ b/src/Common/Arg.fs @@ -1,8 +1,8 @@ -// (c) Microsoft Corporation 2005-2009. +// (c) Microsoft Corporation 2005-2009. namespace FSharp.Text -type ArgType = +type ArgType = | ClearArg of bool ref | FloatArg of (float -> unit) | IntArg of (int -> unit) @@ -19,22 +19,22 @@ type ArgType = static member Unit r = UnitArg r -type ArgInfo (name,action,help) = +type ArgInfo (name,action,help) = member x.Name = name member x.ArgType = action member x.HelpText = help - + exception Bad of string exception HelpText of string [] -type ArgParser() = - static let getUsage specs u = - let sbuf = new System.Text.StringBuilder 100 - let pstring (s:string) = sbuf.Append s |> ignore - let pendline s = pstring s; pstring "\n" +type ArgParser() = + static let getUsage specs u = + let sbuf = new System.Text.StringBuilder 100 + let pstring (s:string) = sbuf.Append s |> ignore + let pendline s = pstring s; pstring "\n" pendline u; - List.iter (fun (arg:ArgInfo) -> + List.iter (fun (arg:ArgInfo) -> match arg.Name, arg.ArgType, arg.HelpText with | (s, (UnitArg _ | SetArg _ | ClearArg _), helpText) -> pstring "\t"; pstring s; pstring ": "; pendline helpText | (s, StringArg _, helpText) -> pstring "\t"; pstring s; pstring " : "; pendline helpText @@ -47,82 +47,82 @@ type ArgParser() = sbuf.ToString() - static member ParsePartial(cursor,argv,argSpecs:seq,?other,?usageText) = - let other = defaultArg other (fun _ -> ()) + static member ParsePartial(cursor,argv,arguments:seq,?otherArgs,?usageText) = + let other = defaultArg otherArgs (fun _ -> ()) let usageText = defaultArg usageText "" - let nargs = Array.length argv + let nargs = Array.length argv incr cursor; - let argSpecs = argSpecs |> Seq.toList + let argSpecs = arguments |> Seq.toList let specs = argSpecs |> List.map (fun (arg:ArgInfo) -> arg.Name, arg.ArgType) while !cursor < nargs do - let arg = argv.[!cursor] - let rec findMatchingArg args = + let arg = argv.[!cursor] + let rec findMatchingArg args = match args with - | ((s, action) :: _) when s = arg -> - let getSecondArg () = - if !cursor + 1 >= nargs then + | ((s, action) :: _) when s = arg -> + let getSecondArg () = + if !cursor + 1 >= nargs then raise(Bad("option "+s+" needs an argument.\n"+getUsage argSpecs usageText)); - argv.[!cursor+1] - - match action with - | UnitArg f -> - f (); + argv.[!cursor+1] + + match action with + | UnitArg f -> + f (); incr cursor | SetArg f -> - f := true; + f := true; incr cursor - | ClearArg f -> - f := false; + | ClearArg f -> + f := false; incr cursor - | StringArg f-> - let arg2 = getSecondArg() - f arg2; + | StringArg f-> + let arg2 = getSecondArg() + f arg2; cursor := !cursor + 2 - | IntArg f -> - let arg2 = getSecondArg () - let arg2 = try int32 arg2 with _ -> raise(Bad(getUsage argSpecs usageText)) in + | IntArg f -> + let arg2 = getSecondArg () + let arg2 = try int32 arg2 with _ -> raise(Bad(getUsage argSpecs usageText)) in f arg2; cursor := !cursor + 2; - | FloatArg f -> - let arg2 = getSecondArg() - let arg2 = try float arg2 with _ -> raise(Bad(getUsage argSpecs usageText)) in - f arg2; + | FloatArg f -> + let arg2 = getSecondArg() + let arg2 = try float arg2 with _ -> raise(Bad(getUsage argSpecs usageText)) in + f arg2; cursor := !cursor + 2; - | RestArg f -> + | RestArg f -> incr cursor; while !cursor < nargs do f (argv.[!cursor]); incr cursor; - | (_ :: more) -> findMatchingArg more - | [] -> + | (_ :: more) -> findMatchingArg more + | [] -> if arg = "-help" || arg = "--help" || arg = "/help" || arg = "/help" || arg = "/?" then raise (HelpText (getUsage argSpecs usageText)) // Note: for '/abc/def' does not count as an argument // Note: '/abc' does elif arg.Length>0 && (arg.[0] = '-' || (arg.[0] = '/' && not (arg.Length > 1 && arg.[1..].Contains ("/")))) then raise (Bad ("unrecognized argument: "+ arg + "\n" + getUsage argSpecs usageText)) - else + else other arg; incr cursor - findMatchingArg specs + findMatchingArg specs - static member Usage (specs,?usage) = + static member Usage (arguments,?usage) = let usage = defaultArg usage "" - System.Console.Error.WriteLine (getUsage (Seq.toList specs) usage) + System.Console.Error.WriteLine (getUsage (Seq.toList arguments) usage) #if FX_NO_COMMAND_LINE_ARGS #else - static member Parse (specs,?other,?usageText) = + static member Parse (arguments,?otherArgs,?usageText) = let current = ref 0 - let argv = System.Environment.GetCommandLineArgs() - try ArgParser.ParsePartial (current, argv, specs, ?other=other, ?usageText=usageText) - with - | Bad h - | HelpText h -> - System.Console.Error.WriteLine h; - System.Console.Error.Flush(); - System.Environment.Exit(1); - | e -> + let argv = System.Environment.GetCommandLineArgs() + try ArgParser.ParsePartial (current, argv, arguments, ?otherArgs=otherArgs, ?usageText=usageText) + with + | Bad h + | HelpText h -> + System.Console.Error.WriteLine h; + System.Console.Error.Flush(); + System.Environment.Exit(1); + | e -> reraise() #endif diff --git a/src/FsLex.Core/fslexast.fs b/src/FsLex.Core/fslexast.fs index 760286ec..91b2a4e6 100644 --- a/src/FsLex.Core/fslexast.fs +++ b/src/FsLex.Core/fslexast.fs @@ -6,8 +6,8 @@ open System.Collections.Generic open System.Globalization open FSharp.Text.Lexing -type Ident = string -type Code = string * Position +type Ident = string * Range +type Code = string * Range type ParseContext = { @@ -144,10 +144,11 @@ type Regexp = | Inp of Input | Star of Regexp | Macro of Ident -type Clause = Regexp * Code -type Rule = (Ident * Ident list * Clause list) -type Macro = Ident * Regexp +type Clause = { Matcher: Regexp; Code: Code } + +type Rule = { Name: Ident; Arguments: Ident list; Clauses: Clause list } +type Macro = { Name: Ident; Matcher: Regexp } type Spec = { TopCode: Code @@ -192,7 +193,7 @@ type NfaNodeMap() = map.[nodeId] <-node node -let LexerStateToNfa ctx (macros: Map) (clauses: Clause list) = +let LexerStateToNfa ctx (macros: Map) (clauses: Clause list) = /// Table allocating node ids let nfaNodeMap = new NfaNodeMap() @@ -201,7 +202,7 @@ let LexerStateToNfa ctx (macros: Map) (clauses: Clause list) = let rec CompileRegexp re dest = match re with | Alt res -> - let trs = res ctx |> List.map (fun re -> (Epsilon,CompileRegexp re dest)) + let trs = res ctx |> List.map (fun re -> (Epsilon, CompileRegexp re dest)) nfaNodeMap.NewNfaNode(trs,[]) | Seq res -> List.foldBack (CompileRegexp) res dest @@ -224,9 +225,9 @@ let LexerStateToNfa ctx (macros: Map) (clauses: Clause list) = let sre = CompileRegexp re nfaNode AddToMultiMap nfaNode.Transitions Epsilon sre nfaNodeMap.NewNfaNode([(Epsilon,sre); (Epsilon,dest)],[]) - | Macro m -> - if not <| macros.ContainsKey(m) then failwithf "The macro %s is not defined" m - CompileRegexp macros.[m] dest + | Macro (name, _) as m -> + if not <| macros.ContainsKey(name) then failwithf "The macro %s is not defined" name + CompileRegexp macros.[name].Matcher dest // These cases unwind the difficult cases in the syntax that rely on knowing the // entire alphabet. @@ -274,13 +275,13 @@ let LexerStateToNfa ctx (macros: Map) (clauses: Clause list) = let actions = new System.Collections.Generic.List<_>() /// Compile an acceptance of a regular expression into the NFA - let sTrans macros nodeId (regexp,code) = + let sTrans macros nodeId { Matcher = regexp; Code = code } = let actionId = actions.Count actions.Add(code) - let sAccept = nfaNodeMap.NewNfaNode([],[(nodeId,actionId)]) + let sAccept = nfaNodeMap.NewNfaNode([], [(nodeId, actionId)]) CompileRegexp regexp sAccept - let trs = clauses |> List.mapi (fun n x -> (Epsilon,sTrans macros n x)) + let trs = clauses |> List.mapi (fun n x -> (Epsilon, sTrans macros n x)) let nfaStartNode = nfaNodeMap.NewNfaNode(trs,[]) nfaStartNode,(actions |> Seq.readonly), nfaNodeMap @@ -407,10 +408,10 @@ let NfaToDfa (nfaNodeMap:NfaNodeMap) nfaStartNode = ruleStartNode,ruleNodes let Compile ctx spec = - let macros = Map.ofList spec.Macros + let macros = Map.ofList (spec.Macros |> List.map (fun m -> fst m.Name, m)) List.foldBack - (fun (name,args,clauses) (perRuleData,dfaNodes) -> - let nfa, actions, nfaNodeMap = LexerStateToNfa ctx macros clauses + (fun rule (perRuleData,dfaNodes) -> + let nfa, actions, nfaNodeMap = LexerStateToNfa ctx macros rule.Clauses let ruleStartNode, ruleNodes = NfaToDfa nfaNodeMap nfa //printfn "name = %s, ruleStartNode = %O" name ruleStartNode.Id (ruleStartNode,actions) :: perRuleData, ruleNodes @ dfaNodes) diff --git a/src/FsLex.Core/fslexdriver.fs b/src/FsLex.Core/fslexdriver.fs index 44519609..a995d2fb 100644 --- a/src/FsLex.Core/fslexdriver.fs +++ b/src/FsLex.Core/fslexdriver.fs @@ -36,7 +36,7 @@ type Writer(fileName) = member x.write fmt = Printf.fprintf os fmt - member x.writeCode (code, pos: Position) = + member x.writeCode (code, { startPos = pos }) = if pos <> Position.Empty // If bottom code is unspecified, then position is empty. then x.writeLine "# %d \"%s\"" pos.Line pos.FileName @@ -175,13 +175,14 @@ let writeRules (rules: Rule list) (perRuleData: PerRuleData) outputFileName (wri // These actions push the additional start state and come first, because they are then typically inlined into later // rules. This means more tailcalls are taken as direct branches, increasing efficiency and // improving stack usage on platforms that do not take tailcalls. - for ((startNode, actions),(ident,args,_)) in List.zip perRuleData rules do + for ((startNode, actions),{ Name = (ident, _); Arguments = args } ) in List.zip perRuleData rules do writer.writeLine "// Rule %s" ident - writer.writeLine "and %s %s lexbuf =" ident (String.Join(" ", Array.ofList args)) + let argumentNames = args |> List.map fst |> Array.ofList + writer.writeLine "and %s %s lexbuf =" ident (String.Join(" ", argumentNames)) writer.writeLine " match _fslex_tables.Interpret(%d,lexbuf) with" startNode.Id - actions |> Seq.iteri (fun i (code:string, pos) -> + actions |> Seq.iteri (fun i (code:string, range) -> writer.writeLine " | %d -> ( " i - writer.writeLine "# %d \"%s\"" pos.Line pos.FileName + writer.writeLine "# %d \"%s\"" range.startPos.Line range.startPos.FileName let lines = code.Split([| '\r'; '\n' |], StringSplitOptions.RemoveEmptyEntries) for line in lines do writer.writeLine " %s" line diff --git a/src/FsLex.Core/fslexlex.fs b/src/FsLex.Core/fslexlex.fs index 6738cdb1..3dc9b3e9 100644 --- a/src/FsLex.Core/fslexlex.fs +++ b/src/FsLex.Core/fslexlex.fs @@ -579,7 +579,7 @@ and code p buff lexbuf = match _fslex_tables.Interpret(28,lexbuf) with | 0 -> ( # 155 "fslexlex.fsl" - CODE (buff.ToString(), p) + CODE (buff.ToString(), { startPos = p; endPos = lexbuf.EndPos }) # 583 "fslexlex.fs" ) | 1 -> ( diff --git a/src/FsLex.Core/fslexlex.fsl b/src/FsLex.Core/fslexlex.fsl index cdc15dc7..ca96fd35 100644 --- a/src/FsLex.Core/fslexlex.fsl +++ b/src/FsLex.Core/fslexlex.fsl @@ -152,7 +152,7 @@ and string p buff = parse | _ { let _ = buff.Append (lexeme lexbuf).[0] in string p buff lexbuf } and code p buff = parse - | "}" { CODE (buff.ToString(), p) } + | "}" { CODE (buff.ToString(), { startPos = p; endPos = lexbuf.EndPos }) } | "{" { let _ = buff.Append (lexeme lexbuf) in ignore(code p buff lexbuf); let _ = buff.Append "}" in diff --git a/src/FsLex.Core/fslexpars.fs b/src/FsLex.Core/fslexpars.fs index 55b628b4..d603122c 100644 --- a/src/FsLex.Core/fslexpars.fs +++ b/src/FsLex.Core/fslexpars.fs @@ -68,6 +68,7 @@ type tokenId = type nonTerminalId = | NONTERM__startspec | NONTERM_spec + | NONTERM_ident | NONTERM_codeopt | NONTERM_Macros | NONTERM_macro @@ -142,22 +143,22 @@ let prodIdxToNonTerminal (prodIdx:int) = match prodIdx with | 0 -> NONTERM__startspec | 1 -> NONTERM_spec - | 2 -> NONTERM_codeopt + | 2 -> NONTERM_ident | 3 -> NONTERM_codeopt - | 4 -> NONTERM_Macros + | 4 -> NONTERM_codeopt | 5 -> NONTERM_Macros - | 6 -> NONTERM_macro - | 7 -> NONTERM_Rules + | 6 -> NONTERM_Macros + | 7 -> NONTERM_macro | 8 -> NONTERM_Rules - | 9 -> NONTERM_rule - | 10 -> NONTERM_args + | 9 -> NONTERM_Rules + | 10 -> NONTERM_rule | 11 -> NONTERM_args - | 12 -> NONTERM_optbar + | 12 -> NONTERM_args | 13 -> NONTERM_optbar - | 14 -> NONTERM_clauses + | 14 -> NONTERM_optbar | 15 -> NONTERM_clauses - | 16 -> NONTERM_clause - | 17 -> NONTERM_regexp + | 16 -> NONTERM_clauses + | 17 -> NONTERM_clause | 18 -> NONTERM_regexp | 19 -> NONTERM_regexp | 20 -> NONTERM_regexp @@ -171,9 +172,10 @@ let prodIdxToNonTerminal (prodIdx:int) = | 28 -> NONTERM_regexp | 29 -> NONTERM_regexp | 30 -> NONTERM_regexp - | 31 -> NONTERM_charset + | 31 -> NONTERM_regexp | 32 -> NONTERM_charset | 33 -> NONTERM_charset + | 34 -> NONTERM_charset | _ -> failwith "prodIdxToNonTerminal: bad production index" let _fsyacc_endOfInputTag = 25 @@ -232,18 +234,18 @@ let _fsyacc_dataOfToken (t:token) = | CODE _fsyacc_x -> Microsoft.FSharp.Core.Operators.box _fsyacc_x | STRING _fsyacc_x -> Microsoft.FSharp.Core.Operators.box _fsyacc_x | IDENT _fsyacc_x -> Microsoft.FSharp.Core.Operators.box _fsyacc_x -let _fsyacc_gotos = [| 0us; 65535us; 1us; 65535us; 0us; 1us; 2us; 65535us; 0us; 2us; 5us; 6us; 2us; 65535us; 2us; 3us; 8us; 9us; 2us; 65535us; 2us; 8us; 8us; 8us; 2us; 65535us; 4us; 5us; 15us; 16us; 2us; 65535us; 4us; 14us; 15us; 14us; 2us; 65535us; 17us; 18us; 23us; 24us; 1us; 65535us; 20us; 21us; 2us; 65535us; 21us; 22us; 27us; 28us; 2us; 65535us; 21us; 26us; 27us; 26us; 10us; 65535us; 12us; 13us; 13us; 37us; 21us; 29us; 27us; 29us; 29us; 37us; 37us; 37us; 38us; 37us; 39us; 37us; 43us; 38us; 44us; 39us; 5us; 65535us; 46us; 47us; 47us; 55us; 49us; 50us; 50us; 55us; 55us; 55us; |] -let _fsyacc_sparseGotoTableRowOffsets = [|0us; 1us; 3us; 6us; 9us; 12us; 15us; 18us; 21us; 23us; 26us; 29us; 40us; |] -let _fsyacc_stateToProdIdxsTableElements = [| 1us; 0us; 1us; 0us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 2us; 1us; 5us; 1us; 5us; 1us; 6us; 1us; 6us; 1us; 6us; 6us; 6us; 23us; 24us; 25us; 26us; 27us; 2us; 7us; 8us; 1us; 7us; 1us; 7us; 1us; 9us; 1us; 9us; 1us; 9us; 1us; 9us; 1us; 9us; 1us; 9us; 1us; 11us; 1us; 11us; 1us; 13us; 2us; 14us; 15us; 1us; 14us; 1us; 14us; 6us; 16us; 23us; 24us; 25us; 26us; 27us; 1us; 16us; 1us; 17us; 1us; 18us; 1us; 19us; 1us; 20us; 1us; 21us; 1us; 22us; 6us; 23us; 23us; 24us; 25us; 26us; 27us; 6us; 23us; 24us; 25us; 26us; 27us; 27us; 6us; 23us; 24us; 25us; 26us; 27us; 28us; 1us; 24us; 1us; 25us; 1us; 26us; 1us; 27us; 1us; 28us; 1us; 28us; 2us; 29us; 30us; 2us; 29us; 33us; 1us; 29us; 1us; 30us; 2us; 30us; 33us; 1us; 30us; 2us; 31us; 32us; 1us; 32us; 1us; 32us; 2us; 33us; 33us; |] -let _fsyacc_stateToProdIdxsTableRowOffsets = [|0us; 2us; 4us; 6us; 8us; 10us; 12us; 14us; 16us; 18us; 20us; 22us; 24us; 26us; 33us; 36us; 38us; 40us; 42us; 44us; 46us; 48us; 50us; 52us; 54us; 56us; 58us; 61us; 63us; 65us; 72us; 74us; 76us; 78us; 80us; 82us; 84us; 86us; 93us; 100us; 107us; 109us; 111us; 113us; 115us; 117us; 119us; 122us; 125us; 127us; 129us; 132us; 134us; 137us; 139us; 141us; |] -let _fsyacc_action_rows = 56 -let _fsyacc_actionTableElements = [|1us; 16387us; 20us; 7us; 0us; 49152us; 1us; 16388us; 14us; 10us; 1us; 32768us; 12us; 4us; 1us; 32768us; 22us; 17us; 1us; 16387us; 20us; 7us; 0us; 16385us; 0us; 16386us; 1us; 16388us; 14us; 10us; 0us; 16389us; 1us; 32768us; 22us; 11us; 1us; 32768us; 6us; 12us; 8us; 32768us; 0us; 33us; 7us; 34us; 8us; 46us; 16us; 44us; 18us; 32us; 19us; 31us; 21us; 35us; 22us; 36us; 12us; 16390us; 0us; 33us; 1us; 43us; 3us; 40us; 4us; 41us; 5us; 42us; 7us; 34us; 8us; 46us; 16us; 44us; 18us; 32us; 19us; 31us; 21us; 35us; 22us; 36us; 1us; 16392us; 15us; 15us; 1us; 32768us; 22us; 17us; 0us; 16391us; 1us; 16394us; 22us; 23us; 1us; 32768us; 6us; 19us; 1us; 32768us; 13us; 20us; 1us; 16396us; 1us; 25us; 8us; 32768us; 0us; 33us; 7us; 34us; 8us; 46us; 16us; 44us; 18us; 32us; 19us; 31us; 21us; 35us; 22us; 36us; 0us; 16393us; 1us; 16394us; 22us; 23us; 0us; 16395us; 0us; 16397us; 1us; 16399us; 1us; 27us; 8us; 32768us; 0us; 33us; 7us; 34us; 8us; 46us; 16us; 44us; 18us; 32us; 19us; 31us; 21us; 35us; 22us; 36us; 0us; 16398us; 13us; 32768us; 0us; 33us; 1us; 43us; 3us; 40us; 4us; 41us; 5us; 42us; 7us; 34us; 8us; 46us; 16us; 44us; 18us; 32us; 19us; 31us; 20us; 30us; 21us; 35us; 22us; 36us; 0us; 16400us; 0us; 16401us; 0us; 16402us; 0us; 16403us; 0us; 16404us; 0us; 16405us; 0us; 16406us; 11us; 16407us; 0us; 33us; 3us; 40us; 4us; 41us; 5us; 42us; 7us; 34us; 8us; 46us; 16us; 44us; 18us; 32us; 19us; 31us; 21us; 35us; 22us; 36us; 11us; 16411us; 0us; 33us; 3us; 40us; 4us; 41us; 5us; 42us; 7us; 34us; 8us; 46us; 16us; 44us; 18us; 32us; 19us; 31us; 21us; 35us; 22us; 36us; 13us; 32768us; 0us; 33us; 1us; 43us; 3us; 40us; 4us; 41us; 5us; 42us; 7us; 34us; 8us; 46us; 16us; 44us; 17us; 45us; 18us; 32us; 19us; 31us; 21us; 35us; 22us; 36us; 0us; 16408us; 0us; 16409us; 0us; 16410us; 8us; 32768us; 0us; 33us; 7us; 34us; 8us; 46us; 16us; 44us; 18us; 32us; 19us; 31us; 21us; 35us; 22us; 36us; 8us; 32768us; 0us; 33us; 7us; 34us; 8us; 46us; 16us; 44us; 18us; 32us; 19us; 31us; 21us; 35us; 22us; 36us; 0us; 16412us; 2us; 32768us; 10us; 49us; 19us; 52us; 2us; 32768us; 9us; 48us; 19us; 52us; 0us; 16413us; 1us; 32768us; 19us; 52us; 2us; 32768us; 9us; 51us; 19us; 52us; 0us; 16414us; 1us; 16415us; 11us; 53us; 1us; 32768us; 19us; 54us; 0us; 16416us; 1us; 16417us; 19us; 52us; |] -let _fsyacc_actionTableRowOffsets = [|0us; 2us; 3us; 5us; 7us; 9us; 11us; 12us; 13us; 15us; 16us; 18us; 20us; 29us; 42us; 44us; 46us; 47us; 49us; 51us; 53us; 55us; 64us; 65us; 67us; 68us; 69us; 71us; 80us; 81us; 95us; 96us; 97us; 98us; 99us; 100us; 101us; 102us; 114us; 126us; 140us; 141us; 142us; 143us; 152us; 161us; 162us; 165us; 168us; 169us; 171us; 174us; 175us; 177us; 179us; 180us; |] -let _fsyacc_reductionSymbolCounts = [|1us; 5us; 1us; 0us; 0us; 2us; 4us; 3us; 1us; 6us; 0us; 2us; 0us; 1us; 3us; 1us; 2us; 1us; 1us; 1us; 1us; 1us; 1us; 2us; 2us; 2us; 2us; 3us; 3us; 3us; 4us; 1us; 3us; 2us; |] -let _fsyacc_productionToNonTerminalTable = [|0us; 1us; 2us; 2us; 3us; 3us; 4us; 5us; 5us; 6us; 7us; 7us; 8us; 8us; 9us; 9us; 10us; 11us; 11us; 11us; 11us; 11us; 11us; 11us; 11us; 11us; 11us; 11us; 11us; 11us; 11us; 12us; 12us; 12us; |] -let _fsyacc_immediateActions = [|65535us; 49152us; 65535us; 65535us; 65535us; 65535us; 16385us; 16386us; 65535us; 16389us; 65535us; 65535us; 65535us; 65535us; 65535us; 65535us; 16391us; 65535us; 65535us; 65535us; 65535us; 65535us; 16393us; 65535us; 16395us; 16397us; 65535us; 65535us; 16398us; 65535us; 16400us; 16401us; 16402us; 16403us; 16404us; 16405us; 16406us; 65535us; 65535us; 65535us; 16408us; 16409us; 16410us; 65535us; 65535us; 16412us; 65535us; 65535us; 16413us; 65535us; 65535us; 16414us; 65535us; 65535us; 16416us; 65535us; |] +let _fsyacc_gotos = [| 0us; 65535us; 1us; 65535us; 0us; 1us; 15us; 65535us; 4us; 18us; 11us; 12us; 13us; 37us; 14us; 37us; 16us; 18us; 18us; 24us; 22us; 37us; 24us; 24us; 28us; 37us; 30us; 37us; 38us; 37us; 39us; 37us; 40us; 37us; 44us; 37us; 45us; 37us; 2us; 65535us; 0us; 2us; 5us; 6us; 2us; 65535us; 2us; 3us; 9us; 10us; 2us; 65535us; 2us; 9us; 9us; 9us; 2us; 65535us; 4us; 5us; 16us; 17us; 2us; 65535us; 4us; 15us; 16us; 15us; 2us; 65535us; 18us; 19us; 24us; 25us; 1us; 65535us; 21us; 22us; 2us; 65535us; 22us; 23us; 28us; 29us; 2us; 65535us; 22us; 27us; 28us; 27us; 10us; 65535us; 13us; 14us; 14us; 38us; 22us; 30us; 28us; 30us; 30us; 38us; 38us; 38us; 39us; 38us; 40us; 38us; 44us; 39us; 45us; 40us; 5us; 65535us; 47us; 48us; 48us; 56us; 50us; 51us; 51us; 56us; 56us; 56us; |] +let _fsyacc_sparseGotoTableRowOffsets = [|0us; 1us; 3us; 19us; 22us; 25us; 28us; 31us; 34us; 37us; 39us; 42us; 45us; 56us; |] +let _fsyacc_stateToProdIdxsTableElements = [| 1us; 0us; 1us; 0us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 2us; 1us; 3us; 1us; 6us; 1us; 6us; 1us; 7us; 1us; 7us; 1us; 7us; 6us; 7us; 24us; 25us; 26us; 27us; 28us; 2us; 8us; 9us; 1us; 8us; 1us; 8us; 1us; 10us; 1us; 10us; 1us; 10us; 1us; 10us; 1us; 10us; 1us; 10us; 1us; 12us; 1us; 12us; 1us; 14us; 2us; 15us; 16us; 1us; 15us; 1us; 15us; 6us; 17us; 24us; 25us; 26us; 27us; 28us; 1us; 17us; 1us; 18us; 1us; 19us; 1us; 20us; 1us; 21us; 1us; 22us; 1us; 23us; 6us; 24us; 24us; 25us; 26us; 27us; 28us; 6us; 24us; 25us; 26us; 27us; 28us; 28us; 6us; 24us; 25us; 26us; 27us; 28us; 29us; 1us; 25us; 1us; 26us; 1us; 27us; 1us; 28us; 1us; 29us; 1us; 29us; 2us; 30us; 31us; 2us; 30us; 34us; 1us; 30us; 1us; 31us; 2us; 31us; 34us; 1us; 31us; 2us; 32us; 33us; 1us; 33us; 1us; 33us; 2us; 34us; 34us; |] +let _fsyacc_stateToProdIdxsTableRowOffsets = [|0us; 2us; 4us; 6us; 8us; 10us; 12us; 14us; 16us; 18us; 20us; 22us; 24us; 26us; 28us; 35us; 38us; 40us; 42us; 44us; 46us; 48us; 50us; 52us; 54us; 56us; 58us; 60us; 63us; 65us; 67us; 74us; 76us; 78us; 80us; 82us; 84us; 86us; 88us; 95us; 102us; 109us; 111us; 113us; 115us; 117us; 119us; 121us; 124us; 127us; 129us; 131us; 134us; 136us; 139us; 141us; 143us; |] +let _fsyacc_action_rows = 57 +let _fsyacc_actionTableElements = [|1us; 16388us; 20us; 8us; 0us; 49152us; 1us; 16389us; 14us; 11us; 1us; 32768us; 12us; 4us; 1us; 32768us; 22us; 7us; 1us; 16388us; 20us; 8us; 0us; 16385us; 0us; 16386us; 0us; 16387us; 1us; 16389us; 14us; 11us; 0us; 16390us; 1us; 32768us; 22us; 7us; 1us; 32768us; 6us; 13us; 8us; 32768us; 0us; 34us; 7us; 35us; 8us; 47us; 16us; 45us; 18us; 33us; 19us; 32us; 21us; 36us; 22us; 7us; 12us; 16391us; 0us; 34us; 1us; 44us; 3us; 41us; 4us; 42us; 5us; 43us; 7us; 35us; 8us; 47us; 16us; 45us; 18us; 33us; 19us; 32us; 21us; 36us; 22us; 7us; 1us; 16393us; 15us; 16us; 1us; 32768us; 22us; 7us; 0us; 16392us; 1us; 16395us; 22us; 7us; 1us; 32768us; 6us; 20us; 1us; 32768us; 13us; 21us; 1us; 16397us; 1us; 26us; 8us; 32768us; 0us; 34us; 7us; 35us; 8us; 47us; 16us; 45us; 18us; 33us; 19us; 32us; 21us; 36us; 22us; 7us; 0us; 16394us; 1us; 16395us; 22us; 7us; 0us; 16396us; 0us; 16398us; 1us; 16400us; 1us; 28us; 8us; 32768us; 0us; 34us; 7us; 35us; 8us; 47us; 16us; 45us; 18us; 33us; 19us; 32us; 21us; 36us; 22us; 7us; 0us; 16399us; 13us; 32768us; 0us; 34us; 1us; 44us; 3us; 41us; 4us; 42us; 5us; 43us; 7us; 35us; 8us; 47us; 16us; 45us; 18us; 33us; 19us; 32us; 20us; 31us; 21us; 36us; 22us; 7us; 0us; 16401us; 0us; 16402us; 0us; 16403us; 0us; 16404us; 0us; 16405us; 0us; 16406us; 0us; 16407us; 11us; 16408us; 0us; 34us; 3us; 41us; 4us; 42us; 5us; 43us; 7us; 35us; 8us; 47us; 16us; 45us; 18us; 33us; 19us; 32us; 21us; 36us; 22us; 7us; 11us; 16412us; 0us; 34us; 3us; 41us; 4us; 42us; 5us; 43us; 7us; 35us; 8us; 47us; 16us; 45us; 18us; 33us; 19us; 32us; 21us; 36us; 22us; 7us; 13us; 32768us; 0us; 34us; 1us; 44us; 3us; 41us; 4us; 42us; 5us; 43us; 7us; 35us; 8us; 47us; 16us; 45us; 17us; 46us; 18us; 33us; 19us; 32us; 21us; 36us; 22us; 7us; 0us; 16409us; 0us; 16410us; 0us; 16411us; 8us; 32768us; 0us; 34us; 7us; 35us; 8us; 47us; 16us; 45us; 18us; 33us; 19us; 32us; 21us; 36us; 22us; 7us; 8us; 32768us; 0us; 34us; 7us; 35us; 8us; 47us; 16us; 45us; 18us; 33us; 19us; 32us; 21us; 36us; 22us; 7us; 0us; 16413us; 2us; 32768us; 10us; 50us; 19us; 53us; 2us; 32768us; 9us; 49us; 19us; 53us; 0us; 16414us; 1us; 32768us; 19us; 53us; 2us; 32768us; 9us; 52us; 19us; 53us; 0us; 16415us; 1us; 16416us; 11us; 54us; 1us; 32768us; 19us; 55us; 0us; 16417us; 1us; 16418us; 19us; 53us; |] +let _fsyacc_actionTableRowOffsets = [|0us; 2us; 3us; 5us; 7us; 9us; 11us; 12us; 13us; 14us; 16us; 17us; 19us; 21us; 30us; 43us; 45us; 47us; 48us; 50us; 52us; 54us; 56us; 65us; 66us; 68us; 69us; 70us; 72us; 81us; 82us; 96us; 97us; 98us; 99us; 100us; 101us; 102us; 103us; 115us; 127us; 141us; 142us; 143us; 144us; 153us; 162us; 163us; 166us; 169us; 170us; 172us; 175us; 176us; 178us; 180us; 181us; |] +let _fsyacc_reductionSymbolCounts = [|1us; 5us; 1us; 1us; 0us; 0us; 2us; 4us; 3us; 1us; 6us; 0us; 2us; 0us; 1us; 3us; 1us; 2us; 1us; 1us; 1us; 1us; 1us; 1us; 2us; 2us; 2us; 2us; 3us; 3us; 3us; 4us; 1us; 3us; 2us; |] +let _fsyacc_productionToNonTerminalTable = [|0us; 1us; 2us; 3us; 3us; 4us; 4us; 5us; 6us; 6us; 7us; 8us; 8us; 9us; 9us; 10us; 10us; 11us; 12us; 12us; 12us; 12us; 12us; 12us; 12us; 12us; 12us; 12us; 12us; 12us; 12us; 12us; 13us; 13us; 13us; |] +let _fsyacc_immediateActions = [|65535us; 49152us; 65535us; 65535us; 65535us; 65535us; 16385us; 16386us; 16387us; 65535us; 16390us; 65535us; 65535us; 65535us; 65535us; 65535us; 65535us; 16392us; 65535us; 65535us; 65535us; 65535us; 65535us; 16394us; 65535us; 16396us; 16398us; 65535us; 65535us; 16399us; 65535us; 16401us; 16402us; 16403us; 16404us; 16405us; 16406us; 16407us; 65535us; 65535us; 65535us; 16409us; 16410us; 16411us; 65535us; 65535us; 16413us; 65535us; 65535us; 16414us; 65535us; 65535us; 16415us; 65535us; 65535us; 16417us; 65535us; |] let _fsyacc_reductions () = [| -# 246 "fslexpars.fs" +# 248 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> AST.Spec in Microsoft.FSharp.Core.Operators.box @@ -252,7 +254,7 @@ let _fsyacc_reductions () = [| raise (FSharp.Text.Parsing.Accept(Microsoft.FSharp.Core.Operators.box _1)) ) : 'gentype__startspec)); -# 255 "fslexpars.fs" +# 257 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_codeopt in let _2 = parseState.GetInput(2) :?> 'gentype_Macros in @@ -267,370 +269,381 @@ let _fsyacc_reductions () = [| ) # 25 "fslexpars.fsy" : AST.Spec)); -# 270 "fslexpars.fs" +# 272 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> - let _1 = parseState.GetInput(1) :?> AST.Code in + let _1 = parseState.GetInput(1) :?> string in Microsoft.FSharp.Core.Operators.box ( ( # 30 "fslexpars.fsy" - _1 + _1, parseState.InputRange 1 ) # 30 "fslexpars.fsy" - : 'gentype_codeopt)); -# 281 "fslexpars.fs" + : 'gentype_ident)); +# 283 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> + let _1 = parseState.GetInput(1) :?> AST.Code in Microsoft.FSharp.Core.Operators.box ( ( -# 31 "fslexpars.fsy" - "", (parseState.ResultRange |> fst) +# 33 "fslexpars.fsy" + _1 ) -# 31 "fslexpars.fsy" +# 33 "fslexpars.fsy" : 'gentype_codeopt)); -# 291 "fslexpars.fs" +# 294 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( # 34 "fslexpars.fsy" - [] + "", parseState.ResultRange ) # 34 "fslexpars.fsy" + : 'gentype_codeopt)); +# 304 "fslexpars.fs" + (fun (parseState : FSharp.Text.Parsing.IParseState) -> + Microsoft.FSharp.Core.Operators.box + ( + ( +# 37 "fslexpars.fsy" + [] + ) +# 37 "fslexpars.fsy" : 'gentype_Macros)); -# 301 "fslexpars.fs" +# 314 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_macro in let _2 = parseState.GetInput(2) :?> 'gentype_Macros in Microsoft.FSharp.Core.Operators.box ( ( -# 35 "fslexpars.fsy" +# 38 "fslexpars.fsy" _1 :: _2 ) -# 35 "fslexpars.fsy" +# 38 "fslexpars.fsy" : 'gentype_Macros)); -# 314 "fslexpars.fs" +# 327 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> - let _2 = parseState.GetInput(2) :?> string in + let _2 = parseState.GetInput(2) :?> 'gentype_ident in let _4 = parseState.GetInput(4) :?> 'gentype_regexp in Microsoft.FSharp.Core.Operators.box ( ( -# 40 "fslexpars.fsy" +# 43 "fslexpars.fsy" - (_2, _4) + { Name = _2; Matcher = _4 } ) -# 40 "fslexpars.fsy" +# 43 "fslexpars.fsy" : 'gentype_macro)); -# 327 "fslexpars.fs" +# 340 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_rule in let _3 = parseState.GetInput(3) :?> 'gentype_Rules in Microsoft.FSharp.Core.Operators.box ( ( -# 45 "fslexpars.fsy" +# 48 "fslexpars.fsy" _1 :: _3 ) -# 45 "fslexpars.fsy" +# 48 "fslexpars.fsy" : 'gentype_Rules)); -# 340 "fslexpars.fs" +# 353 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_rule in Microsoft.FSharp.Core.Operators.box ( ( -# 48 "fslexpars.fsy" +# 51 "fslexpars.fsy" [_1] ) -# 48 "fslexpars.fsy" +# 51 "fslexpars.fsy" : 'gentype_Rules)); -# 351 "fslexpars.fs" +# 364 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> - let _1 = parseState.GetInput(1) :?> string in + let _1 = parseState.GetInput(1) :?> 'gentype_ident in let _2 = parseState.GetInput(2) :?> 'gentype_args in let _5 = parseState.GetInput(5) :?> 'gentype_optbar in let _6 = parseState.GetInput(6) :?> 'gentype_clauses in Microsoft.FSharp.Core.Operators.box ( ( -# 51 "fslexpars.fsy" +# 54 "fslexpars.fsy" - (_1,_2,_6) + { Name = _1; Arguments = _2; Clauses = _6 } ) -# 51 "fslexpars.fsy" +# 54 "fslexpars.fsy" : 'gentype_rule)); -# 366 "fslexpars.fs" +# 379 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 56 "fslexpars.fsy" +# 59 "fslexpars.fsy" [] ) -# 56 "fslexpars.fsy" +# 59 "fslexpars.fsy" : 'gentype_args)); -# 376 "fslexpars.fs" +# 389 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> - let _1 = parseState.GetInput(1) :?> string in + let _1 = parseState.GetInput(1) :?> 'gentype_ident in let _2 = parseState.GetInput(2) :?> 'gentype_args in Microsoft.FSharp.Core.Operators.box ( ( -# 57 "fslexpars.fsy" +# 60 "fslexpars.fsy" _1 :: _2 ) -# 57 "fslexpars.fsy" +# 60 "fslexpars.fsy" : 'gentype_args)); -# 388 "fslexpars.fs" +# 401 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 60 "fslexpars.fsy" +# 63 "fslexpars.fsy" ) -# 60 "fslexpars.fsy" +# 63 "fslexpars.fsy" : 'gentype_optbar)); -# 398 "fslexpars.fs" +# 411 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 61 "fslexpars.fsy" +# 64 "fslexpars.fsy" ) -# 61 "fslexpars.fsy" +# 64 "fslexpars.fsy" : 'gentype_optbar)); -# 408 "fslexpars.fs" +# 421 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_clause in let _3 = parseState.GetInput(3) :?> 'gentype_clauses in Microsoft.FSharp.Core.Operators.box ( ( -# 64 "fslexpars.fsy" +# 67 "fslexpars.fsy" _1 :: _3 ) -# 64 "fslexpars.fsy" +# 67 "fslexpars.fsy" : 'gentype_clauses)); -# 420 "fslexpars.fs" +# 433 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_clause in Microsoft.FSharp.Core.Operators.box ( ( -# 65 "fslexpars.fsy" +# 68 "fslexpars.fsy" [_1] ) -# 65 "fslexpars.fsy" +# 68 "fslexpars.fsy" : 'gentype_clauses)); -# 431 "fslexpars.fs" +# 444 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_regexp in let _2 = parseState.GetInput(2) :?> AST.Code in Microsoft.FSharp.Core.Operators.box ( ( -# 68 "fslexpars.fsy" - _1, _2 +# 71 "fslexpars.fsy" + { Matcher = _1; Code = _2 } ) -# 68 "fslexpars.fsy" +# 71 "fslexpars.fsy" : 'gentype_clause)); -# 443 "fslexpars.fs" +# 456 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> char in Microsoft.FSharp.Core.Operators.box ( ( -# 71 "fslexpars.fsy" +# 74 "fslexpars.fsy" Inp(Alphabet(EncodeChar _1)) ) -# 71 "fslexpars.fsy" +# 74 "fslexpars.fsy" : 'gentype_regexp)); -# 454 "fslexpars.fs" +# 467 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> string in Microsoft.FSharp.Core.Operators.box ( ( -# 72 "fslexpars.fsy" +# 75 "fslexpars.fsy" Inp(UnicodeCategory _1) ) -# 72 "fslexpars.fsy" +# 75 "fslexpars.fsy" : 'gentype_regexp)); -# 465 "fslexpars.fs" +# 478 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 73 "fslexpars.fsy" +# 76 "fslexpars.fsy" Inp(Alphabet(fun ctx -> Eof)) ) -# 73 "fslexpars.fsy" +# 76 "fslexpars.fsy" : 'gentype_regexp)); -# 475 "fslexpars.fs" +# 488 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 74 "fslexpars.fsy" +# 77 "fslexpars.fsy" Inp Any ) -# 74 "fslexpars.fsy" +# 77 "fslexpars.fsy" : 'gentype_regexp)); -# 485 "fslexpars.fs" +# 498 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> string in Microsoft.FSharp.Core.Operators.box ( ( -# 75 "fslexpars.fsy" +# 78 "fslexpars.fsy" Seq([ for n in 0 .. _1.Length - 1 -> Inp(Alphabet(EncodeChar _1.[n]))]) ) -# 75 "fslexpars.fsy" +# 78 "fslexpars.fsy" : 'gentype_regexp)); -# 496 "fslexpars.fs" +# 509 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> - let _1 = parseState.GetInput(1) :?> string in + let _1 = parseState.GetInput(1) :?> 'gentype_ident in Microsoft.FSharp.Core.Operators.box ( ( -# 76 "fslexpars.fsy" +# 79 "fslexpars.fsy" Macro(_1) ) -# 76 "fslexpars.fsy" +# 79 "fslexpars.fsy" : 'gentype_regexp)); -# 507 "fslexpars.fs" +# 520 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_regexp in let _2 = parseState.GetInput(2) :?> 'gentype_regexp in Microsoft.FSharp.Core.Operators.box ( ( -# 77 "fslexpars.fsy" +# 80 "fslexpars.fsy" Seq[_1;_2] ) -# 77 "fslexpars.fsy" +# 80 "fslexpars.fsy" : 'gentype_regexp)); -# 519 "fslexpars.fs" +# 532 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_regexp in Microsoft.FSharp.Core.Operators.box ( ( -# 78 "fslexpars.fsy" +# 81 "fslexpars.fsy" Seq[_1;Star _1] ) -# 78 "fslexpars.fsy" +# 81 "fslexpars.fsy" : 'gentype_regexp)); -# 530 "fslexpars.fs" +# 543 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_regexp in Microsoft.FSharp.Core.Operators.box ( ( -# 79 "fslexpars.fsy" +# 82 "fslexpars.fsy" Star _1 ) -# 79 "fslexpars.fsy" +# 82 "fslexpars.fsy" : 'gentype_regexp)); -# 541 "fslexpars.fs" +# 554 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_regexp in Microsoft.FSharp.Core.Operators.box ( ( -# 80 "fslexpars.fsy" +# 83 "fslexpars.fsy" Alt(fun ctx -> [Seq[];_1]) ) -# 80 "fslexpars.fsy" +# 83 "fslexpars.fsy" : 'gentype_regexp)); -# 552 "fslexpars.fs" +# 565 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_regexp in let _3 = parseState.GetInput(3) :?> 'gentype_regexp in Microsoft.FSharp.Core.Operators.box ( ( -# 81 "fslexpars.fsy" +# 84 "fslexpars.fsy" Alt(fun ctx -> [_1;_3]) ) -# 81 "fslexpars.fsy" +# 84 "fslexpars.fsy" : 'gentype_regexp)); -# 564 "fslexpars.fs" +# 577 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _2 = parseState.GetInput(2) :?> 'gentype_regexp in Microsoft.FSharp.Core.Operators.box ( ( -# 82 "fslexpars.fsy" +# 85 "fslexpars.fsy" _2 ) -# 82 "fslexpars.fsy" +# 85 "fslexpars.fsy" : 'gentype_regexp)); -# 575 "fslexpars.fs" +# 588 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _2 = parseState.GetInput(2) :?> 'gentype_charset in Microsoft.FSharp.Core.Operators.box ( ( -# 83 "fslexpars.fsy" +# 86 "fslexpars.fsy" Alt (fun ctx -> [ for c in (_2 ctx) -> Inp(Alphabet(fun ctx -> c)) ]) ) -# 83 "fslexpars.fsy" +# 86 "fslexpars.fsy" : 'gentype_regexp)); -# 586 "fslexpars.fs" +# 599 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _3 = parseState.GetInput(3) :?> 'gentype_charset in Microsoft.FSharp.Core.Operators.box ( ( -# 84 "fslexpars.fsy" +# 87 "fslexpars.fsy" Inp(NotCharSet(fun ctx -> _3 ctx)) ) -# 84 "fslexpars.fsy" +# 87 "fslexpars.fsy" : 'gentype_regexp)); -# 597 "fslexpars.fs" +# 610 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> char in Microsoft.FSharp.Core.Operators.box ( ( -# 87 "fslexpars.fsy" +# 90 "fslexpars.fsy" fun ctx -> Set.singleton(EncodeChar _1 ctx) ) -# 87 "fslexpars.fsy" +# 90 "fslexpars.fsy" : 'gentype_charset)); -# 608 "fslexpars.fs" +# 621 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> char in let _3 = parseState.GetInput(3) :?> char in Microsoft.FSharp.Core.Operators.box ( ( -# 88 "fslexpars.fsy" +# 91 "fslexpars.fsy" fun ctx -> Set.ofSeq [ for c in _1 .. _3 -> EncodeChar c ctx ] ) -# 88 "fslexpars.fsy" +# 91 "fslexpars.fsy" : 'gentype_charset)); -# 620 "fslexpars.fs" +# 633 "fslexpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_charset in let _2 = parseState.GetInput(2) :?> 'gentype_charset in Microsoft.FSharp.Core.Operators.box ( ( -# 89 "fslexpars.fsy" +# 92 "fslexpars.fsy" fun ctx -> Set.union (_1 ctx) (_2 ctx) ) -# 89 "fslexpars.fsy" +# 92 "fslexpars.fsy" : 'gentype_charset)); |] -# 633 "fslexpars.fs" +# 646 "fslexpars.fs" let tables : FSharp.Text.Parsing.Tables<_> = { reductions= _fsyacc_reductions (); endOfInputTag = _fsyacc_endOfInputTag; diff --git a/src/FsLex.Core/fslexpars.fsi b/src/FsLex.Core/fslexpars.fsi index 955445a6..f32c2265 100644 --- a/src/FsLex.Core/fslexpars.fsi +++ b/src/FsLex.Core/fslexpars.fsi @@ -53,6 +53,7 @@ type tokenId = type nonTerminalId = | NONTERM__startspec | NONTERM_spec + | NONTERM_ident | NONTERM_codeopt | NONTERM_Macros | NONTERM_macro diff --git a/src/FsLex.Core/fslexpars.fsy b/src/FsLex.Core/fslexpars.fsy index 69e28cce..ba053457 100644 --- a/src/FsLex.Core/fslexpars.fsy +++ b/src/FsLex.Core/fslexpars.fsy @@ -26,9 +26,12 @@ spec: { TopCode=$1;Macros=$2;Rules=$4;BottomCode=$5 } } +ident: +| IDENT { $1, parseState.InputRange 1 } + codeopt: | CODE { $1 } -| { "", (parseState.ResultRange |> fst) } +| { "", parseState.ResultRange } Macros: | { [] } @@ -37,8 +40,8 @@ Macros: } macro: -| LET IDENT EQUALS regexp { - ($2, $4) +| LET ident EQUALS regexp { + { Name = $2; Matcher = $4 } } Rules: @@ -48,13 +51,13 @@ Rules: | rule { [$1] } rule: -| IDENT args EQUALS PARSE optbar clauses { - ($1,$2,$6) +| ident args EQUALS PARSE optbar clauses { + { Name = $1; Arguments = $2; Clauses = $6 } } args: | { [] } -| IDENT args { $1 :: $2 } +| ident args { $1 :: $2 } optbar: | { } @@ -65,7 +68,7 @@ clauses: | clause { [$1] } clause: -| regexp CODE { $1, $2 } +| regexp CODE { { Matcher = $1; Code = $2 } } regexp: | CHAR { Inp(Alphabet(EncodeChar $1))} @@ -73,7 +76,7 @@ regexp: | EOF { Inp(Alphabet(fun ctx -> Eof))} | UNDERSCORE { Inp Any } | STRING { Seq([ for n in 0 .. $1.Length - 1 -> Inp(Alphabet(EncodeChar $1.[n]))])} -| IDENT { Macro($1) } +| ident { Macro($1) } | regexp regexp %prec regexp_seq { Seq[$1;$2] } | regexp PLUS %prec regexp_plus { Seq[$1;Star $1] } | regexp STAR %prec regexp_star { Star $1 } diff --git a/src/FsLexYacc.Runtime/Lexing.fs b/src/FsLexYacc.Runtime/Lexing.fs index 990f39bb..c4bb9bc7 100644 --- a/src/FsLexYacc.Runtime/Lexing.fs +++ b/src/FsLexYacc.Runtime/Lexing.fs @@ -6,6 +6,7 @@ module FSharp.Text.Lexing open System.Collections.Generic // REVIEW: This type showed up on a parsing-intensive performance measurement. Consider whether it can be a struct-record later when we have this feature. -jomo + [] type Position = { pos_fname : string @@ -61,6 +62,9 @@ type Position = pos_bol= 0 pos_cnum=0 } +type [] Range = { startPos: Position; endPos: Position } +with static member Empty = { startPos = Position.Empty; endPos = Position.Empty } + type LexBufferFiller<'char> = { fillSync : (LexBuffer<'char> -> unit) option fillAsync : (LexBuffer<'char> -> Async) option } diff --git a/src/FsLexYacc.Runtime/Lexing.fsi b/src/FsLexYacc.Runtime/Lexing.fsi index 9a903107..d24a4bd8 100644 --- a/src/FsLexYacc.Runtime/Lexing.fsi +++ b/src/FsLexYacc.Runtime/Lexing.fsi @@ -70,6 +70,9 @@ type Position = /// Get a position corresponding to the first line (line number 1) in a given file static member FirstLine : filename:string -> Position +type [] Range = { startPos: Position; endPos: Position } +with static member Empty: Range + [] /// Input buffers consumed by lexers generated by fslex.exe type LexBuffer<'char> = diff --git a/src/FsLexYacc.Runtime/Parsing.fs b/src/FsLexYacc.Runtime/Parsing.fs index 1080749f..f104979a 100644 --- a/src/FsLexYacc.Runtime/Parsing.fs +++ b/src/FsLexYacc.Runtime/Parsing.fs @@ -7,13 +7,13 @@ open System open System.Collections.Generic type IParseState = - abstract InputRange: int -> Position * Position + abstract InputRange: int -> Range abstract InputEndPosition: int -> Position abstract InputStartPosition: int -> Position - abstract ResultRange: Position * Position + abstract ResultRange: Range abstract GetInput: int -> obj @@ -254,11 +254,11 @@ module Implementation = let parseState = { new IParseState with - member __.InputRange(n) = ruleStartPoss.[n-1], ruleEndPoss.[n-1] + member __.InputRange(n) = { startPos = ruleStartPoss.[n-1]; endPos = ruleEndPoss.[n-1] } member __.InputStartPosition(n) = ruleStartPoss.[n-1] member __.InputEndPosition(n) = ruleEndPoss.[n-1] member __.GetInput(n) = ruleValues.[n-1] - member __.ResultRange = (lhsPos.[0], lhsPos.[1]) + member __.ResultRange = { startPos = lhsPos.[0]; endPos = lhsPos.[1] } member __.ParserLocalStore = (localStore :> IDictionary<_,_>) member __.RaiseError() = raise RecoverableParseError (* NOTE: this binding tests the fairly complex logic associated with an object expression implementing a generic abstract method *) } @@ -495,8 +495,8 @@ module Implementation = valueStack.Peep().value type Tables<'tok> with - member tables.Interpret (lexer,lexbuf,initialState) = - Implementation.interpret tables lexer lexbuf initialState + member tables.Interpret (lexer,lexbuf, startState) = + Implementation.interpret tables lexer lexbuf startState module ParseHelpers = let parse_error (_s:string) = () diff --git a/src/FsLexYacc.Runtime/Parsing.fsi b/src/FsLexYacc.Runtime/Parsing.fsi index e4e73294..55697f23 100644 --- a/src/FsLexYacc.Runtime/Parsing.fsi +++ b/src/FsLexYacc.Runtime/Parsing.fsi @@ -10,7 +10,7 @@ open System.Collections.Generic /// The information accessible via the parseState value within parser actions. type IParseState = /// Get the start and end position for the terminal or non-terminal at a given index matched by the production - abstract InputRange: index:int -> Position * Position + abstract InputRange: index:int -> Range /// Get the end position for the terminal or non-terminal at a given index matched by the production abstract InputEndPosition: int -> Position @@ -19,7 +19,7 @@ type IParseState = abstract InputStartPosition: int -> Position /// Get the full range of positions matched by the production - abstract ResultRange: Position * Position + abstract ResultRange: Range /// Get the value produced by the terminal or non-terminal at the given position abstract GetInput : int -> obj diff --git a/src/FsYacc.Core/fsyaccast.fs b/src/FsYacc.Core/fsyaccast.fs index 6a3214ad..bf352f6b 100644 --- a/src/FsYacc.Core/fsyaccast.fs +++ b/src/FsYacc.Core/fsyaccast.fs @@ -15,22 +15,22 @@ open FSharp.Text.Lexing let (|KeyValue|) (kvp:KeyValuePair<_,_>) = kvp.Key,kvp.Value -type Identifier = string -type Code = string * Position +type Identifier = string * Range +type Code = string * Range type Associativity = LeftAssoc | RightAssoc | NonAssoc type Rule = Rule of Identifier list * Identifier option * Code option -type ParserSpec= +type ParserSpec= { Header : Code; Tokens : (Identifier * string option) list; Types : (Identifier * string) list; Associativities: (Identifier * Associativity) list list; // suggest to do: (Associativity * Identifier list) list StartSymbols : Identifier list; Rules : (Identifier * Rule list) list } - -type Terminal = string -type NonTerminal = string + +type Terminal = string * Range +type NonTerminal = string * Range type Symbol = Terminal of Terminal | NonTerminal of NonTerminal type Symbols = Symbol list @@ -38,7 +38,7 @@ type Symbols = Symbol list //--------------------------------------------------------------------- // Output Raw Parser Spec AST -let StringOfSym sym = match sym with Terminal s -> "'" ^ s ^ "'" | NonTerminal s -> s +let StringOfSym sym = match sym with Terminal (name, range) -> "'" ^ name ^ "'" | NonTerminal (name, range) -> name let OutputSym os sym = fprintf os "%s" (StringOfSym sym) @@ -48,8 +48,8 @@ let OutputSyms os syms = let OutputTerminalSet os (tset:string seq) = fprintf os "%s" (String.Join(";", tset |> Seq.toArray)) -let OutputAssoc os p = - match p with +let OutputAssoc os p = + match p with | LeftAssoc -> fprintf os "left" | RightAssoc -> fprintf os "right" | NonAssoc -> fprintf os "nonassoc" @@ -58,71 +58,81 @@ let OutputAssoc os p = //--------------------------------------------------------------------- // PreProcess Raw Parser Spec AST -type PrecedenceInfo = - | ExplicitPrec of Associativity * int +type PrecedenceInfo = + | ExplicitPrec of Associativity * int | NoPrecedence - + type Production = Production of NonTerminal * PrecedenceInfo * Symbols * Code option -type ProcessedParserSpec = +type ProcessedParserSpec = { Terminals: (Terminal * PrecedenceInfo) list; NonTerminals: NonTerminal list; Productions: Production list; StartSymbols: NonTerminal list } -let ProcessParserSpecAst (spec: ParserSpec) = - let explicitPrecInfo = - spec.Associativities +let ProcessParserSpecAst (spec: ParserSpec) = + let explicitPrecInfo = + spec.Associativities |> List.mapi (fun n precSpecs -> precSpecs |> List.map (fun (precSym, assoc) -> precSym,ExplicitPrec (assoc, 9999 - n))) |> List.concat - - for (key,_) in explicitPrecInfo |> Seq.countBy fst |> Seq.filter (fun (_,n) -> n > 1) do + + for ((key,_),_) in explicitPrecInfo |> Seq.countBy fst |> Seq.filter (fun (_,n) -> n > 1) do failwithf "%s is given two associativities" key - - let explicitPrecInfo = + + let explicitPrecInfo = explicitPrecInfo |> Map.ofList let implicitSymPrecInfo = NoPrecedence - let terminals = List.map fst spec.Tokens @ ["error"] - let terminalSet = Set.ofList terminals - let IsTerminal z = terminalSet.Contains(z) - let prec_of_terminal sym implicitPrecInfo = + let terminals = List.map fst spec.Tokens @ [ "error", Range.Empty ] + let terminalNamesSet = Set.ofList (terminals |> List.map fst) + let IsTerminal z = terminalNamesSet.Contains(z) + let prec_of_terminal sym implicitPrecInfo = if explicitPrecInfo.ContainsKey(sym) then explicitPrecInfo.[sym] else match implicitPrecInfo with Some x -> x | None -> implicitSymPrecInfo - - let mkSym s = if IsTerminal s then Terminal s else NonTerminal s - let prods = - spec.Rules |> List.mapi (fun i (nonterm,rules) -> - rules |> List.mapi (fun j (Rule(syms,precsym,code)) -> - let precInfo = - let precsym = List.foldBack (fun x acc -> match acc with Some _ -> acc | None -> match x with z when IsTerminal z -> Some z | _ -> acc) syms precsym + + let mkSym ((name, range) as ident) = if IsTerminal name then Terminal ident else NonTerminal ident + let prods = + spec.Rules |> List.mapi (fun i (nonterm,rules) -> + rules |> List.mapi (fun j (Rule(syms,precsym,code)) -> + let precInfo = + let precsym = + (syms, precsym) + ||> List.foldBack (fun x acc -> + match acc with + | Some _ -> acc + | None -> + match x with + | (name, _range) as z when IsTerminal name -> Some z + | _ -> acc + ) let implicitPrecInfo = NoPrecedence - match precsym with - | None -> implicitPrecInfo + match precsym with + | None -> implicitPrecInfo | Some sym -> prec_of_terminal sym None Production(nonterm, precInfo, List.map mkSym syms, code))) |> List.concat let nonTerminals = List.map fst spec.Rules - let nonTerminalSet = Set.ofList nonTerminals - let checkNonTerminal nt = - if nt <> "error" && not (nonTerminalSet.Contains(nt)) then - failwith (sprintf "NonTerminal '%s' has no productions" nt) + let nonTerminalNamesSet = Set.ofList (nonTerminals |> List.map fst) + + let checkNonTerminal (name, range) = + if name <> "error" && not (nonTerminalNamesSet.Contains name) then + failwithf "%s(%d,%d): NonTerminal '%s' has no productions" range.startPos.FileName range.startPos.Line range.startPos.Column name for (Production(nt,_,syms,_)) in prods do - for sym in syms do - match sym with - | NonTerminal nt -> - checkNonTerminal nt - | Terminal t -> - if not (IsTerminal t) then failwith (sprintf "token %s is not declared" t) - - if spec.StartSymbols= [] then (failwith "at least one %start declaration is required"); - - for (nt,_) in spec.Types do + for sym in syms do + match sym with + | NonTerminal nt -> + checkNonTerminal nt + | Terminal (name, range) -> + if not (IsTerminal name) then failwithf "%s(%d,%d) token %s is not declared" range.startPos.FileName range.startPos.Line range.startPos.Column name + + if spec.StartSymbols = [] then (failwith "at least one start declaration is required"); + + for (nt,_) in spec.Types do checkNonTerminal nt; - let terminals = terminals |> List.map (fun t -> (t,prec_of_terminal t None)) + let terminals = terminals |> List.map (fun t -> (t,prec_of_terminal t None)) { Terminals=terminals; NonTerminals=nonTerminals; @@ -136,22 +146,22 @@ let ProcessParserSpecAst (spec: ParserSpec) = type ProductionIndex = int type ProdictionDotIndex = int -/// Represent (ProductionIndex,ProdictionDotIndex) as one integer -type Item0 = uint32 +/// Represent (ProductionIndex,ProdictionDotIndex) as one integer +type Item0 = uint32 let mkItem0 (prodIdx,dotIdx) : Item0 = (uint32 prodIdx <<< 16) ||| uint32 dotIdx let prodIdx_of_item0 (item0:Item0) = int32 (item0 >>> 16) let dotIdx_of_item0 (item0:Item0) = int32 (item0 &&& 0xFFFFu) /// Part of the output of CompilerLalrParserSpec -type Action = +type Action = | Shift of int | Reduce of ProductionIndex | Accept | Error - -let outputPrecInfo os p = - match p with + +let outputPrecInfo os p = + match p with | ExplicitPrec (assoc,n) -> fprintf os "explicit %a %d" OutputAssoc assoc n | NoPrecedence -> fprintf os "noprec" @@ -167,7 +177,7 @@ type TerminalIndex = int type NonTerminalIndex = int /// Representation of Symbols. -/// Ideally would be declared as +/// Ideally would be declared as /// type SymbolIndex = PTerminal of TerminalIndex | PNonTerminal of NonTerminalIndex /// but for performance reasons we embed as a simple integer (saves ~10%) /// @@ -202,22 +212,22 @@ let GotoItemIdx (i1:KernelIdx,i2:SymbolIndex) = (uint64 (uint32 i1) <<< 32) ||| let (|GotoItemIdx|) (i64:uint64) = int32 ((i64 >>> 32) &&& 0xFFFFFFFFUL), int32 (i64 &&& 0xFFFFFFFFUL) /// Create a work list and loop until it is exhausted, calling a worker function for -/// each element. Pass a function to queue additional work on the work list +/// each element. Pass a function to queue additional work on the work list /// to the worker function let ProcessWorkList start f = let work = ref (start : 'a list) let queueWork = (fun x -> work := x :: !work) - let rec loop() = - match !work with + let rec loop() = + match !work with | [] -> () - | x::t -> - work := t; + | x::t -> + work := t; f queueWork x; loop() loop() /// A standard utility to compute a least fixed point of a set under a generative computation -let LeastFixedPoint f set = +let LeastFixedPoint f set = let acc = ref set ProcessWorkList (Set.toList set) (fun queueWork item -> f(item) |> List.iter (fun i2 -> if not (Set.contains i2 !acc) then (acc := Set.add i2 !acc; queueWork i2)) ) @@ -225,35 +235,35 @@ let LeastFixedPoint f set = /// A general standard memoization utility. Be sure to apply to only one (function) argument to build the /// residue function! -let Memoize f = +let Memoize f = let t = new Dictionary<_,_>(1000) - fun x -> - let ok,v = t.TryGetValue(x) - if ok then v else let res = f x in t.[x] <- res; res + fun x -> + let ok,v = t.TryGetValue(x) + if ok then v else let res = f x in t.[x] <- res; res /// A standard utility to create a dictionary from a list of pairs -let CreateDictionary xs = +let CreateDictionary xs = let dict = new Dictionary<_,_>() for x,y in xs do dict.Add(x,y) dict /// Allocate indexes for each non-terminal -type NonTerminalTable(nonTerminals:NonTerminal list) = +type NonTerminalTable(nonTerminals:NonTerminal list) = let nonterminalsWithIdxs = List.mapi (fun (i:NonTerminalIndex) n -> (i,n)) nonTerminals let nonterminalIdxs = List.map fst nonterminalsWithIdxs let a = Array.ofList nonTerminals - let b = CreateDictionary [ for i,x in nonterminalsWithIdxs -> x,i ]; + let b = CreateDictionary [ for i,(name, range) in nonterminalsWithIdxs -> name,i ]; member table.OfIndex(i) = a.[i] member table.ToIndex(i) = b.[i] member table.Indexes = nonterminalIdxs /// Allocate indexes for each terminal -type TerminalTable(terminals:(Terminal * PrecedenceInfo) list) = +type TerminalTable(terminals:(Terminal * PrecedenceInfo) list) = let terminalsWithIdxs = List.mapi (fun i (t,_) -> (i,t)) terminals let terminalIdxs = List.map fst terminalsWithIdxs let a = Array.ofList (List.map fst terminals) let b = Array.ofList (List.map snd terminals) - let c = CreateDictionary [ for i,x in terminalsWithIdxs -> x,i ] + let c = CreateDictionary [ for i, (name, range) in terminalsWithIdxs -> name, i ] member table.OfIndex(i) = a.[i] member table.PrecInfoOfIndex(i) = b.[i] @@ -261,48 +271,48 @@ type TerminalTable(terminals:(Terminal * PrecedenceInfo) list) = member table.Indexes = terminalIdxs /// Allocate indexes for each production -type ProductionTable(ntTab:NonTerminalTable, termTab:TerminalTable, nonTerminals:string list, prods: Production list) = +type ProductionTable(ntTab: NonTerminalTable, termTab: TerminalTable, nonTerminals: NonTerminal list, prods: Production list) = let prodsWithIdxs = List.mapi (fun i n -> (i,n)) prods - let a = + let a = prodsWithIdxs - |> List.map(fun (_,Production(_,_,syms,_)) -> - syms - |> Array.ofList - |> Array.map (function - | Terminal t -> PTerminal (termTab.ToIndex t) - | NonTerminal nt -> PNonTerminal (ntTab.ToIndex nt )) ) + |> List.map(fun (_,Production(_,_,syms,_)) -> + syms + |> Array.ofList + |> Array.map (function + | Terminal (name, _) -> PTerminal (termTab.ToIndex name) + | NonTerminal (name, _) -> PNonTerminal (ntTab.ToIndex name )) ) |> Array.ofList - let b = Array.ofList (List.map (fun (_,Production(nt,_,_,_)) -> ntTab.ToIndex nt) prodsWithIdxs) + let b = Array.ofList (List.map (fun (_,Production((name, _),_,_,_)) -> ntTab.ToIndex name) prodsWithIdxs) let c = Array.ofList (List.map (fun (_,Production(_,prec,_,_)) -> prec) prodsWithIdxs) - let productions = + let productions = nonTerminals - |> List.map(fun nt -> (ntTab.ToIndex nt, List.choose (fun (i,Production(nt2,prec,syms,_)) -> if nt2=nt then Some i else None) prodsWithIdxs)) + |> List.map(fun (name1, range1) -> (ntTab.ToIndex name1, List.choose (fun (i, Production((name2, range2),prec,syms,_)) -> if name2 = name1 then Some i else None) prodsWithIdxs)) |> CreateDictionary member prodTab.Symbols(i) = a.[i] member prodTab.NonTerminal(i) = b.[i] member prodTab.Precedence(i) = c.[i] - member prodTab.Symbol i n = + member prodTab.Symbol i n = let syms = prodTab.Symbols i if n >= syms.Length then None else Some (syms.[n]) member prodTab.Productions = productions /// A mutable table maping kernels to sets of lookahead tokens -type LookaheadTable() = +type LookaheadTable() = let t = new Dictionary>() - member table.Add(x,y) = - let prev = if t.ContainsKey(x) then t.[x] else Set.empty + member table.Add(x,y) = + let prev = if t.ContainsKey(x) then t.[x] else Set.empty t.[x] <- prev.Add(y) member table.Contains(x,y) = t.ContainsKey(x) && t.[x].Contains(y) - member table.GetLookaheads(idx:KernelItemIndex) = - let ok,v = t.TryGetValue(idx) + member table.GetLookaheads(idx:KernelItemIndex) = + let ok,v = t.TryGetValue(idx) if ok then v else Set.empty member table.Count = t |> Seq.fold(fun acc (KeyValue(_,v)) -> v.Count+acc) 0 /// A mutable table giving an index to each LR(0) kernel. Kernels are referred to only by index. type KernelTable(kernels) = - // Give an index to each LR(0) kernel, and from now on refer to them only by index - // Also develop "kernelItemIdx" to refer to individual items within a kernel + // Give an index to each LR(0) kernel, and from now on refer to them only by index + // Also develop "kernelItemIdx" to refer to individual items within a kernel let kernelsAndIdxs = List.mapi (fun i x -> (i,x)) kernels let kernelIdxs = List.map fst kernelsAndIdxs let toIdxMap = Map.ofList [ for i,x in kernelsAndIdxs -> x,i ] @@ -312,9 +322,9 @@ type KernelTable(kernels) = member t.Kernel(i) = ofIdxMap.[i] /// Hold the results of cpmuting the LALR(1) closure of an LR(0) kernel -type Closure1Table() = +type Closure1Table() = let t = new Dictionary>() - member table.Add(a,b) = + member table.Add(a,b) = if not (t.ContainsKey(a)) then t.[a] <- new HashSet<_>(HashIdentity.Structural) t.[a].Add(b) member table.Count = t.Count @@ -323,9 +333,9 @@ type Closure1Table() = /// A mutable table giving a lookahead set Set for each kernel. The terminals represent the /// "spontaneous" items for the kernel. TODO: document this more w.r.t. the Dragon book. -type SpontaneousTable() = +type SpontaneousTable() = let t = new Dictionary>() - member table.Add(a,b) = + member table.Add(a,b) = if not (t.ContainsKey(a)) then t.[a] <- new HashSet<_>(HashIdentity.Structural) t.[a].Add(b) member table.Count = t.Count @@ -333,14 +343,14 @@ type SpontaneousTable() = /// A mutable table giving a Set for each kernel. The kernels represent the /// "propagate" items for the kernel. TODO: document this more w.r.t. the Dragon book. -type PropagateTable() = +type PropagateTable() = let t = new Dictionary>() - member table.Add(a,b) = + member table.Add(a,b) = if not (t.ContainsKey(a)) then t.[a] <- new HashSet(HashIdentity.Structural) t.[a].Add(b) - member table.Item - with get(a) = - let ok,v = t.TryGetValue(a) + member table.Item + with get(a) = + let ok,v = t.TryGetValue(a) if ok then v :> seq<_> else Seq.empty member table.Count = t.Count @@ -348,7 +358,7 @@ type PropagateTable() = type Prod = NonTerminal * int * Symbols * option type ActionTable = (PrecedenceInfo * Action) array array -type CompiledSpec = +type CompiledSpec = { prods: Prod [] states: int list [] startStates: int list @@ -357,7 +367,7 @@ type CompiledSpec = gotoTable: int option [] [] endOfInputTerminalIdx: int errorTerminalIdx: int - nonTerminals: string list + nonTerminals: NonTerminal list } /// Compile a pre-processed LALR parser spec to tables following the Dragon book algorithm @@ -366,174 +376,178 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = let reportTime() = printfn " time: %A" stopWatch.Elapsed; stopWatch.Reset(); stopWatch.Start() stopWatch.Start() - // Augment the grammar - let fakeStartNonTerminals = spec.StartSymbols |> List.map(fun nt -> "_start"^nt) - let nonTerminals = fakeStartNonTerminals@spec.NonTerminals - let endOfInputTerminal = "$$" - let dummyLookahead = "#" + // Augment the grammar + let fakeStartNonTerminals = spec.StartSymbols |> List.map(fun (name, r) -> "_start" ^ name, r) + let nonTerminals = fakeStartNonTerminals @ spec.NonTerminals + let endOfInputTerminal = "$$", Range.Empty + let dummyLookahead = "#", Range.Empty let dummyPrec = NoPrecedence - let terminals = spec.Terminals @ [(dummyLookahead,dummyPrec); (endOfInputTerminal,dummyPrec)] - let prods = List.map2 (fun a b -> Production(a, dummyPrec,[NonTerminal b],None)) fakeStartNonTerminals spec.StartSymbols @ spec.Productions + let terminals = + spec.Terminals @ [(dummyLookahead, dummyPrec); (endOfInputTerminal, dummyPrec)] + let prods = + List.map2 (fun a b -> Production(a, dummyPrec, [NonTerminal b], None)) fakeStartNonTerminals spec.StartSymbols + @ spec.Productions let startNonTerminalIdx_to_prodIdx (i:int) = i - // Build indexed tables + // Build indexed tables let ntTab = NonTerminalTable(nonTerminals) let termTab = TerminalTable(terminals) - let prodTab = ProductionTable(ntTab,termTab,nonTerminals,prods) - let dummyLookaheadIdx = termTab.ToIndex dummyLookahead - let endOfInputTerminalIdx = termTab.ToIndex endOfInputTerminal + let prodTab = ProductionTable(ntTab, termTab, nonTerminals, prods) + let dummyLookaheadIdx = termTab.ToIndex (fst dummyLookahead) + let endOfInputTerminalIdx = termTab.ToIndex (fst endOfInputTerminal) let errorTerminalIdx = termTab.ToIndex "error" // Compute the FIRST function printf "computing first function..."; stdout.Flush(); - let computedFirstTable = - let seed = + let computedFirstTable = + let seed = Map.ofList [ for term in termTab.Indexes do yield (PTerminal(term),Set.singleton (Some term)) - for nonTerm in ntTab.Indexes do - yield - (PNonTerminal nonTerm, - List.foldBack - (fun prodIdx acc -> match prodTab.Symbol prodIdx 0 with None -> Set.add None acc | Some _ -> acc) - prodTab.Productions.[nonTerm] + for nonTerm in ntTab.Indexes do + yield + (PNonTerminal nonTerm, + List.foldBack + (fun prodIdx acc -> match prodTab.Symbol prodIdx 0 with None -> Set.add None acc | Some _ -> acc) + prodTab.Productions.[nonTerm] Set.empty) ] - - let add changed ss (x,y) = + + let add changed ss (x,y) = let s = Map.find x ss - if Set.contains y s then ss + if Set.contains y s then ss else (changed := true; Map.add x (Set.add y s) ss) - let oneRound (ss:Map<_,_>) = + let oneRound (ss:Map<_,_>) = let changed = ref false - let frontier = + let frontier = let res = ref [] - for nonTermX in ntTab.Indexes do + for nonTermX in ntTab.Indexes do for prodIdx in prodTab.Productions.[nonTermX] do let rhs = Array.toList (prodTab.Symbols prodIdx) let rec place l = match l with - | (yi::t) -> - res := - List.choose - (function None -> None | Some a -> Some (PNonTerminal nonTermX,Some a)) - (Set.toList ss.[yi]) + | (yi::t) -> + res := + List.choose + (function None -> None | Some a -> Some (PNonTerminal nonTermX,Some a)) + (Set.toList ss.[yi]) @ !res; if ss.[yi].Contains(None) then place t; - | [] -> + | [] -> res := (PNonTerminal nonTermX,None) :: !res place rhs !res let ss' = List.fold (add changed) ss frontier !changed, ss' - let rec loop ss = + let rec loop ss = let changed, ss' = oneRound ss if changed then loop ss' else ss' - loop seed - - + loop seed + + /// Compute the first set of the given sequence of non-terminals. If any of the non-terminals - /// have an empty token in the first set then we have to iterate through those. + /// have an empty token in the first set then we have to iterate through those. let ComputeFirstSetOfTokenList = - Memoize (fun (str,term) -> + Memoize (fun (str,term) -> let acc = new System.Collections.Generic.List<_>() - let rec add l = - match l with + let rec add l = + match l with | [] -> acc.Add(term) - | sym::moreSyms -> + | sym::moreSyms -> let firstSetOfSym = computedFirstTable.[sym] - firstSetOfSym |> Set.iter (function None -> () | Some v -> acc.Add(v)) - if firstSetOfSym.Contains(None) then add moreSyms + firstSetOfSym |> Set.iter (function None -> () | Some v -> acc.Add(v)) + if firstSetOfSym.Contains(None) then add moreSyms add str; Set.ofSeq acc) - - // (int,int) representation of LR(0) items - let prodIdx_to_item0 idx = mkItem0(idx,0) + + // (int,int) representation of LR(0) items + let prodIdx_to_item0 idx = mkItem0(idx,0) let prec_of_item0 item0 = prodTab.Precedence (prodIdx_of_item0 item0) let ntIdx_of_item0 item0 = prodTab.NonTerminal (prodIdx_of_item0 item0) - let lsyms_of_item0 item0 = + let lsyms_of_item0 item0 = let prodIdx = prodIdx_of_item0 item0 let dotIdx = dotIdx_of_item0 item0 let syms = prodTab.Symbols prodIdx if dotIdx <= 0 then [||] else syms.[..dotIdx-1] - let rsyms_of_item0 item0 = + let rsyms_of_item0 item0 = let prodIdx = prodIdx_of_item0 item0 let dotIdx = dotIdx_of_item0 item0 let syms = prodTab.Symbols prodIdx syms.[dotIdx..] - let rsym_of_item0 item0 = + let rsym_of_item0 item0 = let prodIdx = prodIdx_of_item0 item0 let dotIdx = dotIdx_of_item0 item0 prodTab.Symbol prodIdx dotIdx - let advance_of_item0 item0 = + let advance_of_item0 item0 = let prodIdx = prodIdx_of_item0 item0 let dotIdx = dotIdx_of_item0 item0 mkItem0(prodIdx,dotIdx+1) - let fakeStartNonTerminalsSet = Set.ofList (fakeStartNonTerminals |> List.map ntTab.ToIndex) + + let fakeStartNonTerminalsSet = Set.ofList (fakeStartNonTerminals |> List.map (fst >> ntTab.ToIndex)) let IsStartItem item0 = fakeStartNonTerminalsSet.Contains(ntIdx_of_item0 item0) let IsKernelItem item0 = (IsStartItem item0 || dotIdx_of_item0 item0 <> 0) - let StringOfSym sym = match sym with PTerminal s -> "'" ^ termTab.OfIndex s ^ "'" | PNonTerminal s -> ntTab.OfIndex s + let StringOfSym sym = match sym with PTerminal s -> "'" ^ fst (termTab.OfIndex s) ^ "'" | PNonTerminal s -> fst (ntTab.OfIndex s) let OutputSym os sym = fprintf os "%s" (StringOfSym sym) let OutputSyms os syms = fprintf os "%s" (String.Join(" ",Array.map StringOfSym syms)) - // Print items and other stuff + // Print items and other stuff let OutputItem0 os item0 = - fprintf os " %s -> %a . %a" (ntTab.OfIndex (ntIdx_of_item0 item0)) (* outputPrecInfo precInfo *) OutputSyms (lsyms_of_item0 item0) OutputSyms (rsyms_of_item0 item0) - - let OutputItem0Set os s = + fprintf os " %s -> %a . %a" (fst (ntTab.OfIndex (ntIdx_of_item0 item0))) (* outputPrecInfo precInfo *) OutputSyms (lsyms_of_item0 item0) OutputSyms (rsyms_of_item0 item0) + + let OutputItem0Set os s = Set.iter (fun item -> fprintfn os "%a" OutputItem0 item) s - let OutputFirstSet os m = + let OutputFirstSet os m = Set.iter (function None -> fprintf os "" | Some x -> fprintfn os " term %s" x) m - let OutputFirstMap os m = + let OutputFirstMap os m = Map.iter (fun x y -> fprintf os "first '%a' = " OutputSym x; fprintfn os "%a" OutputFirstSet y) m - let OutputAction os m = - match m with - | Shift n -> fprintf os " shift %d" n - | Reduce prodIdx -> fprintf os " reduce %s --> %a" (ntTab.OfIndex (prodTab.NonTerminal prodIdx)) OutputSyms (prodTab.Symbols prodIdx) + let OutputAction os m = + match m with + | Shift n -> fprintf os " shift %d" n + | Reduce prodIdx -> fprintf os " reduce %s --> %a" (fst (ntTab.OfIndex (prodTab.NonTerminal prodIdx))) OutputSyms (prodTab.Symbols prodIdx) | Error -> fprintf os " error" - | Accept -> fprintf os " accept" - - let OutputActions os m = - Array.iteri (fun i (prec,action) -> let term = termTab.OfIndex i in fprintfn os " action '%s' (%a): %a" term outputPrecInfo prec OutputAction action) m + | Accept -> fprintf os " accept" + + let OutputActions os m = + Array.iteri (fun i (prec,action) -> let (name, range) = termTab.OfIndex i in fprintfn os " action '%s' (%a): %a" name outputPrecInfo prec OutputAction action) m - let OutputActionTable os m = + let OutputActionTable os m = Array.iteri (fun i n -> fprintfn os "state %d:" i; fprintfn os "%a" OutputActions n) m - let OutputImmediateActions os m = - match m with + let OutputImmediateActions os m = + match m with | None -> fprintf os "" | Some a -> OutputAction os a - - let OutputGotos os m = - Array.iteri (fun ntIdx s -> let nonterm = ntTab.OfIndex ntIdx in match s with Some st -> fprintfn os " goto %s: %d" nonterm st | None -> ()) m - - let OutputCombined os m = - Array.iteri (fun i (a,b,c,d) -> + + let OutputGotos os m = + Array.iteri (fun ntIdx s -> let (name, range) = ntTab.OfIndex ntIdx in match s with Some st -> fprintfn os " goto %s: %d" name st | None -> ()) m + + let OutputCombined os m = + Array.iteri (fun i (a,b,c,d) -> fprintf os "state %d:" i fprintf os " items:" - fprintf os "%a" OutputItem0Set a + fprintf os "%a" OutputItem0Set a fprintf os " actions:" - fprintf os "%a" OutputActions b + fprintf os "%a" OutputActions b fprintf os " immediate action: " - fprintf os "%a" OutputImmediateActions c + fprintf os "%a" OutputImmediateActions c fprintf os " gotos:" fprintf os "%a" OutputGotos d) m - - let OutputLalrTables os (prods,states, startStates,actionTable,immediateActionTable,gotoTable,endOfInputTerminalIdx,errorTerminalIdx) = + + let OutputLalrTables os (prods,states, startStates,actionTable,immediateActionTable,gotoTable,endOfInputTerminalIdx,errorTerminalIdx) = let combined = Array.ofList (List.map2 (fun x (y,(z,w)) -> x,y,z,w) (Array.toList states) (List.zip (Array.toList actionTable) (List.zip (Array.toList immediateActionTable) (Array.toList gotoTable)))) fprintfn os "------------------------"; fprintfn os "states = "; @@ -542,12 +556,12 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = fprintfn os "------------------------" - // Closure of LR(0) nonTerminals, items etc - let ComputeClosure0NonTerminal = - Memoize (fun nt -> + // Closure of LR(0) nonTerminals, items etc + let ComputeClosure0NonTerminal = + Memoize (fun nt -> let seed = (List.foldBack (prodIdx_to_item0 >> Set.add) prodTab.Productions.[nt] Set.empty) - LeastFixedPoint - (fun item0 -> + LeastFixedPoint + (fun item0 -> match rsym_of_item0 item0 with | None -> [] | Some(PNonTerminal ntB) -> List.map prodIdx_to_item0 prodTab.Productions.[ntB] @@ -555,52 +569,52 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = seed) // Close a symbol under epsilon moves - let ComputeClosure0Symbol rsym acc = + let ComputeClosure0Symbol rsym acc = match rsym with | Some (PNonTerminal nt) -> Set.union (ComputeClosure0NonTerminal nt) acc | _ -> acc // Close a set under epsilon moves - let ComputeClosure0 iset = - Set.fold (fun acc x -> ComputeClosure0Symbol (rsym_of_item0 x) acc) iset iset + let ComputeClosure0 iset = + Set.fold (fun acc x -> ComputeClosure0Symbol (rsym_of_item0 x) acc) iset iset // Right symbols after closing under epsilon moves let RelevantSymbolsOfKernel kernel = let kernelClosure0 = ComputeClosure0 kernel - Set.fold (fun acc x -> Option.fold (fun acc x -> Set.add x acc) acc (rsym_of_item0 x)) Set.empty kernelClosure0 + Set.fold (fun acc x -> Option.fold (fun acc x -> Set.add x acc) acc (rsym_of_item0 x)) Set.empty kernelClosure0 - // Goto set of a kernel of LR(0) nonTerminals, items etc + // Goto set of a kernel of LR(0) nonTerminals, items etc // Input is kernel, output is kernel - let ComputeGotosOfKernel iset sym = + let ComputeGotosOfKernel iset sym = let isetClosure = ComputeClosure0 iset let acc = new System.Collections.Generic.List<_>(10) - isetClosure |> Set.iter (fun item0 -> - match rsym_of_item0 item0 with - | Some sym2 when sym = sym2 -> acc.Add(advance_of_item0 item0) - | _ -> ()) + isetClosure |> Set.iter (fun item0 -> + match rsym_of_item0 item0 with + | Some sym2 when sym = sym2 -> acc.Add(advance_of_item0 item0) + | _ -> ()) Set.ofSeq acc - - // Build the full set of LR(0) kernels + + // Build the full set of LR(0) kernels reportTime(); printf "building kernels..."; stdout.Flush(); let startItems = List.mapi (fun i _ -> prodIdx_to_item0 (startNonTerminalIdx_to_prodIdx i)) fakeStartNonTerminals let startKernels = List.map Set.singleton startItems - let kernels = + let kernels = /// We use a set-of-sets here. F# sets support structural comparison but at the time of writing - /// did not structural hashing. + /// did not structural hashing. let acc = ref Set.empty - ProcessWorkList startKernels (fun addToWorkList kernel -> + ProcessWorkList startKernels (fun addToWorkList kernel -> if not ((!acc).Contains(kernel)) then acc := (!acc).Add(kernel); - for csym in RelevantSymbolsOfKernel kernel do - let gotoKernel = ComputeGotosOfKernel kernel csym + for csym in RelevantSymbolsOfKernel kernel do + let gotoKernel = ComputeGotosOfKernel kernel csym assert (gotoKernel.Count > 0) addToWorkList gotoKernel ) - + !acc |> Seq.toList |> List.map (Set.filter IsKernelItem) - + reportTime(); printf "building kernel table..."; stdout.Flush(); - // Give an index to each LR(0) kernel, and from now on refer to them only by index + // Give an index to each LR(0) kernel, and from now on refer to them only by index let kernelTab = new KernelTable(kernels) let startKernelIdxs = List.map kernelTab.Index startKernels let startKernelItemIdxs = List.map2 (fun a b -> KernelItemIdx(a,b)) startKernelIdxs startItems @@ -608,93 +622,93 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = let outputKernelItemIdx os (kernelIdx,item0) = fprintf os "kernel %d, item %a" kernelIdx OutputItem0 item0 - /// A cached version of the "goto" computation on LR(0) kernels - let gotoKernel = - Memoize (fun (GotoItemIdx(kernelIdx,sym)) -> + /// A cached version of the "goto" computation on LR(0) kernels + let gotoKernel = + Memoize (fun (GotoItemIdx(kernelIdx,sym)) -> let gset = ComputeGotosOfKernel (kernelTab.Kernel kernelIdx) sym if gset.IsEmpty then None else Some (kernelTab.Index gset)) /// Iterate (iset,sym) pairs such that (gotoKernel kernelIdx sym) is not empty let IterateGotosOfKernel kernelIdx f = - for sym in RelevantSymbolsOfKernel (kernelTab.Kernel kernelIdx) do - match gotoKernel (GotoItemIdx(kernelIdx,sym)) with + for sym in RelevantSymbolsOfKernel (kernelTab.Kernel kernelIdx) do + match gotoKernel (GotoItemIdx(kernelIdx,sym)) with | None -> () | Some k -> f sym k - - // This is used to compute the closure of an LALR(1) kernel + + // This is used to compute the closure of an LALR(1) kernel // // For each item [A --> X.BY, a] in I // For each production B -> g in G' // For each terminal b in FIRST(Ya) // such that [B --> .g, b] is not in I do // add [B --> .g, b] to I - - let ComputeClosure1 iset = + + let ComputeClosure1 iset = let acc = new Closure1Table() ProcessWorkList iset (fun addToWorkList (item0,pretokens:Set) -> - pretokens |> Set.iter (fun pretoken -> + pretokens |> Set.iter (fun pretoken -> if not (acc.Contains(item0,pretoken)) then acc.Add(item0,pretoken) |> ignore - let rsyms = rsyms_of_item0 item0 - if rsyms.Length > 0 then - match rsyms.[0] with - | (PNonTerminal ntB) -> + let rsyms = rsyms_of_item0 item0 + if rsyms.Length > 0 then + match rsyms.[0] with + | (PNonTerminal ntB) -> let firstSet = ComputeFirstSetOfTokenList (Array.toList rsyms.[1..],pretoken) for prodIdx in prodTab.Productions.[ntB] do addToWorkList (prodIdx_to_item0 prodIdx,firstSet) | PTerminal _ -> ())) acc - // Compute the "spontaneous" and "propagate" maps for each LR(0) kernelItem + // Compute the "spontaneous" and "propagate" maps for each LR(0) kernelItem // // Input: The kernal K of a set of LR(0) items I and a grammar symbol X // - // Output: The lookaheads generated spontaneously by items in I for kernel items + // Output: The lookaheads generated spontaneously by items in I for kernel items // in goto(I,X) and the items I from which lookaheads are propagated to kernel // items in goto(I,X) // // Method // 1. Construct LR(0) kernel items (done - above) - // 2. - // TODO: this is very, very slow. + // 2. + // TODO: this is very, very slow. // // PLAN TO OPTIMIZE THIS; // - Clarify and comment what's going on here // - verify if we really have to do these enormouos closure computations // - assess if it's possible to use the symbol we're looking for to help trim the jset - + reportTime(); printf "computing lookahead relations..."; stdout.Flush(); - + let spontaneous, propagate = - let closure1OfItem0WithDummy = + let closure1OfItem0WithDummy = Memoize (fun item0 -> ComputeClosure1 [(item0,Set.ofList [dummyLookaheadIdx])]) let spontaneous = new SpontaneousTable() let propagate = new PropagateTable() - let count = ref 0 + let count = ref 0 for kernelIdx in kernelTab.Indexes do printf "."; stdout.Flush(); //printf "kernelIdx = %d\n" kernelIdx; stdout.Flush(); let kernel = kernelTab.Kernel(kernelIdx) - for item0 in kernel do + for item0 in kernel do let item0Idx = KernelItemIdx(kernelIdx,item0) let jset = closure1OfItem0WithDummy item0 //printf "#jset = %d\n" jset.Count; stdout.Flush(); for (KeyValue(closureItem0, lookaheadTokens)) in jset.IEnumerable do incr count - match rsym_of_item0 closureItem0 with + match rsym_of_item0 closureItem0 with | None -> () | Some rsym -> - match gotoKernel (GotoItemIdx(kernelIdx,rsym)) with + match gotoKernel (GotoItemIdx(kernelIdx,rsym)) with | None -> () | Some gotoKernelIdx -> let gotoItem = advance_of_item0 closureItem0 let gotoItemIdx = KernelItemIdx(gotoKernelIdx,gotoItem) for lookaheadToken in lookaheadTokens do - if lookaheadToken = dummyLookaheadIdx + if lookaheadToken = dummyLookaheadIdx then propagate.Add(item0Idx, gotoItemIdx) |> ignore else spontaneous.Add(gotoItemIdx, lookaheadToken) |> ignore @@ -702,14 +716,14 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = //printfn "#kernelIdxs = %d, count = %d" kernelTab.Indexes.Length !count spontaneous, propagate - + //printfn "#spontaneous = %d, #propagate = %d" spontaneous.Count propagate.Count; stdout.Flush(); - + //exit 0; - // Repeatedly use the "spontaneous" and "propagate" maps to build the full set - // of lookaheads for each LR(0) kernelItem. + // Repeatedly use the "spontaneous" and "propagate" maps to build the full set + // of lookaheads for each LR(0) kernelItem. reportTime(); printf "building lookahead table..."; stdout.Flush(); - let lookaheadTable = + let lookaheadTable = // Seed the table with the startKernelItems and the spontaneous info let initialWork = @@ -721,12 +735,12 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = let acc = new LookaheadTable() // Compute the closure - ProcessWorkList + ProcessWorkList initialWork (fun queueWork (kernelItemIdx,lookahead) -> acc.Add(kernelItemIdx,lookahead) for gotoKernelIdx in propagate.[kernelItemIdx] do - if not (acc.Contains(gotoKernelIdx,lookahead)) then + if not (acc.Contains(gotoKernelIdx,lookahead)) then queueWork(gotoKernelIdx,lookahead)) acc @@ -735,24 +749,24 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = reportTime(); printf "building action table..."; stdout.Flush(); let shiftReduceConflicts = ref 0 let reduceReduceConflicts = ref 0 - let actionTable, immediateActionTable = + let actionTable, immediateActionTable = - // Now build the action tables. First a utility to merge the given action - // into the table, taking into account precedences etc. and reporting errors. - let addResolvingPrecedence (arr: _[]) kernelIdx termIdx (precNew, actionNew) = - // printf "DEBUG: state %d: adding action for %s, precNew = %a, actionNew = %a\n" kernelIdx (termTab.OfIndex termIdx) outputPrec precNew OutputAction actionNew; - // We add in order of precedence - however the precedences may be the same, and we give warnings when rpecedence resolution is based on implicit file orderings + // Now build the action tables. First a utility to merge the given action + // into the table, taking into account precedences etc. and reporting errors. + let addResolvingPrecedence (arr: _[]) kernelIdx termIdx (precNew, actionNew) = + // printf "DEBUG: state %d: adding action for %s, precNew = %a, actionNew = %a\n" kernelIdx (termTab.OfIndex termIdx) outputPrec precNew OutputAction actionNew; + // We add in order of precedence - however the precedences may be the same, and we give warnings when rpecedence resolution is based on implicit file orderings let (precSoFar, actionSoFar) as itemSoFar = arr.[termIdx] - // printf "DEBUG: state %d: adding action for %s, precNew = %a, precSoFar = %a, actionSoFar = %a\n" kernelIdx (termTab.OfIndex termIdx) outputPrec precNew outputPrec precSoFar OutputAction actionSoFar; - // if compare_prec precSoFar precNew = -1 then failwith "addResolvingPrecedence"; + // printf "DEBUG: state %d: adding action for %s, precNew = %a, precSoFar = %a, actionSoFar = %a\n" kernelIdx (termTab.OfIndex termIdx) outputPrec precNew outputPrec precSoFar OutputAction actionSoFar; + // if compare_prec precSoFar precNew = -1 then failwith "addResolvingPrecedence"; - let itemNew = (precNew, actionNew) - let winner = + let itemNew = (precNew, actionNew) + let winner = let reportConflict x1 x2 reason = let reportAction (p, a) = - let an, astr = + let an, astr = match a with | Shift x -> "shift", sprintf "shift(%d)" x | Reduce x -> @@ -760,39 +774,39 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = "reduce", prodTab.Symbols x |> Array.map StringOfSym |> String.concat " " - |> sprintf "reduce(%s:%s)" (ntTab.OfIndex nt) + |> sprintf "reduce(%s:%s)" (fst (ntTab.OfIndex nt)) | _ -> "", "" - let pstr = - match p with - | ExplicitPrec (assoc,n) -> - let astr = - match assoc with + let pstr = + match p with + | ExplicitPrec (assoc,n) -> + let astr = + match assoc with | LeftAssoc -> "left" | RightAssoc -> "right" | NonAssoc -> "nonassoc" sprintf "[explicit %s %d]" astr n - | NoPrecedence -> + | NoPrecedence -> "noprec" an, "{" + pstr + " " + astr + "}" let a1n, astr1 = reportAction x1 let a2n, astr2 = reportAction x2 - printfn " %s/%s error at state %d on terminal %s between %s and %s - assuming the former because %s" a1n a2n kernelIdx (termTab.OfIndex termIdx) astr1 astr2 reason - match itemSoFar,itemNew with - | (_,Shift s1),(_, Shift s2) -> - if actionSoFar <> actionNew then + printfn " %s/%s error at state %d on terminal %s between %s and %s - assuming the former because %s" a1n a2n kernelIdx (fst (termTab.OfIndex termIdx)) astr1 astr2 reason + match itemSoFar,itemNew with + | (_,Shift s1),(_, Shift s2) -> + if actionSoFar <> actionNew then reportConflict itemSoFar itemNew "internal error" itemSoFar - | (((precShift,Shift sIdx) as shiftItem), + | (((precShift,Shift sIdx) as shiftItem), ((precReduce,Reduce prodIdx) as reduceItem)) - | (((precReduce,Reduce prodIdx) as reduceItem), - ((precShift,Shift sIdx) as shiftItem)) -> - match precReduce, precShift with - | (ExplicitPrec (_,p1), ExplicitPrec(assocNew,p2)) -> + | (((precReduce,Reduce prodIdx) as reduceItem), + ((precShift,Shift sIdx) as shiftItem)) -> + match precReduce, precShift with + | (ExplicitPrec (_,p1), ExplicitPrec(assocNew,p2)) -> if p1 < p2 then shiftItem elif p1 > p2 then reduceItem else - match assocNew with + match assocNew with | LeftAssoc -> reduceItem | RightAssoc -> shiftItem | NonAssoc -> @@ -803,24 +817,24 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = reportConflict shiftItem reduceItem "we prefer shift when unable to compare precedences" incr shiftReduceConflicts; shiftItem - | ((_,Reduce prodIdx1),(_, Reduce prodIdx2)) -> + | ((_,Reduce prodIdx1),(_, Reduce prodIdx2)) -> "we prefer the rule earlier in the file" |> if prodIdx1 < prodIdx2 then reportConflict itemSoFar itemNew else reportConflict itemNew itemSoFar incr reduceReduceConflicts; if prodIdx1 < prodIdx2 then itemSoFar else itemNew - | _ -> itemNew + | _ -> itemNew arr.[termIdx] <- winner - - // This build the action table for one state. - let ComputeActions kernelIdx = + + // This build the action table for one state. + let ComputeActions kernelIdx = let kernel = kernelTab.Kernel kernelIdx let arr = Array.create terminals.Length (NoPrecedence,Error) //printf "building lookahead table LR(1) items for kernelIdx %d\n" kernelIdx; stdout.Flush(); // Compute the LR(1) items based on lookaheads - let items = + let items = [ for item0 in kernel do let kernelItemIdx = KernelItemIdx(kernelIdx,item0) let lookaheads = lookaheadTable.GetLookaheads(kernelItemIdx) @@ -830,37 +844,37 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = for (KeyValue(item0,lookaheads)) in items.IEnumerable do let nonTermA = ntIdx_of_item0 item0 - match rsym_of_item0 item0 with - | Some (PTerminal termIdx) -> + match rsym_of_item0 item0 with + | Some (PTerminal termIdx) -> let action = - match gotoKernel (GotoItemIdx(kernelIdx,PTerminal termIdx)) with + match gotoKernel (GotoItemIdx(kernelIdx,PTerminal termIdx)) with | None -> failwith "action on terminal should have found a non-empty goto state" | Some gkernelItemIdx -> Shift gkernelItemIdx let prec = termTab.PrecInfoOfIndex termIdx - addResolvingPrecedence arr kernelIdx termIdx (prec, action) + addResolvingPrecedence arr kernelIdx termIdx (prec, action) | None -> for lookahead in lookaheads do if not (IsStartItem(item0)) then let prodIdx = prodIdx_of_item0 item0 let prec = prec_of_item0 item0 let action = (prec, Reduce prodIdx) - addResolvingPrecedence arr kernelIdx lookahead action + addResolvingPrecedence arr kernelIdx lookahead action elif lookahead = endOfInputTerminalIdx then let prec = prec_of_item0 item0 let action = (prec,Accept) - addResolvingPrecedence arr kernelIdx lookahead action + addResolvingPrecedence arr kernelIdx lookahead action else () | _ -> () - // If there is a single item A -> B C . and no Shift or Accept actions (i.e. only Error or Reduce, so the choice of terminal - // cannot affect what we do) then we emit an immediate reduce action for the rule corresponding to that item - // Also do the same for Accept rules. + // If there is a single item A -> B C . and no Shift or Accept actions (i.e. only Error or Reduce, so the choice of terminal + // cannot affect what we do) then we emit an immediate reduce action for the rule corresponding to that item + // Also do the same for Accept rules. let closure = (ComputeClosure0 kernel) let immediateAction = match Set.toList closure with | [item0] -> - match (rsym_of_item0 item0) with + match (rsym_of_item0 item0) with | None when (let reduceOrErrorAction = function Error | Reduce _ -> true | Shift _ | Accept -> false termTab.Indexes |> List.forall(fun terminalIdx -> reduceOrErrorAction (snd(arr.[terminalIdx])))) -> Some (Reduce (prodIdx_of_item0 item0)) @@ -872,13 +886,13 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = | _ -> None | _ -> None - // A -> B C . rules give rise to reductions in favour of errors + // A -> B C . rules give rise to reductions in favour of errors for item0 in ComputeClosure0 kernel do let prec = prec_of_item0 item0 - match rsym_of_item0 item0 with + match rsym_of_item0 item0 with | None -> - for terminalIdx in termTab.Indexes do - if snd(arr.[terminalIdx]) = Error then + for terminalIdx in termTab.Indexes do + if snd(arr.[terminalIdx]) = Error then let prodIdx = prodIdx_of_item0 item0 let action = (prec, (if IsStartItem(item0) then Accept else Reduce prodIdx)) addResolvingPrecedence arr kernelIdx terminalIdx action @@ -890,10 +904,10 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = Array.ofList (List.map fst actionInfo), Array.ofList (List.map snd actionInfo) - // The goto table is much simpler - it is based on LR(0) kernels alone. + // The goto table is much simpler - it is based on LR(0) kernels alone. reportTime(); printf " building goto table..."; stdout.Flush(); - let gotoTable = + let gotoTable = let gotos kernelIdx = Array.ofList (List.map (fun nt -> gotoKernel (GotoItemIdx(kernelIdx,PNonTerminal nt))) ntTab.Indexes) Array.ofList (List.map gotos kernelTab.Indexes) @@ -903,12 +917,12 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = if !shiftReduceConflicts > 0 || !reduceReduceConflicts > 0 then printfn " consider setting precedences explicitly using %%left %%right and %%nonassoc on terminals and/or setting explicit precedence on rules using %%prec" /// The final results - let states = kernels |> Array.ofList - let prods = Array.ofList (List.map (fun (Production(nt,prec,syms,code)) -> (nt, ntTab.ToIndex nt, syms,code)) prods) + let states = kernels |> Array.ofList + let prods = Array.ofList (List.map (fun (Production((name, range) as nt, prec, syms, code)) -> (nt, ntTab.ToIndex name, syms,code)) prods) - logf (fun logStream -> + logf (fun logStream -> printfn "writing tables to log"; stdout.Flush(); - OutputLalrTables logStream (prods, states, startKernelIdxs, actionTable, immediateActionTable, gotoTable, (termTab.ToIndex endOfInputTerminal), errorTerminalIdx)); + OutputLalrTables logStream (prods, states, startKernelIdxs, actionTable, immediateActionTable, gotoTable, (termTab.ToIndex (fst endOfInputTerminal)), errorTerminalIdx)); let states = states |> Array.map (Set.toList >> List.map prodIdx_of_item0) { prods = prods @@ -917,16 +931,16 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec): CompiledSpec = actionTable = actionTable immediateActionTable = immediateActionTable gotoTable = gotoTable - endOfInputTerminalIdx = termTab.ToIndex endOfInputTerminal + endOfInputTerminalIdx = termTab.ToIndex (fst endOfInputTerminal) errorTerminalIdx = errorTerminalIdx nonTerminals = nonTerminals } - -(* Some examples for testing *) + +(* Some examples for testing *) (* -let example1 = - let e = "E" +let example1 = + let e = "E" let t = "Terminal" let plus = "+" let mul = "*" @@ -934,12 +948,12 @@ let example1 = let lparen = "(" let rparen = ")" let id = "id" - + let terminals = [plus; mul; lparen; rparen; id] let nonTerminals = [e; t; f] - + let p2 = e, (NonAssoc, ExplicitPrec 1), [NonTerminal e; Terminal plus; NonTerminal t], None - let p3 = e, (NonAssoc, ExplicitPrec 2), [NonTerminal t], None in + let p3 = e, (NonAssoc, ExplicitPrec 2), [NonTerminal t], None in let p4 = t, (NonAssoc, ExplicitPrec 3), [NonTerminal t; Terminal mul; NonTerminal f], None let p5 = t, (NonAssoc, ExplicitPrec 4), [NonTerminal f], None let p6 = f, (NonAssoc, ExplicitPrec 5), [Terminal lparen; NonTerminal e; Terminal rparen], None @@ -948,13 +962,13 @@ let example1 = let prods = [p2;p3;p4;p5;p6;p7] Spec(terminals,nonTerminals,prods, [e]) -let example2 = - let prods = [ "S", (NonAssoc, ExplicitPrec 1), [NonTerminal "C";NonTerminal "C"], None; +let example2 = + let prods = [ "S", (NonAssoc, ExplicitPrec 1), [NonTerminal "C";NonTerminal "C"], None; "C", (NonAssoc, ExplicitPrec 2), [Terminal "c";NonTerminal "C"], None ; "C", (NonAssoc, ExplicitPrec 3), [Terminal "d"] , None ]in Spec(["c";"d"],["S";"C"],prods, ["S"]) -let example3 = +let example3 = let terminals = ["+"; "*"; "("; ")"; "id"] let nonTerminals = ["E"; "Terminal"; "E'"; "F"; "Terminal'"] let prods = [ "E", (NonAssoc, ExplicitPrec 1), [ NonTerminal "Terminal"; NonTerminal "E'" ], None; @@ -967,7 +981,7 @@ let example3 = "F", (NonAssoc, ExplicitPrec 8), [ Terminal "id"], None ] Spec(terminals,nonTerminals,prods, ["E"]) -let example4 = +let example4 = let terminals = ["+"; "*"; "("; ")"; "id"] let nonTerminals = ["E"] let prods = [ "E", (NonAssoc, ExplicitPrec 1), [ NonTerminal "E"; Terminal "+"; NonTerminal "E" ], None; @@ -976,7 +990,7 @@ let example4 = "E", (NonAssoc, ExplicitPrec 8), [ Terminal "id"], None ] Spec(terminals,nonTerminals,prods, ["E"]) -let example5 = +let example5 = let terminals = ["+"; "*"; "("; ")"; "id"] let nonTerminals = ["E"] let prods = [ "E", (NonAssoc, ExplicitPrec 1), [ NonTerminal "E"; Terminal "+"; NonTerminal "E" ], None; @@ -985,7 +999,7 @@ let example5 = "E", (NonAssoc, ExplicitPrec 8), [ Terminal "id"], None ] Spec(terminals,nonTerminals,prods, ["E"]) -let example6 = +let example6 = let terminals = ["+"; "*"; "("; ")"; "id"; "-"] let nonTerminals = ["E"] let prods = [ "E", (RightAssoc, ExplicitPrec 1), [ NonTerminal "E"; Terminal "-"; NonTerminal "E" ], None; @@ -996,11 +1010,11 @@ let example6 = Spec(terminals,nonTerminals,prods, ["E"]) -let example7 = - let prods = [ "S", (NonAssoc, ExplicitPrec 1), [NonTerminal "L";Terminal "="; NonTerminal "R"], None; +let example7 = + let prods = [ "S", (NonAssoc, ExplicitPrec 1), [NonTerminal "L";Terminal "="; NonTerminal "R"], None; "S", (NonAssoc, ExplicitPrec 2), [NonTerminal "R"], None ; "L", (NonAssoc, ExplicitPrec 3), [Terminal "*"; NonTerminal "R"], None; - "L", (NonAssoc, ExplicitPrec 3), [Terminal "id"], None; + "L", (NonAssoc, ExplicitPrec 3), [Terminal "id"], None; "R", (NonAssoc, ExplicitPrec 3), [NonTerminal "L"], None; ] Spec(["*";"=";"id"],["S";"L";"R"],prods, ["S"]) @@ -1010,8 +1024,8 @@ let test ex = CompilerLalrParserSpec stdout ex (* let _ = test example2*) (* let _ = exit 1*) -(* let _ = test example3 -let _ = test example1 +(* let _ = test example3 +let _ = test example1 let _ = test example4 let _ = test example5 let _ = test example6 *) diff --git a/src/FsYacc.Core/fsyaccdriver.fs b/src/FsYacc.Core/fsyaccdriver.fs index b72691a9..c8920b84 100644 --- a/src/FsYacc.Core/fsyaccdriver.fs +++ b/src/FsYacc.Core/fsyaccdriver.fs @@ -102,8 +102,8 @@ type Writer(outputFileName, outputFileInterface) = member x.WriteUInt16 (i: int) = fprintf os "%dus;" i - member x.WriteCode (code, pos) = - x.WriteLine "# %d \"%s\"" pos.pos_lnum pos.pos_fname + member x.WriteCode (code, range: Range) = + x.WriteLine "# %d \"%s\"" range.startPos.pos_lnum range.startPos.pos_fname x.WriteLine "%s" code let codeLines = code.Replace("\r","").Split([| '\n' |]).Length outputLineCount <- outputLineCount + codeLines @@ -131,7 +131,7 @@ type Writer(outputFileName, outputFileInterface) = // This is to avoid name conflicts against keywords. -let generic_nt_name nt = "'gentype_" + nt +let generic_nt_name (nt, _range) = "'gentype_" + nt let anyMarker = 0xffff let actionCoding = @@ -215,7 +215,7 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine "type token = "; writer.WriteLineInterface "type token = "; - for id,typ in spec.Tokens do + for (id, _range), typ in spec.Tokens do match typ with | None -> writer.WriteLine " | %s" id @@ -228,7 +228,7 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine "// This type is used to give symbolic names to token indexes, useful for error messages"; writer.WriteLine "type tokenId = "; writer.WriteLineInterface "type tokenId = "; - for id,typ in spec.Tokens do + for (id, _range), typ in spec.Tokens do writer.WriteLine " | TOKEN_%s" id; writer.WriteLineInterface " | TOKEN_%s" id; writer.WriteLine " | TOKEN_end_of_input"; @@ -239,7 +239,7 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine "// This type is used to give symbolic names to token indexes, useful for error messages"; writer.WriteLine "type nonTerminalId = "; writer.WriteLineInterface "type nonTerminalId = "; - for nt in compiledSpec.nonTerminals do + for (nt, _range) in compiledSpec.nonTerminals do writer.WriteLine " | NONTERM_%s" nt; writer.WriteLineInterface " | NONTERM_%s" nt; @@ -248,7 +248,7 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine "// This function maps tokens to integer indexes"; writer.WriteLine "let tagOfToken (t:token) = "; writer.WriteLine " match t with"; - spec.Tokens |> List.iteri (fun i (id,typ) -> + spec.Tokens |> List.iteri (fun i ((id, _range), typ) -> writer.WriteLine " | %s %s -> %d " id (match typ with Some _ -> "_" | None -> "") i); writer.WriteLineInterface "/// This function maps tokens to integer indexes"; writer.WriteLineInterface "val tagOfToken: token -> int"; @@ -257,7 +257,7 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine "// This function maps integer indexes to symbolic token ids"; writer.WriteLine "let tokenTagToTokenId (tokenIdx:int) = "; writer.WriteLine " match tokenIdx with"; - spec.Tokens |> List.iteri (fun i (id,typ) -> writer.WriteLine " | %d -> TOKEN_%s " i id) + spec.Tokens |> List.iteri (fun i ((id, _range), typ) -> writer.WriteLine " | %d -> TOKEN_%s " i id) writer.WriteLine " | %d -> TOKEN_end_of_input" compiledSpec.endOfInputTerminalIdx; writer.WriteLine " | %d -> TOKEN_error" compiledSpec.errorTerminalIdx; writer.WriteLine " | _ -> failwith \"tokenTagToTokenId: bad token\"" @@ -270,7 +270,7 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine "/// This function maps production indexes returned in syntax errors to strings representing the non terminal that would be produced by that production"; writer.WriteLine "let prodIdxToNonTerminal (prodIdx:int) = "; writer.WriteLine " match prodIdx with"; - compiledSpec.prods |> Array.iteri (fun i (nt,ntIdx,syms,code) -> writer.WriteLine " | %d -> NONTERM_%s " i nt); + compiledSpec.prods |> Array.iteri (fun i ((nt, _range), ntIdx, syms, code) -> writer.WriteLine " | %d -> NONTERM_%s " i nt); writer.WriteLine " | _ -> failwith \"prodIdxToNonTerminal: bad production index\"" writer.WriteLineInterface ""; @@ -284,7 +284,7 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine "// This function gets the name of a token as a string"; writer.WriteLine "let token_to_string (t:token) = "; writer.WriteLine " match t with "; - spec.Tokens |> List.iteri (fun i (id,typ) -> writer.WriteLine " | %s %s -> \"%s\" " id (match typ with Some _ -> "_" | None -> "") id); + spec.Tokens |> List.iteri (fun i ((id, _range), typ) -> writer.WriteLine " | %s %s -> \"%s\" " id (match typ with Some _ -> "_" | None -> "") id); writer.WriteLineInterface ""; writer.WriteLineInterface "/// This function gets the name of a token as a string"; @@ -295,7 +295,7 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine "let _fsyacc_dataOfToken (t:token) = "; writer.WriteLine " match t with "; - for (id,typ) in spec.Tokens do + for ((id, _range), typ) in spec.Tokens do writer.WriteLine " | %s %s -> %s " id (match typ with Some _ -> "_fsyacc_x" | None -> "") @@ -303,14 +303,14 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile let tychar = "'cty" - for (key,_) in spec.Types |> Seq.countBy fst |> Seq.filter (fun (_,n) -> n > 1) do + for ((key, _range), _) in spec.Types |> Seq.countBy fst |> Seq.filter (fun (_,n) -> n > 1) do failwithf "%s is given multiple %%type declarations" key; - for (key,_) in spec.Tokens |> Seq.countBy fst |> Seq.filter (fun (_,n) -> n > 1) do + for ((key, _range), _) in spec.Tokens |> Seq.countBy fst |> Seq.filter (fun (_,n) -> n > 1) do failwithf "%s is given %%token declarations" key - let types = Map.ofList spec.Types - let tokens = Map.ofList spec.Tokens + let types = Map.ofList (spec.Types |> List.map (fun ((name, _range), rest) -> name, rest)) + let tokens = Map.ofList (spec.Tokens |> List.map (fun ((name, _range), rest) -> name, rest)) let nStates = compiledSpec.states.Length begin @@ -457,7 +457,11 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine "|]" ; end; - let getType nt = if types.ContainsKey nt then types.[nt] else generatorState.generate_nonterminal_name nt + let getType ((name, _range) as nt) = + types + |> Map.tryFind name + |> Option.defaultWith (fun _ -> generatorState.generate_nonterminal_name nt) + begin writer.Write "let _fsyacc_reductions () =" ; writer.WriteLine " [| " ; @@ -469,10 +473,7 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile syms |> List.iteri (fun i sym -> let tyopt = match sym with - | Terminal t -> - if tokens.ContainsKey t then - tokens.[t] - else None + | Terminal (name, range) -> tokens |> Map.tryFind name |> Option.flatten | NonTerminal nt -> Some (getType nt) match tyopt with | Some ty -> writer.WriteLine " let _%d = parseState.GetInput(%d) :?> %s in" (i+1) (i+1) ty @@ -481,7 +482,7 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine " ("; writer.WriteLine " ("; match code with - | Some (_,pos) -> writer.WriteLine "# %d \"%s\"" pos.pos_lnum pos.pos_fname + | Some (_,{ startPos = pos }) -> writer.WriteLine "# %d \"%s\"" pos.pos_lnum pos.pos_fname | None -> () match code with | Some (code,_) -> @@ -500,9 +501,9 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine " )"; // Place the line count back for the type constraint match code with - | Some (_,pos) -> writer.WriteLine "# %d \"%s\"" pos.pos_lnum pos.pos_fname + | Some (_, { startPos = pos }) -> writer.WriteLine "# %d \"%s\"" pos.pos_lnum pos.pos_fname | None -> () - writer.WriteLine " : %s));" (if types.ContainsKey nt then types.[nt] else generatorState.generate_nonterminal_name nt); + writer.WriteLine " : %s));" (getType nt); done; writer.WriteLine "|]" ; end; @@ -530,14 +531,14 @@ let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compile writer.WriteLine " productionToNonTerminalTable = _fsyacc_productionToNonTerminalTable }" writer.WriteLine "let engine lexer lexbuf startState = tables.Interpret(lexer, lexbuf, startState)" - for (id,startState) in List.zip spec.StartSymbols compiledSpec.startStates do + for ((id, _range), startState) in List.zip spec.StartSymbols compiledSpec.startStates do if not (types.ContainsKey id) then - failwith ("a %type declaration is required for for start token "+id); + failwith ("a %type declaration is required for for start token "+ id); let ty = types.[id] in writer.WriteLine "let %s lexer lexbuf : %s =" id ty; writer.WriteLine " engine lexer lexbuf %d :?> _" startState - for id in spec.StartSymbols do + for (id, _range) in spec.StartSymbols do if not (types.ContainsKey id) then failwith ("a %type declaration is required for start token "+id); let ty = types.[id] in diff --git a/src/FsYacc.Core/fsyacclex.fs b/src/FsYacc.Core/fsyacclex.fs index 5b99ce61..b4838aeb 100644 --- a/src/FsYacc.Core/fsyacclex.fs +++ b/src/FsYacc.Core/fsyacclex.fs @@ -447,7 +447,7 @@ and header p buff lexbuf = match _fslex_tables.Interpret(42,lexbuf) with | 0 -> ( # 74 "fsyacclex.fsl" - HEADER (buff.ToString(), p) + HEADER (buff.ToString(), { startPos = p; endPos = lexbuf.EndPos }) # 451 "fsyacclex.fs" ) | 1 -> ( @@ -499,7 +499,7 @@ and code p buff lexbuf = match _fslex_tables.Interpret(23,lexbuf) with | 0 -> ( # 95 "fsyacclex.fsl" - CODE (buff.ToString(), p) + CODE (buff.ToString(), { startPos = p; endPos = lexbuf.EndPos }) # 503 "fsyacclex.fs" ) | 1 -> ( diff --git a/src/FsYacc.Core/fsyacclex.fsl b/src/FsYacc.Core/fsyacclex.fsl index 02d4c48e..41a3e98e 100644 --- a/src/FsYacc.Core/fsyacclex.fsl +++ b/src/FsYacc.Core/fsyacclex.fsl @@ -71,7 +71,7 @@ and fs_type = parse | _ { appendBuf(lexeme lexbuf); fs_type lexbuf } and header p buff = parse - | "%}" { HEADER (buff.ToString(), p) } + | "%}" { HEADER (buff.ToString(), { startPos = p; endPos = lexbuf.EndPos }) } | newline { newline lexbuf; ignore <| buff.Append System.Environment.NewLine; header p buff lexbuf } @@ -92,7 +92,7 @@ and header p buff = parse | _ { ignore <| buff.Append(lexeme lexbuf).[0]; header p buff lexbuf } and code p buff = parse - | "}" { CODE (buff.ToString(), p) } + | "}" { CODE (buff.ToString(), { startPos = p; endPos = lexbuf.EndPos }) } | "{" { ignore <| buff.Append (lexeme lexbuf); ignore(code p buff lexbuf); ignore <| buff.Append "}"; diff --git a/src/FsYacc.Core/fsyaccpars.fs b/src/FsYacc.Core/fsyaccpars.fs index 504e1cb5..41e43d3e 100644 --- a/src/FsYacc.Core/fsyaccpars.fs +++ b/src/FsYacc.Core/fsyaccpars.fs @@ -65,6 +65,7 @@ type nonTerminalId = | NONTERM_headeropt | NONTERM_decls | NONTERM_decl + | NONTERM_ident | NONTERM_idents | NONTERM_rules | NONTERM_rule @@ -137,23 +138,24 @@ let prodIdxToNonTerminal (prodIdx:int) = | 9 -> NONTERM_decl | 10 -> NONTERM_decl | 11 -> NONTERM_decl - | 12 -> NONTERM_idents + | 12 -> NONTERM_ident | 13 -> NONTERM_idents - | 14 -> NONTERM_rules + | 14 -> NONTERM_idents | 15 -> NONTERM_rules - | 16 -> NONTERM_rule - | 17 -> NONTERM_optbar + | 16 -> NONTERM_rules + | 17 -> NONTERM_rule | 18 -> NONTERM_optbar - | 19 -> NONTERM_optsemi + | 19 -> NONTERM_optbar | 20 -> NONTERM_optsemi - | 21 -> NONTERM_clauses + | 21 -> NONTERM_optsemi | 22 -> NONTERM_clauses - | 23 -> NONTERM_clause - | 24 -> NONTERM_syms + | 23 -> NONTERM_clauses + | 24 -> NONTERM_clause | 25 -> NONTERM_syms | 26 -> NONTERM_syms - | 27 -> NONTERM_optprec + | 27 -> NONTERM_syms | 28 -> NONTERM_optprec + | 29 -> NONTERM_optprec | _ -> failwith "prodIdxToNonTerminal: bad production index" let _fsyacc_endOfInputTag = 20 @@ -202,18 +204,18 @@ let _fsyacc_dataOfToken (t:token) = | HEADER _fsyacc_x -> Microsoft.FSharp.Core.Operators.box _fsyacc_x | CODE _fsyacc_x -> Microsoft.FSharp.Core.Operators.box _fsyacc_x | IDENT _fsyacc_x -> Microsoft.FSharp.Core.Operators.box _fsyacc_x -let _fsyacc_gotos = [| 0us; 65535us; 1us; 65535us; 0us; 1us; 1us; 65535us; 0us; 2us; 2us; 65535us; 2us; 3us; 7us; 8us; 2us; 65535us; 2us; 7us; 7us; 7us; 7us; 65535us; 9us; 10us; 11us; 12us; 13us; 14us; 15us; 16us; 17us; 18us; 19us; 20us; 21us; 22us; 2us; 65535us; 4us; 5us; 23us; 24us; 2us; 65535us; 4us; 23us; 23us; 23us; 1us; 65535us; 26us; 27us; 1us; 65535us; 28us; 29us; 2us; 65535us; 27us; 28us; 33us; 34us; 2us; 65535us; 27us; 32us; 33us; 32us; 4us; 65535us; 27us; 35us; 33us; 35us; 38us; 39us; 40us; 41us; 1us; 65535us; 35us; 36us; |] -let _fsyacc_sparseGotoTableRowOffsets = [|0us; 1us; 3us; 5us; 8us; 11us; 19us; 22us; 25us; 27us; 29us; 32us; 35us; 40us; |] -let _fsyacc_stateToProdIdxsTableElements = [| 1us; 0us; 1us; 0us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 2us; 1us; 5us; 1us; 5us; 1us; 6us; 1us; 6us; 1us; 7us; 1us; 7us; 1us; 8us; 1us; 8us; 1us; 9us; 1us; 9us; 1us; 10us; 1us; 10us; 1us; 11us; 1us; 11us; 1us; 12us; 1us; 12us; 2us; 14us; 15us; 1us; 14us; 1us; 16us; 1us; 16us; 1us; 16us; 1us; 16us; 1us; 16us; 1us; 18us; 1us; 20us; 2us; 21us; 22us; 1us; 21us; 1us; 21us; 1us; 23us; 1us; 23us; 1us; 23us; 1us; 24us; 1us; 24us; 1us; 25us; 1us; 25us; 1us; 28us; 1us; 28us; |] -let _fsyacc_stateToProdIdxsTableRowOffsets = [|0us; 2us; 4us; 6us; 8us; 10us; 12us; 14us; 16us; 18us; 20us; 22us; 24us; 26us; 28us; 30us; 32us; 34us; 36us; 38us; 40us; 42us; 44us; 46us; 49us; 51us; 53us; 55us; 57us; 59us; 61us; 63us; 65us; 68us; 70us; 72us; 74us; 76us; 78us; 80us; 82us; 84us; 86us; 88us; |] -let _fsyacc_action_rows = 44 -let _fsyacc_actionTableElements = [|1us; 16387us; 15us; 6us; 0us; 49152us; 6us; 16388us; 0us; 9us; 1us; 11us; 4us; 13us; 5us; 15us; 6us; 17us; 7us; 19us; 1us; 32768us; 3us; 4us; 1us; 32768us; 17us; 25us; 0us; 16385us; 0us; 16386us; 6us; 16388us; 0us; 9us; 1us; 11us; 4us; 13us; 5us; 15us; 6us; 17us; 7us; 19us; 0us; 16389us; 1us; 16397us; 17us; 21us; 0us; 16390us; 1us; 16397us; 17us; 21us; 0us; 16391us; 1us; 16397us; 17us; 21us; 0us; 16392us; 1us; 16397us; 17us; 21us; 0us; 16393us; 1us; 16397us; 17us; 21us; 0us; 16394us; 1us; 16397us; 17us; 21us; 0us; 16395us; 1us; 16397us; 17us; 21us; 0us; 16396us; 1us; 16399us; 17us; 25us; 0us; 16398us; 1us; 32768us; 10us; 26us; 1us; 16401us; 2us; 30us; 2us; 16410us; 14us; 40us; 17us; 38us; 1us; 16403us; 12us; 31us; 0us; 16400us; 0us; 16402us; 0us; 16404us; 1us; 16406us; 2us; 33us; 2us; 16410us; 14us; 40us; 17us; 38us; 0us; 16405us; 1us; 16411us; 11us; 42us; 1us; 32768us; 16us; 37us; 0us; 16407us; 2us; 16410us; 14us; 40us; 17us; 38us; 0us; 16408us; 2us; 16410us; 14us; 40us; 17us; 38us; 0us; 16409us; 1us; 32768us; 17us; 43us; 0us; 16412us; |] -let _fsyacc_actionTableRowOffsets = [|0us; 2us; 3us; 10us; 12us; 14us; 15us; 16us; 23us; 24us; 26us; 27us; 29us; 30us; 32us; 33us; 35us; 36us; 38us; 39us; 41us; 42us; 44us; 45us; 47us; 48us; 50us; 52us; 55us; 57us; 58us; 59us; 60us; 62us; 65us; 66us; 68us; 70us; 71us; 74us; 75us; 78us; 79us; 81us; |] -let _fsyacc_reductionSymbolCounts = [|1us; 4us; 1us; 0us; 0us; 2us; 2us; 2us; 2us; 2us; 2us; 2us; 2us; 0us; 2us; 1us; 5us; 0us; 1us; 0us; 1us; 3us; 1us; 3us; 2us; 2us; 0us; 0us; 2us; |] -let _fsyacc_productionToNonTerminalTable = [|0us; 1us; 2us; 2us; 3us; 3us; 4us; 4us; 4us; 4us; 4us; 4us; 5us; 5us; 6us; 6us; 7us; 8us; 8us; 9us; 9us; 10us; 10us; 11us; 12us; 12us; 12us; 13us; 13us; |] -let _fsyacc_immediateActions = [|65535us; 49152us; 65535us; 65535us; 65535us; 16385us; 16386us; 65535us; 16389us; 65535us; 16390us; 65535us; 16391us; 65535us; 16392us; 65535us; 16393us; 65535us; 16394us; 65535us; 16395us; 65535us; 16396us; 65535us; 16398us; 65535us; 65535us; 65535us; 65535us; 16400us; 16402us; 16404us; 65535us; 65535us; 16405us; 65535us; 65535us; 16407us; 65535us; 16408us; 65535us; 16409us; 65535us; 16412us; |] +let _fsyacc_gotos = [| 0us; 65535us; 1us; 65535us; 0us; 1us; 1us; 65535us; 0us; 2us; 2us; 65535us; 2us; 3us; 7us; 8us; 2us; 65535us; 2us; 7us; 7us; 7us; 14us; 65535us; 4us; 26us; 9us; 22us; 11us; 22us; 13us; 22us; 15us; 22us; 17us; 22us; 19us; 22us; 22us; 22us; 24us; 26us; 28us; 39us; 34us; 39us; 39us; 39us; 41us; 39us; 43us; 44us; 7us; 65535us; 9us; 10us; 11us; 12us; 13us; 14us; 15us; 16us; 17us; 18us; 19us; 20us; 22us; 23us; 2us; 65535us; 4us; 5us; 24us; 25us; 2us; 65535us; 4us; 24us; 24us; 24us; 1us; 65535us; 27us; 28us; 1us; 65535us; 29us; 30us; 2us; 65535us; 28us; 29us; 34us; 35us; 2us; 65535us; 28us; 33us; 34us; 33us; 4us; 65535us; 28us; 36us; 34us; 36us; 39us; 40us; 41us; 42us; 1us; 65535us; 36us; 37us; |] +let _fsyacc_sparseGotoTableRowOffsets = [|0us; 1us; 3us; 5us; 8us; 11us; 26us; 34us; 37us; 40us; 42us; 44us; 47us; 50us; 55us; |] +let _fsyacc_stateToProdIdxsTableElements = [| 1us; 0us; 1us; 0us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 1us; 2us; 1us; 5us; 1us; 5us; 1us; 6us; 1us; 6us; 1us; 7us; 1us; 7us; 1us; 8us; 1us; 8us; 1us; 9us; 1us; 9us; 1us; 10us; 1us; 10us; 1us; 11us; 1us; 11us; 1us; 12us; 1us; 13us; 1us; 13us; 2us; 15us; 16us; 1us; 15us; 1us; 17us; 1us; 17us; 1us; 17us; 1us; 17us; 1us; 17us; 1us; 19us; 1us; 21us; 2us; 22us; 23us; 1us; 22us; 1us; 22us; 1us; 24us; 1us; 24us; 1us; 24us; 1us; 25us; 1us; 25us; 1us; 26us; 1us; 26us; 1us; 29us; 1us; 29us; |] +let _fsyacc_stateToProdIdxsTableRowOffsets = [|0us; 2us; 4us; 6us; 8us; 10us; 12us; 14us; 16us; 18us; 20us; 22us; 24us; 26us; 28us; 30us; 32us; 34us; 36us; 38us; 40us; 42us; 44us; 46us; 48us; 51us; 53us; 55us; 57us; 59us; 61us; 63us; 65us; 67us; 70us; 72us; 74us; 76us; 78us; 80us; 82us; 84us; 86us; 88us; 90us; |] +let _fsyacc_action_rows = 45 +let _fsyacc_actionTableElements = [|1us; 16387us; 15us; 6us; 0us; 49152us; 6us; 16388us; 0us; 9us; 1us; 11us; 4us; 13us; 5us; 15us; 6us; 17us; 7us; 19us; 1us; 32768us; 3us; 4us; 1us; 32768us; 17us; 21us; 0us; 16385us; 0us; 16386us; 6us; 16388us; 0us; 9us; 1us; 11us; 4us; 13us; 5us; 15us; 6us; 17us; 7us; 19us; 0us; 16389us; 1us; 16398us; 17us; 21us; 0us; 16390us; 1us; 16398us; 17us; 21us; 0us; 16391us; 1us; 16398us; 17us; 21us; 0us; 16392us; 1us; 16398us; 17us; 21us; 0us; 16393us; 1us; 16398us; 17us; 21us; 0us; 16394us; 1us; 16398us; 17us; 21us; 0us; 16395us; 0us; 16396us; 1us; 16398us; 17us; 21us; 0us; 16397us; 1us; 16400us; 17us; 21us; 0us; 16399us; 1us; 32768us; 10us; 27us; 1us; 16402us; 2us; 31us; 2us; 16411us; 14us; 41us; 17us; 21us; 1us; 16404us; 12us; 32us; 0us; 16401us; 0us; 16403us; 0us; 16405us; 1us; 16407us; 2us; 34us; 2us; 16411us; 14us; 41us; 17us; 21us; 0us; 16406us; 1us; 16412us; 11us; 43us; 1us; 32768us; 16us; 38us; 0us; 16408us; 2us; 16411us; 14us; 41us; 17us; 21us; 0us; 16409us; 2us; 16411us; 14us; 41us; 17us; 21us; 0us; 16410us; 1us; 32768us; 17us; 21us; 0us; 16413us; |] +let _fsyacc_actionTableRowOffsets = [|0us; 2us; 3us; 10us; 12us; 14us; 15us; 16us; 23us; 24us; 26us; 27us; 29us; 30us; 32us; 33us; 35us; 36us; 38us; 39us; 41us; 42us; 43us; 45us; 46us; 48us; 49us; 51us; 53us; 56us; 58us; 59us; 60us; 61us; 63us; 66us; 67us; 69us; 71us; 72us; 75us; 76us; 79us; 80us; 82us; |] +let _fsyacc_reductionSymbolCounts = [|1us; 4us; 1us; 0us; 0us; 2us; 2us; 2us; 2us; 2us; 2us; 2us; 1us; 2us; 0us; 2us; 1us; 5us; 0us; 1us; 0us; 1us; 3us; 1us; 3us; 2us; 2us; 0us; 0us; 2us; |] +let _fsyacc_productionToNonTerminalTable = [|0us; 1us; 2us; 2us; 3us; 3us; 4us; 4us; 4us; 4us; 4us; 4us; 5us; 6us; 6us; 7us; 7us; 8us; 9us; 9us; 10us; 10us; 11us; 11us; 12us; 13us; 13us; 13us; 14us; 14us; |] +let _fsyacc_immediateActions = [|65535us; 49152us; 65535us; 65535us; 65535us; 16385us; 16386us; 65535us; 16389us; 65535us; 16390us; 65535us; 16391us; 65535us; 16392us; 65535us; 16393us; 65535us; 16394us; 65535us; 16395us; 16396us; 65535us; 16397us; 65535us; 16399us; 65535us; 65535us; 65535us; 65535us; 16401us; 16403us; 16405us; 65535us; 65535us; 16406us; 65535us; 65535us; 16408us; 65535us; 16409us; 65535us; 16410us; 65535us; 16413us; |] let _fsyacc_reductions () = [| -# 216 "fsyaccpars.fs" +# 218 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> AST.ParserSpec in Microsoft.FSharp.Core.Operators.box @@ -222,7 +224,7 @@ let _fsyacc_reductions () = [| raise (FSharp.Text.Parsing.Accept(Microsoft.FSharp.Core.Operators.box _1)) ) : 'gentype__startspec)); -# 225 "fsyaccpars.fs" +# 227 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_headeropt in let _2 = parseState.GetInput(2) :?> 'gentype_decls in @@ -231,244 +233,258 @@ let _fsyacc_reductions () = [| ( ( # 25 "fsyaccpars.fsy" - List.foldBack (fun f x -> f x) _2 { Header=_1;Tokens=[];Types=[];Associativities=[];StartSymbols=[];Rules=_4 } + + (_2, { Header=_1;Tokens=[];Types=[];Associativities=[];StartSymbols=[];Rules=_4 }) + ||> List.foldBack (fun f x -> f x) + ) # 25 "fsyaccpars.fsy" : AST.ParserSpec)); -# 238 "fsyaccpars.fs" +# 243 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> AST.Code in Microsoft.FSharp.Core.Operators.box ( ( -# 29 "fsyaccpars.fsy" +# 32 "fsyaccpars.fsy" _1 ) -# 29 "fsyaccpars.fsy" +# 32 "fsyaccpars.fsy" : 'gentype_headeropt)); -# 249 "fsyaccpars.fs" +# 254 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 31 "fsyaccpars.fsy" - "", (parseState.ResultRange |> fst) +# 34 "fsyaccpars.fsy" + "", parseState.ResultRange ) -# 31 "fsyaccpars.fsy" +# 34 "fsyaccpars.fsy" : 'gentype_headeropt)); -# 259 "fsyaccpars.fs" +# 264 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 34 "fsyaccpars.fsy" +# 37 "fsyaccpars.fsy" [] ) -# 34 "fsyaccpars.fsy" +# 37 "fsyaccpars.fsy" : 'gentype_decls)); -# 269 "fsyaccpars.fs" +# 274 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_decl in let _2 = parseState.GetInput(2) :?> 'gentype_decls in Microsoft.FSharp.Core.Operators.box ( ( -# 35 "fsyaccpars.fsy" +# 38 "fsyaccpars.fsy" _1 :: _2 ) -# 35 "fsyaccpars.fsy" +# 38 "fsyaccpars.fsy" : 'gentype_decls)); -# 281 "fsyaccpars.fs" +# 286 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> string option in let _2 = parseState.GetInput(2) :?> 'gentype_idents in Microsoft.FSharp.Core.Operators.box ( ( -# 38 "fsyaccpars.fsy" +# 41 "fsyaccpars.fsy" (fun x -> {x with Tokens = x.Tokens @ (List.map (fun x -> (x,_1)) _2)}) ) -# 38 "fsyaccpars.fsy" +# 41 "fsyaccpars.fsy" : 'gentype_decl)); -# 293 "fsyaccpars.fs" +# 298 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> string in let _2 = parseState.GetInput(2) :?> 'gentype_idents in Microsoft.FSharp.Core.Operators.box ( ( -# 39 "fsyaccpars.fsy" +# 42 "fsyaccpars.fsy" (fun x -> {x with Types = x.Types @ (List.map (fun x -> (x,_1)) _2)} ) ) -# 39 "fsyaccpars.fsy" +# 42 "fsyaccpars.fsy" : 'gentype_decl)); -# 305 "fsyaccpars.fs" +# 310 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _2 = parseState.GetInput(2) :?> 'gentype_idents in Microsoft.FSharp.Core.Operators.box ( ( -# 40 "fsyaccpars.fsy" +# 43 "fsyaccpars.fsy" (fun x -> {x with StartSymbols = x.StartSymbols @ _2} ) ) -# 40 "fsyaccpars.fsy" +# 43 "fsyaccpars.fsy" : 'gentype_decl)); -# 316 "fsyaccpars.fs" +# 321 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _2 = parseState.GetInput(2) :?> 'gentype_idents in Microsoft.FSharp.Core.Operators.box ( ( -# 41 "fsyaccpars.fsy" +# 44 "fsyaccpars.fsy" (fun x -> {x with Associativities = x.Associativities @ [(List.map (fun x -> (x,LeftAssoc)) _2)]} ) ) -# 41 "fsyaccpars.fsy" +# 44 "fsyaccpars.fsy" : 'gentype_decl)); -# 327 "fsyaccpars.fs" +# 332 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _2 = parseState.GetInput(2) :?> 'gentype_idents in Microsoft.FSharp.Core.Operators.box ( ( -# 42 "fsyaccpars.fsy" +# 45 "fsyaccpars.fsy" (fun x -> {x with Associativities = x.Associativities @ [(List.map (fun x -> (x,RightAssoc)) _2)]} ) ) -# 42 "fsyaccpars.fsy" +# 45 "fsyaccpars.fsy" : 'gentype_decl)); -# 338 "fsyaccpars.fs" +# 343 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _2 = parseState.GetInput(2) :?> 'gentype_idents in Microsoft.FSharp.Core.Operators.box ( ( -# 43 "fsyaccpars.fsy" +# 46 "fsyaccpars.fsy" (fun x -> {x with Associativities = x.Associativities @ [(List.map (fun x -> (x,NonAssoc)) _2)]} ) ) -# 43 "fsyaccpars.fsy" +# 46 "fsyaccpars.fsy" : 'gentype_decl)); -# 349 "fsyaccpars.fs" +# 354 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> string in + Microsoft.FSharp.Core.Operators.box + ( + ( +# 49 "fsyaccpars.fsy" + _1, parseState.InputRange 1 + ) +# 49 "fsyaccpars.fsy" + : 'gentype_ident)); +# 365 "fsyaccpars.fs" + (fun (parseState : FSharp.Text.Parsing.IParseState) -> + let _1 = parseState.GetInput(1) :?> 'gentype_ident in let _2 = parseState.GetInput(2) :?> 'gentype_idents in Microsoft.FSharp.Core.Operators.box ( ( -# 45 "fsyaccpars.fsy" +# 51 "fsyaccpars.fsy" _1 :: _2 ) -# 45 "fsyaccpars.fsy" +# 51 "fsyaccpars.fsy" : 'gentype_idents)); -# 361 "fsyaccpars.fs" +# 377 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 45 "fsyaccpars.fsy" +# 51 "fsyaccpars.fsy" [] ) -# 45 "fsyaccpars.fsy" +# 51 "fsyaccpars.fsy" : 'gentype_idents)); -# 371 "fsyaccpars.fs" +# 387 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_rule in let _2 = parseState.GetInput(2) :?> 'gentype_rules in Microsoft.FSharp.Core.Operators.box ( ( -# 46 "fsyaccpars.fsy" +# 52 "fsyaccpars.fsy" _1 :: _2 ) -# 46 "fsyaccpars.fsy" +# 52 "fsyaccpars.fsy" : 'gentype_rules)); -# 383 "fsyaccpars.fs" +# 399 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_rule in Microsoft.FSharp.Core.Operators.box ( ( -# 46 "fsyaccpars.fsy" +# 52 "fsyaccpars.fsy" [_1] ) -# 46 "fsyaccpars.fsy" +# 52 "fsyaccpars.fsy" : 'gentype_rules)); -# 394 "fsyaccpars.fs" +# 410 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> - let _1 = parseState.GetInput(1) :?> string in + let _1 = parseState.GetInput(1) :?> 'gentype_ident in let _3 = parseState.GetInput(3) :?> 'gentype_optbar in let _4 = parseState.GetInput(4) :?> 'gentype_clauses in let _5 = parseState.GetInput(5) :?> 'gentype_optsemi in Microsoft.FSharp.Core.Operators.box ( ( -# 47 "fsyaccpars.fsy" +# 53 "fsyaccpars.fsy" (_1,_4) ) -# 47 "fsyaccpars.fsy" +# 53 "fsyaccpars.fsy" : 'gentype_rule)); -# 408 "fsyaccpars.fs" +# 424 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 48 "fsyaccpars.fsy" +# 54 "fsyaccpars.fsy" ) -# 48 "fsyaccpars.fsy" +# 54 "fsyaccpars.fsy" : 'gentype_optbar)); -# 418 "fsyaccpars.fs" +# 434 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 48 "fsyaccpars.fsy" +# 54 "fsyaccpars.fsy" ) -# 48 "fsyaccpars.fsy" +# 54 "fsyaccpars.fsy" : 'gentype_optbar)); -# 428 "fsyaccpars.fs" +# 444 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 49 "fsyaccpars.fsy" +# 55 "fsyaccpars.fsy" ) -# 49 "fsyaccpars.fsy" +# 55 "fsyaccpars.fsy" : 'gentype_optsemi)); -# 438 "fsyaccpars.fs" +# 454 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 49 "fsyaccpars.fsy" +# 55 "fsyaccpars.fsy" ) -# 49 "fsyaccpars.fsy" +# 55 "fsyaccpars.fsy" : 'gentype_optsemi)); -# 448 "fsyaccpars.fs" +# 464 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_clause in let _3 = parseState.GetInput(3) :?> 'gentype_clauses in Microsoft.FSharp.Core.Operators.box ( ( -# 50 "fsyaccpars.fsy" +# 56 "fsyaccpars.fsy" _1 :: _3 ) -# 50 "fsyaccpars.fsy" +# 56 "fsyaccpars.fsy" : 'gentype_clauses)); -# 460 "fsyaccpars.fs" +# 476 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_clause in Microsoft.FSharp.Core.Operators.box ( ( -# 50 "fsyaccpars.fsy" +# 56 "fsyaccpars.fsy" [_1] ) -# 50 "fsyaccpars.fsy" +# 56 "fsyaccpars.fsy" : 'gentype_clauses)); -# 471 "fsyaccpars.fs" +# 487 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _1 = parseState.GetInput(1) :?> 'gentype_syms in let _2 = parseState.GetInput(2) :?> 'gentype_optprec in @@ -476,67 +492,67 @@ let _fsyacc_reductions () = [| Microsoft.FSharp.Core.Operators.box ( ( -# 51 "fsyaccpars.fsy" +# 57 "fsyaccpars.fsy" Rule(_1,_2,Some _3) ) -# 51 "fsyaccpars.fsy" +# 57 "fsyaccpars.fsy" : 'gentype_clause)); -# 484 "fsyaccpars.fs" +# 500 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> - let _1 = parseState.GetInput(1) :?> string in + let _1 = parseState.GetInput(1) :?> 'gentype_ident in let _2 = parseState.GetInput(2) :?> 'gentype_syms in Microsoft.FSharp.Core.Operators.box ( ( -# 52 "fsyaccpars.fsy" +# 58 "fsyaccpars.fsy" _1 :: _2 ) -# 52 "fsyaccpars.fsy" +# 58 "fsyaccpars.fsy" : 'gentype_syms)); -# 496 "fsyaccpars.fs" +# 512 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> let _2 = parseState.GetInput(2) :?> 'gentype_syms in Microsoft.FSharp.Core.Operators.box ( ( -# 52 "fsyaccpars.fsy" - "error" :: _2 +# 58 "fsyaccpars.fsy" + ("error", Range.Empty) :: _2 ) -# 52 "fsyaccpars.fsy" +# 58 "fsyaccpars.fsy" : 'gentype_syms)); -# 507 "fsyaccpars.fs" +# 523 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 52 "fsyaccpars.fsy" - [] +# 58 "fsyaccpars.fsy" + [] ) -# 52 "fsyaccpars.fsy" +# 58 "fsyaccpars.fsy" : 'gentype_syms)); -# 517 "fsyaccpars.fs" +# 533 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> Microsoft.FSharp.Core.Operators.box ( ( -# 53 "fsyaccpars.fsy" +# 59 "fsyaccpars.fsy" None ) -# 53 "fsyaccpars.fsy" +# 59 "fsyaccpars.fsy" : 'gentype_optprec)); -# 527 "fsyaccpars.fs" +# 543 "fsyaccpars.fs" (fun (parseState : FSharp.Text.Parsing.IParseState) -> - let _2 = parseState.GetInput(2) :?> string in + let _2 = parseState.GetInput(2) :?> 'gentype_ident in Microsoft.FSharp.Core.Operators.box ( ( -# 53 "fsyaccpars.fsy" +# 59 "fsyaccpars.fsy" Some _2 ) -# 53 "fsyaccpars.fsy" +# 59 "fsyaccpars.fsy" : 'gentype_optprec)); |] -# 539 "fsyaccpars.fs" +# 555 "fsyaccpars.fs" let tables : FSharp.Text.Parsing.Tables<_> = { reductions= _fsyacc_reductions (); endOfInputTag = _fsyacc_endOfInputTag; diff --git a/src/FsYacc.Core/fsyaccpars.fsi b/src/FsYacc.Core/fsyaccpars.fsi index 53429603..3192330b 100644 --- a/src/FsYacc.Core/fsyaccpars.fsi +++ b/src/FsYacc.Core/fsyaccpars.fsi @@ -46,6 +46,7 @@ type nonTerminalId = | NONTERM_headeropt | NONTERM_decls | NONTERM_decl + | NONTERM_ident | NONTERM_idents | NONTERM_rules | NONTERM_rule diff --git a/src/FsYacc.Core/fsyaccpars.fsy b/src/FsYacc.Core/fsyaccpars.fsy index 4dd623c3..44ec104e 100644 --- a/src/FsYacc.Core/fsyaccpars.fsy +++ b/src/FsYacc.Core/fsyaccpars.fsy @@ -22,13 +22,16 @@ open FsLexYacc.FsYacc.AST spec: headeropt decls PERCENT_PERCENT rules - { List.foldBack (fun f x -> f x) $2 { Header=$1;Tokens=[];Types=[];Associativities=[];StartSymbols=[];Rules=$4 } } + { + ($2, { Header=$1;Tokens=[];Types=[];Associativities=[];StartSymbols=[];Rules=$4 }) + ||> List.foldBack (fun f x -> f x) + } headeropt: | HEADER { $1 } | - { "", (parseState.ResultRange |> fst)} + { "", parseState.ResultRange } decls: { [] } @@ -42,14 +45,17 @@ decl: | RIGHT idents { (fun x -> {x with Associativities = x.Associativities @ [(List.map (fun x -> (x,RightAssoc)) $2)]} ) } | NONASSOC idents { (fun x -> {x with Associativities = x.Associativities @ [(List.map (fun x -> (x,NonAssoc)) $2)]} ) } -idents: IDENT idents { $1 :: $2 } | { [] } +ident: +| IDENT { $1, parseState.InputRange 1 } + +idents: ident idents { $1 :: $2 } | { [] } rules: rule rules { $1 :: $2 } | rule { [$1] } -rule: IDENT COLON optbar clauses optsemi { ($1,$4) } +rule: ident COLON optbar clauses optsemi { ($1,$4) } optbar: { } | BAR { } optsemi: { } | SEMI { } clauses: clause BAR clauses {$1 :: $3 } | clause { [$1] } clause: syms optprec CODE { Rule($1,$2,Some $3) } -syms: IDENT syms { $1 :: $2 } | ERROR syms { "error" :: $2 } | { [] } -optprec: { None } | PREC IDENT { Some $2 } +syms: ident syms { $1 :: $2 } | ERROR syms { ("error", Range.Empty) :: $2 } | { [] } +optprec: { None } | PREC ident { Some $2 } diff --git a/tests/FsLex.Core.Tests/FsLex.Core.Tests.fsproj b/tests/FsLex.Core.Tests/FsLex.Core.Tests.fsproj index 2855b0a1..f3815261 100644 --- a/tests/FsLex.Core.Tests/FsLex.Core.Tests.fsproj +++ b/tests/FsLex.Core.Tests/FsLex.Core.Tests.fsproj @@ -7,7 +7,6 @@ - diff --git a/tests/FsLex.Core.Tests/Main.fs b/tests/FsLex.Core.Tests/Main.fs index fd905faa..c3cb32a7 100644 --- a/tests/FsLex.Core.Tests/Main.fs +++ b/tests/FsLex.Core.Tests/Main.fs @@ -1,11 +1,9 @@ module FsLex.Core.Tests open Expecto -open FsLex.Core - -let parse file = - FsLexYacc.FsLex.AST.Compile +open FsLexYacc.FsLex +let parse file = AST.Compile [] let main argv = diff --git a/tests/FsYacc.Core.Tests/FsYacc.Core.Tests.fsproj b/tests/FsYacc.Core.Tests/FsYacc.Core.Tests.fsproj index 1d21ffc8..075a7ed5 100644 --- a/tests/FsYacc.Core.Tests/FsYacc.Core.Tests.fsproj +++ b/tests/FsYacc.Core.Tests/FsYacc.Core.Tests.fsproj @@ -7,7 +7,6 @@ - diff --git a/tests/FsYacc.Core.Tests/Main.fs b/tests/FsYacc.Core.Tests/Main.fs index 7f512186..10004c6e 100644 --- a/tests/FsYacc.Core.Tests/Main.fs +++ b/tests/FsYacc.Core.Tests/Main.fs @@ -1,6 +1,8 @@ module FsYacc.Core.Tests open Expecto +open FsLexYacc.FsYacc + [] let main argv = Tests.runTestsInAssembly defaultConfig argv diff --git a/tests/JsonLexAndYaccExample/Parser.fsy b/tests/JsonLexAndYaccExample/Parser.fsy index 10d81ba2..e26741ac 100644 --- a/tests/JsonLexAndYaccExample/Parser.fsy +++ b/tests/JsonLexAndYaccExample/Parser.fsy @@ -23,7 +23,8 @@ open JsonParsing %% -start: prog { $1 } +start: + | prog { $1 } prog: | EOF { None } diff --git a/tests/fsyacc/parsing.fs b/tests/fsyacc/parsing.fs index bdc201f1..1688478f 100644 --- a/tests/fsyacc/parsing.fs +++ b/tests/fsyacc/parsing.fs @@ -23,21 +23,21 @@ let dummyProvider = let mutable parse_information = dummyProvider let set_parse_state (x:IParseState) = parse_information <- x -let enforce_nonnull_pos p = - match (box p) with - | null -> Position.Empty - | _ -> p +let enforce_nonnull_pos (r: Range) = + match (box r) with + | null -> Range.Empty + | _ -> r -let symbol_start_pos () = parse_information.ResultRange |> fst |> enforce_nonnull_pos -let symbol_end_pos () = parse_information.ResultRange |> snd |> enforce_nonnull_pos -let rhs_start_pos (n:int) = parse_information.InputRange(n) |> fst |> enforce_nonnull_pos -let rhs_end_pos (n:int) = parse_information.InputRange(n) |> snd |> enforce_nonnull_pos +let symbol_start_pos () = parse_information.ResultRange |> enforce_nonnull_pos +let symbol_end_pos () = parse_information.ResultRange |> enforce_nonnull_pos +let rhs_start_pos (n:int) = parse_information.InputRange(n) |> enforce_nonnull_pos +let rhs_end_pos (n:int) = parse_information.InputRange(n) |> enforce_nonnull_pos exception Parse_error = RecoverableParseError let parse_error s = parse_information.RaiseError()(failwith s : unit) -let symbol_start () = (symbol_start_pos()).pos_cnum -let symbol_end () = (symbol_end_pos()).pos_cnum -let rhs_start n = (rhs_start_pos n).pos_cnum -let rhs_end n = (rhs_end_pos n).pos_cnum +let symbol_start () = (symbol_start_pos()).startPos.pos_cnum +let symbol_end () = (symbol_end_pos()).endPos.pos_cnum +let rhs_start n = (rhs_start_pos n).startPos.pos_cnum +let rhs_end n = (rhs_end_pos n).endPos.pos_cnum