Feature/output encoding.0

src/FsLex.Core/fslexdriver.fs

@@ -27,10 +27,16 @@ type GeneratorState =
 type PerRuleData = list<DfaNode * seq<Code>>
 type DfaNodes = list<DfaNode>
 
-type Writer(outputFileName, outputFileInterface) =
-    let os = File.CreateText outputFileName :> TextWriter
+type Writer(outputFileName, outputFileInterface, isWithUnicodeByteOrderMark) =
+
+    let os =
+        new StreamWriter(outputFileName, false, new System.Text.UTF8Encoding(isWithUnicodeByteOrderMark)) :> TextWriter
+
     let mutable lineCount = 0
-    let osi = File.CreateText outputFileInterface :> TextWriter
+
+    let osi =
+        new StreamWriter(outputFileInterface, false, new System.Text.UTF8Encoding(isWithUnicodeByteOrderMark)) :> TextWriter
+
     let mutable interfaceLineCount = 0
     let incr () = lineCount <- lineCount + 1
 
@@ -297,9 +303,9 @@ let writeBottomCode code (writer: Writer) = writer.WriteCode code
 let writeFooter outputFileName (writer: Writer) =
     writer.WriteLine "# 3000000 \"%s\"" outputFileName
 
-let writeSpecToFile (state: GeneratorState) (spec: Spec) (perRuleData: PerRuleData) (dfaNodes: DfaNodes) =
+let writeSpecToFile (state: GeneratorState) (spec: Spec) (perRuleData: PerRuleData) (dfaNodes: DfaNodes) isWithUnicodeByteOrderMark =
     let output, outputi = state.outputFileName, String.Concat(state.outputFileName, "i")
-    use writer = new Writer(output, outputi)
+    use writer = new Writer(output, outputi, isWithUnicodeByteOrderMark)
     writeLightMode state.disableLightMode state.outputFileName writer
     writeModuleExpression state.generatedModuleName state.generateInternalModule writer
     writeOpens state.opens writer

src/FsLex/fslex.fs

@@ -21,10 +21,16 @@ let mutable opens = []
 let mutable lexlib = "FSharp.Text.Lexing"
 let mutable unicode = false
 let mutable caseInsensitive = false
+let mutable isOutputWithUnicodeByteOrderMark = false
 
 let usage =
     [
         ArgInfo("-o", ArgType.String(fun s -> out <- Some s), "Name the output file.")
+        ArgInfo(
+            "--out-bom",
+            ArgType.Unit(fun () -> isOutputWithUnicodeByteOrderMark <- true),
+            "Generate output files with Unicode Byte Order Mark."
+        )
         ArgInfo("--module", ArgType.String(fun s -> modname <- Some s), "Define the F# module name to host the generated parser.")
         ArgInfo("--internal", ArgType.Unit(fun () -> internal_module <- true), "Generate an internal module")
         ArgInfo(
@@ -118,7 +124,7 @@ let main () =
                 domain = if unicode then Unicode else ASCII
             }
 
-        writeSpecToFile state spec perRuleData dfaNodes
+        writeSpecToFile state spec perRuleData dfaNodes isOutputWithUnicodeByteOrderMark
 
     with e ->
         eprintf

src/FsYacc.Core/fsyaccdriver.fs

@@ -101,10 +101,15 @@ type NullLogger() =
     interface IDisposable with
         member _.Dispose() = ()
 
-type Writer(outputFileName, outputFileInterface) =
-    let os = File.CreateText outputFileName :> TextWriter
+type Writer(outputFileName, outputFileInterface, isWithUnicodeByteOrderMark) =
+    let os =
+        new StreamWriter(outputFileName, false, new System.Text.UTF8Encoding(isWithUnicodeByteOrderMark)) :> TextWriter
+
     let mutable outputLineCount = 0
-    let osi = File.CreateText outputFileInterface :> TextWriter
+
+    let osi =
+        new StreamWriter(outputFileInterface, false, new System.Text.UTF8Encoding(isWithUnicodeByteOrderMark)) :> TextWriter
+
     let mutable interfaceLineCount = 0
 
     member x.Write format = fprintf os format
@@ -198,12 +203,12 @@ type GeneratorState =
             bufferTypeArgument = "'cty"
         }
 
-let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compiledSpec: CompiledSpec) =
+let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compiledSpec: CompiledSpec) isWithUnicodeByteOrderMark =
     let output, outputi =
         deriveOutputFileNames (generatorState.input, generatorState.output)
 
     generatorState.logger.Log "        Output file describing compiled parser placed in %s and %s" output outputi
-    use writer = new Writer(output, outputi)
+    use writer = new Writer(output, outputi, isWithUnicodeByteOrderMark)
     writer.WriteLine "// Implementation file for parser generated by fsyacc"
     writer.WriteLineInterface "// Signature file for parser generated by fsyacc"
 

src/FsYacc/fsyacc.fs

@@ -23,10 +23,16 @@ let mutable inputCodePage = None
 let mutable lexlib = "FSharp.Text.Lexing"
 let mutable parslib = "FSharp.Text.Parsing"
 let mutable bufferTypeArgument = "'cty"
+let mutable isOutputWithUnicodeByteOrderMark = false
 
 let usage =
     [
         ArgInfo("-o", ArgType.String(fun s -> out <- Some s), "Name the output file.")
+        ArgInfo(
+            "--out-bom",
+            ArgType.Unit(fun () -> isOutputWithUnicodeByteOrderMark <- true),
+            "Generate output files with Unicode Byte Order Mark."
+        )
         ArgInfo("-v", ArgType.Unit(fun () -> log <- true), "Produce a listing file.")
         ArgInfo("--module", ArgType.String(fun s -> modname <- Some s), "Define the F# module name to host the generated parser.")
         ArgInfo("--internal", ArgType.Unit(fun () -> internal_module <- true), "Generate an internal module")
@@ -120,7 +126,7 @@ let main () =
             bufferTypeArgument = bufferTypeArgument
         }
 
-    writeSpecToFile generatorState spec compiledSpec
+    writeSpecToFile generatorState spec compiledSpec isOutputWithUnicodeByteOrderMark
 
 let result =
     try

tests/LexAndYaccMiniProject/LexAndYaccMiniProject.fsproj

@@ -8,10 +8,10 @@
   </PropertyGroup>
   <ItemGroup>
     <FsYacc Include="Parser.fsy">
-      <OtherFlags>--module Parser</OtherFlags>
+      <OtherFlags>--module Parser --out-bom</OtherFlags>
     </FsYacc>
     <FsLex Include="Lexer.fsl">
-      <OtherFlags>--module Lexer --unicode</OtherFlags>
+      <OtherFlags>--module Lexer --unicode --out-bom</OtherFlags>
     </FsLex>
     <Compile Include="Parser.fsi" />
     <Compile Include="Parser.fs" />