diff --git a/src/FsLex.Core/fslexdriver.fs b/src/FsLex.Core/fslexdriver.fs index 4412a14..31e361c 100644 --- a/src/FsLex.Core/fslexdriver.fs +++ b/src/FsLex.Core/fslexdriver.fs @@ -27,10 +27,16 @@ type GeneratorState = type PerRuleData = list> type DfaNodes = list -type Writer(outputFileName, outputFileInterface) = - let os = File.CreateText outputFileName :> TextWriter +type Writer(outputFileName, outputFileInterface, isWithUnicodeByteOrderMark) = + + let os = + new StreamWriter(outputFileName, false, new System.Text.UTF8Encoding(isWithUnicodeByteOrderMark)) :> TextWriter + let mutable lineCount = 0 - let osi = File.CreateText outputFileInterface :> TextWriter + + let osi = + new StreamWriter(outputFileInterface, false, new System.Text.UTF8Encoding(isWithUnicodeByteOrderMark)) :> TextWriter + let mutable interfaceLineCount = 0 let incr () = lineCount <- lineCount + 1 @@ -297,9 +303,9 @@ let writeBottomCode code (writer: Writer) = writer.WriteCode code let writeFooter outputFileName (writer: Writer) = writer.WriteLine "# 3000000 \"%s\"" outputFileName -let writeSpecToFile (state: GeneratorState) (spec: Spec) (perRuleData: PerRuleData) (dfaNodes: DfaNodes) = +let writeSpecToFile (state: GeneratorState) (spec: Spec) (perRuleData: PerRuleData) (dfaNodes: DfaNodes) isWithUnicodeByteOrderMark = let output, outputi = state.outputFileName, String.Concat(state.outputFileName, "i") - use writer = new Writer(output, outputi) + use writer = new Writer(output, outputi, isWithUnicodeByteOrderMark) writeLightMode state.disableLightMode state.outputFileName writer writeModuleExpression state.generatedModuleName state.generateInternalModule writer writeOpens state.opens writer diff --git a/src/FsLex/fslex.fs b/src/FsLex/fslex.fs index 39fbeff..4d75eed 100644 --- a/src/FsLex/fslex.fs +++ b/src/FsLex/fslex.fs @@ -21,10 +21,16 @@ let mutable opens = [] let mutable lexlib = "FSharp.Text.Lexing" let mutable unicode = false let mutable caseInsensitive = false +let mutable isOutputWithUnicodeByteOrderMark = false let usage = [ ArgInfo("-o", ArgType.String(fun s -> out <- Some s), "Name the output file.") + ArgInfo( + "--out-bom", + ArgType.Unit(fun () -> isOutputWithUnicodeByteOrderMark <- true), + "Generate output files with Unicode Byte Order Mark." + ) ArgInfo("--module", ArgType.String(fun s -> modname <- Some s), "Define the F# module name to host the generated parser.") ArgInfo("--internal", ArgType.Unit(fun () -> internal_module <- true), "Generate an internal module") ArgInfo( @@ -118,7 +124,7 @@ let main () = domain = if unicode then Unicode else ASCII } - writeSpecToFile state spec perRuleData dfaNodes + writeSpecToFile state spec perRuleData dfaNodes isOutputWithUnicodeByteOrderMark with e -> eprintf diff --git a/src/FsYacc.Core/fsyaccdriver.fs b/src/FsYacc.Core/fsyaccdriver.fs index de61173..1843d16 100644 --- a/src/FsYacc.Core/fsyaccdriver.fs +++ b/src/FsYacc.Core/fsyaccdriver.fs @@ -101,10 +101,15 @@ type NullLogger() = interface IDisposable with member _.Dispose() = () -type Writer(outputFileName, outputFileInterface) = - let os = File.CreateText outputFileName :> TextWriter +type Writer(outputFileName, outputFileInterface, isWithUnicodeByteOrderMark) = + let os = + new StreamWriter(outputFileName, false, new System.Text.UTF8Encoding(isWithUnicodeByteOrderMark)) :> TextWriter + let mutable outputLineCount = 0 - let osi = File.CreateText outputFileInterface :> TextWriter + + let osi = + new StreamWriter(outputFileInterface, false, new System.Text.UTF8Encoding(isWithUnicodeByteOrderMark)) :> TextWriter + let mutable interfaceLineCount = 0 member x.Write format = fprintf os format @@ -198,12 +203,12 @@ type GeneratorState = bufferTypeArgument = "'cty" } -let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compiledSpec: CompiledSpec) = +let writeSpecToFile (generatorState: GeneratorState) (spec: ParserSpec) (compiledSpec: CompiledSpec) isWithUnicodeByteOrderMark = let output, outputi = deriveOutputFileNames (generatorState.input, generatorState.output) generatorState.logger.Log " Output file describing compiled parser placed in %s and %s" output outputi - use writer = new Writer(output, outputi) + use writer = new Writer(output, outputi, isWithUnicodeByteOrderMark) writer.WriteLine "// Implementation file for parser generated by fsyacc" writer.WriteLineInterface "// Signature file for parser generated by fsyacc" diff --git a/src/FsYacc/fsyacc.fs b/src/FsYacc/fsyacc.fs index 23fde41..b3523d6 100644 --- a/src/FsYacc/fsyacc.fs +++ b/src/FsYacc/fsyacc.fs @@ -23,10 +23,16 @@ let mutable inputCodePage = None let mutable lexlib = "FSharp.Text.Lexing" let mutable parslib = "FSharp.Text.Parsing" let mutable bufferTypeArgument = "'cty" +let mutable isOutputWithUnicodeByteOrderMark = false let usage = [ ArgInfo("-o", ArgType.String(fun s -> out <- Some s), "Name the output file.") + ArgInfo( + "--out-bom", + ArgType.Unit(fun () -> isOutputWithUnicodeByteOrderMark <- true), + "Generate output files with Unicode Byte Order Mark." + ) ArgInfo("-v", ArgType.Unit(fun () -> log <- true), "Produce a listing file.") ArgInfo("--module", ArgType.String(fun s -> modname <- Some s), "Define the F# module name to host the generated parser.") ArgInfo("--internal", ArgType.Unit(fun () -> internal_module <- true), "Generate an internal module") @@ -120,7 +126,7 @@ let main () = bufferTypeArgument = bufferTypeArgument } - writeSpecToFile generatorState spec compiledSpec + writeSpecToFile generatorState spec compiledSpec isOutputWithUnicodeByteOrderMark let result = try diff --git a/tests/LexAndYaccMiniProject/LexAndYaccMiniProject.fsproj b/tests/LexAndYaccMiniProject/LexAndYaccMiniProject.fsproj index 90bb7d9..e647ac7 100644 --- a/tests/LexAndYaccMiniProject/LexAndYaccMiniProject.fsproj +++ b/tests/LexAndYaccMiniProject/LexAndYaccMiniProject.fsproj @@ -8,10 +8,10 @@ - --module Parser + --module Parser --out-bom - --module Lexer --unicode + --module Lexer --unicode --out-bom