fix(cpp): preserve quoted CSV line endings in tsfile-cli (#996)

* fix(cpp): preserve quoted CSV line endings in tsfile-cli

* test(cpp): write embedded-newline CSV fixture in binary mode

Author: ColinLeeo

cpp/test/tools/command_e2e_test.cc

@@ -849,7 +849,8 @@ TEST(CliE2E, WriteImportsQuotedFieldWithEmbeddedNewline) {
     std::string csv_path =
         tsfile_cli_test::unique_temp_path("tsfile_cli_nl_in", ".csv");
     {
-        std::ofstream o(csv_path.c_str());
+        // Keep the fixture's LF bytes unchanged on Windows as well.
+        std::ofstream o(csv_path.c_str(), std::ios::binary);
         // The note field on the first row spans two physical lines inside
         // quotes; it must import as a single row, not be split into two.
         o << "time,id1,note\n0,dev,\"line one\nline two\"\n1,dev,plain\n";
@@ -1985,3 +1986,90 @@ TEST(CliE2E, WriteCatCsvWritePreservesLeadingBackslashes) {
     std::remove(exported.c_str());
     std::remove(second.c_str());
 }
+
+TEST(CliE2E, WriteCatAndExportCsvPreserveQuotedLineEndings) {
+    const std::string input =
+        tsfile_cli_test::unique_temp_path("tsfile_cli_crlf_input", ".csv");
+    const std::string first =
+        tsfile_cli_test::unique_temp_path("tsfile_cli_crlf_first", ".tsfile");
+    const std::string expected_csv =
+        "time,site,message,note\n"
+        "0,\"s\r\n1\",\"first\r\nsecond\",\"\"\"quoted\"\"\r\n\r\nlast\"\n"
+        "1,\"s\r\n1\",\"lf\nonly\",\"mixed\r\nand\nmore\"\n"
+        "2,\"s\r\n1\",\"carriage\rreturn\",\"\r\n\"\n"
+        "3,\"s\r\n1\",\"ends\r\",\"line\r\nend\"\n";
+    {
+        std::ofstream csv(input.c_str(), std::ios::binary);
+        ASSERT_TRUE(csv.good());
+        // CRLF record boundaries, embedded CRLF/LF/CR, and no final newline.
+        csv << "time,site,message,note\r\n"
+               "0,\"s\r\n1\",\"first\r\nsecond\","
+               "\"\"\"quoted\"\"\r\n\r\nlast\"\r\n"
+               "1,\"s\r\n1\",\"lf\nonly\",\"mixed\r\nand\nmore\"\r\n"
+               "2,\"s\r\n1\",\"carriage\rreturn\",\"\r\n\"\r\n"
+               "3,\"s\r\n1\",\"ends\r\",\"line\r\nend\"";
+    }
+    const auto write = [&](const std::string& source,
+                           const std::string& target) {
+        std::ostringstream out, err;
+        const int code = tsfile_cli::run_cli(
+            {"write", "--table", "t", "--tag", "site", "STRING", "--field",
+             "message", "STRING", "--field", "note", "TEXT", "-i", source, "-o",
+             target},
+            out, err);
+        EXPECT_EQ(code, 0) << err.str();
+        return code;
+    };
+    const auto read = [&](const std::string& path, const std::string& format) {
+        std::ostringstream out, err;
+        EXPECT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", format, path},
+                                      out, err),
+                  0)
+            << err.str();
+        return out.str();
+    };
+    const std::string expected_ndjson =
+        R"json({"time":"0","site":"s\r\n1","message":"first\r\nsecond","note":"\"quoted\"\r\n\r\nlast"})json"
+        "\n"
+        R"json({"time":"1","site":"s\r\n1","message":"lf\nonly","note":"mixed\r\nand\nmore"})json"
+        "\n"
+        R"json({"time":"2","site":"s\r\n1","message":"carriage\rreturn","note":"\r\n"})json"
+        "\n"
+        R"json({"time":"3","site":"s\r\n1","message":"ends\r","note":"line\r\nend"})json"
+        "\n";
+    ASSERT_EQ(write(input, first), 0);
+    EXPECT_EQ(read(first, "ndjson"), expected_ndjson);
+
+    for (const std::string command : {"cat", "export"}) {
+        SCOPED_TRACE(command);
+        const std::string exported = tsfile_cli_test::unique_temp_path(
+            "tsfile_cli_crlf_" + command, ".csv");
+        const std::string second = tsfile_cli_test::unique_temp_path(
+            "tsfile_cli_crlf_" + command, ".tsfile");
+        if (command == "cat") {
+            const std::string csv = read(first, "csv");
+            EXPECT_EQ(csv, expected_csv);
+            std::ofstream file(exported.c_str(), std::ios::binary);
+            ASSERT_TRUE(file.good());
+            file << csv;
+        } else {
+            std::ostringstream out, err;
+            ASSERT_EQ(tsfile_cli::run_cli({"export", "-t", "t", "--type", "csv",
+                                           "-o", exported, first},
+                                          out, err),
+                      0)
+                << err.str();
+            std::ifstream file(exported.c_str(), std::ios::binary);
+            ASSERT_TRUE(file.good());
+            std::ostringstream bytes;
+            bytes << file.rdbuf();
+            EXPECT_EQ(bytes.str(), expected_csv);
+        }
+        ASSERT_EQ(write(exported, second), 0);
+        EXPECT_EQ(read(second, "ndjson"), expected_ndjson);
+        std::remove(exported.c_str());
+        std::remove(second.c_str());
+    }
+    std::remove(input.c_str());
+    std::remove(first.c_str());
+}

cpp/test/tools/input_format_test.cc

@@ -158,16 +158,52 @@ TEST(InputFormatTest, ReadRecordStripsCarriageReturn) {
     long long n = 0;
     ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n));
     EXPECT_EQ(rec, "a,b");
+    EXPECT_EQ(n, 1);
+    ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n));
+    EXPECT_EQ(rec, "c,d");
+    EXPECT_EQ(n, 1);
+    EXPECT_FALSE(tsfile_cli::read_record(in, true, rec, n));
+}
+
+TEST(InputFormatTest, ReadRecordPreservesQuotedLineEndings) {
+    std::istringstream in(
+        "1,\"line one\r\n\r\nline \"\"two\"\"\nline three\",x\r\n"
+        "2,plain,y\r\n");
+    std::string rec;
+    long long n = 0;
+    ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n));
+    EXPECT_EQ(rec, "1,\"line one\r\n\r\nline \"\"two\"\"\nline three\",x");
+    EXPECT_EQ(n, 4);
+    const auto fields = tsfile_cli::split_line(rec, ',', true);
+    ASSERT_EQ(fields.size(), 3u);
+    EXPECT_EQ(fields[1], "line one\r\n\r\nline \"two\"\nline three");
+    ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n));
+    EXPECT_EQ(rec, "2,plain,y");
+    EXPECT_EQ(n, 1);
+    EXPECT_FALSE(tsfile_cli::read_record(in, true, rec, n));
+}
+
+TEST(InputFormatTest, ReadRecordPreservesQuotedCarriageReturnAtEof) {
+    std::istringstream in("1,\"never closed\r");
+    std::string rec;
+    long long n = 0;
+    ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n));
+    EXPECT_EQ(rec, "1,\"never closed\r");
+    EXPECT_EQ(n, 1);
+    EXPECT_FALSE(tsfile_cli::read_record(in, true, rec, n));
 }
 
 TEST(InputFormatTest, ReadRecordTsvIgnoresQuotes) {
     // With csv_quotes false a quote is just data; no line joining happens.
-    std::istringstream in("1\t\"open\n2\tclosed\n");
+    std::istringstream in("1\t\"open\r\n2\tclosed\r\n");
     std::string rec;
     long long n = 0;
     ASSERT_TRUE(tsfile_cli::read_record(in, false, rec, n));
     EXPECT_EQ(rec, "1\t\"open");
     EXPECT_EQ(n, 1);
+    ASSERT_TRUE(tsfile_cli::read_record(in, false, rec, n));
+    EXPECT_EQ(rec, "2\tclosed");
+    EXPECT_EQ(n, 1);
 }
 
 TEST(InputFormatTest, ReadRecordUnterminatedQuoteAtEof) {

cpp/tools/commands/cmd_write.cc

@@ -562,7 +562,7 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/,
             return kExitFile;
         }
         has_input_stat = true;
-        fin.open(args.file.c_str());
+        fin.open(args.file.c_str(), std::ios::binary);
         if (!fin.is_open()) {
             err << "Error: cannot open input: " << args.file << "\n";
             return kExitFile;

cpp/tools/format/input_format.cc

@@ -44,23 +44,26 @@ bool read_record(std::istream& in, bool csv_quotes, std::string& record,
     std::string physical;
     bool open_quote = false;
     while (std::getline(in, physical)) {
-        strip_cr(physical);
         ++lines_consumed;
-        if (!record.empty()) {
-            record.push_back('\n');  // restore the newline inside the field
-        }
-        record += physical;
         if (csv_quotes) {
             for (char c : physical) {
                 if (c == '"') {
                     open_quote = !open_quote;
                 }
             }
-            if (open_quote) {
-                continue;  // quote still open: the field spans the next line
-            }
         }
-        return true;
+        if (!open_quote) {
+            // Strip CR from record boundaries, but keep it as field data
+            // when the quoted field continues onto the next physical line.
+            strip_cr(physical);
+        }
+        if (!record.empty()) {
+            record.push_back('\n');  // restore the newline inside the field
+        }
+        record += physical;
+        if (!open_quote) {
+            return true;
+        }
     }
     return lines_consumed > 0;  // trailing record with an unterminated quote
 }
@@ -80,8 +83,7 @@ std::vector<std::string> split_line(const std::string& line, char delim,
 }
 
 bool normalize_write_columns(const std::vector<WriteColumnSpec>& specs,
-                             std::vector<ColumnDef>& out,
-                             std::string& error) {
+                             std::vector<ColumnDef>& out, std::string& error) {
     out.clear();
     if (specs.empty()) {
         error = "empty write column declarations";

cpp/tools/format/input_format.h

@@ -38,8 +38,7 @@ struct ColumnDef {
 
 bool parse_datatype_name(const std::string& s, common::TSDataType& out);
 bool normalize_write_columns(const std::vector<WriteColumnSpec>& specs,
-                             std::vector<ColumnDef>& out,
-                             std::string& error);
+                             std::vector<ColumnDef>& out, std::string& error);
 std::vector<std::string> split_line(const std::string& line, char delim,
                                     bool csv_quotes);
 bool parse_bool_cell(const std::string& s, bool& out);
@@ -49,9 +48,12 @@ bool validate_identifier(const std::string& name, std::string& error);
 
 // Read one logical record from `in`. When `csv_quotes` is true a field may span
 // multiple physical lines if it opens a double-quote that is not yet closed, so
-// continuation lines are appended (newline preserved) until the quote closes.
-// `lines_consumed` returns how many physical lines were read, so callers can
-// keep accurate line numbers. Returns false at end of input with no record.
+// continuation lines are appended (LF or CRLF preserved) until the quote
+// closes. CRLF outside quotes is a record boundary, not field data. File
+// streams should be opened in binary mode to prevent platform newline
+// translation. `lines_consumed` returns how many physical lines were read, so
+// callers can keep accurate line numbers. Returns false at end of input with no
+// record.
 bool read_record(std::istream& in, bool csv_quotes, std::string& record,
                  long long& lines_consumed);
 

cpp/tools/tools_main.cc

@@ -19,7 +19,12 @@
 
 #include <exception>
 #include <iostream>
-#ifndef _WIN32
+#ifdef _WIN32
+#include <fcntl.h>
+#include <io.h>
+
+#include <cstdio>
+#else
 #include <csignal>
 #endif
 #include <string>
@@ -29,7 +34,19 @@
 #include "cli/run_cli.h"
 
 int main(int argc, char** argv) {
-#ifndef _WIN32
+#ifdef _WIN32
+    // Preserve CSV field bytes in pipes and redirected files. CRT text mode
+    // otherwise converts CRLF on input and LF on output, including inside
+    // quotes.
+    const auto binary_if_redirected = [](FILE* stream) {
+        const int fd = _fileno(stream);
+        if (fd >= 0 && _isatty(fd) == 0) {
+            _setmode(fd, _O_BINARY);
+        }
+    };
+    binary_if_redirected(stdin);
+    binary_if_redirected(stdout);
+#else
     std::signal(SIGPIPE, SIG_IGN);
 #endif
     std::vector<std::string> args(argv + 1, argv + argc);