Fix counting of UTF-8 codepoints in TextFlow

For line-wrapping bytes were counted instead of codepoints. This
resulted in line-breaks being inserted at the wrong position and also
breaking UTF-8 characters.

Fixes #1022.

Author: jhasse

src/catch2/internal/catch_textflow.cpp

@@ -26,6 +26,10 @@ namespace {
         return std::memchr( chars, c, sizeof( chars ) - 1 ) != nullptr;
     }
 
+    bool isUtf8ContinuationByte( char c ) {
+        return ( static_cast<unsigned char>( c ) & 0xC0 ) == 0x80;
+    }
+
 } // namespace
 
 namespace Catch {
@@ -52,6 +56,11 @@ namespace Catch {
                 if ( it != m_string.end() ) {
                     ++m_size;
                     ++it;
+                    // Skip UTF-8 continuation bytes
+                    while ( it != m_string.end() &&
+                            isUtf8ContinuationByte( *it ) ) {
+                        ++it;
+                    }
                 }
             }
         }
@@ -114,6 +123,11 @@ namespace Catch {
         void AnsiSkippingString::const_iterator::advance() {
             assert( m_it != m_string->end() );
             m_it++;
+            // Skip UTF-8 continuation bytes
+            while ( m_it != m_string->end() &&
+                    isUtf8ContinuationByte( *m_it ) ) {
+                m_it++;
+            }
             tryParseAnsiEscapes();
         }
 
@@ -133,6 +147,11 @@ namespace Catch {
                 assert( *m_it == '\033' );
                 m_it--;
             }
+            // Skip back over UTF-8 continuation bytes to the leading byte
+            while ( isUtf8ContinuationByte( *m_it ) ) {
+                assert( m_it != m_string->begin() );
+                m_it--;
+            }
         }
 
         static bool isBoundary( AnsiSkippingString const& line,

tests/SelfTest/IntrospectiveTests/TextFlow.tests.cpp

@@ -381,6 +381,81 @@ TEST_CASE( "TextFlow::AnsiSkippingString substrings properly",
     }
 }
 
+TEST_CASE( "TextFlow::AnsiSkippingString counts UTF-8 codepoints",
+           "[TextFlow][ansiskippingstring][approvals]" ) {
+    SECTION( "2-byte codepoints" ) {
+        AnsiSkippingString str( "\xC3\xA4\xC3\xB6\xC3\xBC" ); // äöü
+        CHECK( str.size() == 3 );
+    }
+    SECTION( "3-byte codepoints" ) {
+        AnsiSkippingString str( "\xE4\xB8\xAD\xE6\x96\x87" ); // 中文
+        CHECK( str.size() == 2 );
+    }
+    SECTION( "4-byte codepoints" ) {
+        // U+1F600 U+1F60E
+        AnsiSkippingString str( "\xF0\x9F\x98\x80\xF0\x9F\x98\x8E" );
+        CHECK( str.size() == 2 );
+    }
+    SECTION( "mixed ASCII and UTF-8" ) {
+        AnsiSkippingString str( "a\xC3\xA4" "b" ); // aäb
+        CHECK( str.size() == 3 );
+    }
+    SECTION( "UTF-8 with ANSI escapes" ) {
+        AnsiSkippingString str( "\033[31m\xC3\xA4\xC3\xB6\xC3\xBC\033[0m" );
+        CHECK( str.size() == 3 );
+    }
+}
+
+TEST_CASE( "TextFlow::AnsiSkippingString iterates UTF-8 codepoints",
+           "[TextFlow][ansiskippingstring][approvals]" ) {
+    // "aäb" = 'a' (0x61), 'ä' (0xC3 0xA4), 'b' (0x62)
+    std::string text = "a\xC3\xA4" "b";
+    AnsiSkippingString str( text );
+
+    SECTION( "forward iteration has correct count" ) {
+        int count = 0;
+        for ( auto it = str.begin(); it != str.end(); ++it ) {
+            ++count;
+        }
+        CHECK( count == 3 );
+    }
+    SECTION( "backward iteration has correct count" ) {
+        auto it = str.end();
+        int count = 0;
+        while ( it != str.begin() ) {
+            --it;
+            ++count;
+        }
+        CHECK( count == 3 );
+    }
+    SECTION( "substring preserves full UTF-8 bytes" ) {
+        auto a = str.begin();
+        auto b = str.begin();
+        ++b; // past 'a'
+        ++b; // past 'ä'
+        CHECK( str.substring( a, b ) == "a\xC3\xA4" );
+    }
+}
+
+TEST_CASE( "TextFlow::Column wraps UTF-8 text correctly",
+           "[TextFlow][column][approvals]" ) {
+    // "äöü äöü äöü" = 11 codepoints, 17 bytes
+    Column col( "\xC3\xA4\xC3\xB6\xC3\xBC \xC3\xA4\xC3\xB6\xC3\xBC \xC3\xA4\xC3\xB6\xC3\xBC" );
+
+    SECTION( "width=8" ) {
+        col.width( 8 );
+        // 7 visible codepoints "äöü äöü" fit, then wrap
+        REQUIRE( as_written( col ) ==
+                 "\xC3\xA4\xC3\xB6\xC3\xBC \xC3\xA4\xC3\xB6\xC3\xBC\n"
+                 "\xC3\xA4\xC3\xB6\xC3\xBC" );
+    }
+    SECTION( "width=80" ) {
+        col.width( 80 );
+        REQUIRE( as_written( col ) ==
+                 "\xC3\xA4\xC3\xB6\xC3\xBC \xC3\xA4\xC3\xB6\xC3\xBC \xC3\xA4\xC3\xB6\xC3\xBC" );
+    }
+}
+
 TEST_CASE( "TextFlow::Column skips ansi escape sequences",
            "[TextFlow][column][approvals]" ) {
     std::string text = "\033[38;2;98;174;239m\033[38;2;198;120;221mThe quick brown \033[38;2;198;120;221mfox jumped over the lazy dog\033[0m";