@@ -1031,6 +1031,52 @@ def test_worker_queue_saturation_drains_detached_waiting_file
10311031 assert_empty driver . instance . instance_variable_get ( :@worker_deferred_unwatch )
10321032 end
10331033
1034+ def test_oversized_worker_batch_does_not_block_other_files
1035+ tail_input = Fluent ::Plugin ::TailInput
1036+ original_byte_limit = tail_input . const_get ( :WORKER_QUEUE_BYTES )
1037+ tail_input . send ( :remove_const , :WORKER_QUEUE_BYTES )
1038+ tail_input . const_set ( :WORKER_QUEUE_BYTES , 6 )
1039+ large_path = "#{ @tmp_dir } /tail-large.txt"
1040+ short_path = "#{ @tmp_dir } /tail-short.txt"
1041+ [ large_path , short_path ] . each { |path | Fluent ::FileWrapper . open ( path , 'wb' ) { } }
1042+ config = config_element ( 'ROOT' , '' , {
1043+ 'path' => "#{ @tmp_dir } /tail-*.txt" ,
1044+ 'pos_file' => "#{ @tmp_dir } /tail.pos" ,
1045+ 'tag' => 't1' ,
1046+ 'read_from_head' => true ,
1047+ 'format' => '/(?<message>.*)/' ,
1048+ 'num_threads' => 2 ,
1049+ 'read_lines_limit' => 1 ,
1050+ 'enable_stat_watcher' => false ,
1051+ } )
1052+ driver = create_driver ( config , false )
1053+ large_started = Queue . new
1054+ continue_large = Queue . new
1055+ released = false
1056+
1057+ driver . run ( expect_records : 2 , timeout : 10 ) do
1058+ pool = driver . instance . instance_variable_get ( :@worker_pool )
1059+ wrap_worker_pool_process ( pool ) do |batch |
1060+ if batch . lines . any? { |line | line . include? ( 'oversized-record' ) }
1061+ large_started << true
1062+ continue_large . pop
1063+ end
1064+ end
1065+ Fluent ::FileWrapper . open ( large_path , 'ab' ) { |file | file . puts ( 'oversized-record' ) }
1066+ Fluent ::FileWrapper . open ( short_path , 'ab' ) { |file | file . puts ( 'x' ) }
1067+ waiting ( 5 ) { sleep 0.01 until large_started . size == 1 }
1068+ waiting ( 5 ) { sleep 0.01 until driver . events . any? { |event | event [ 2 ] [ 'message' ] == 'x' } }
1069+ continue_large << true
1070+ released = true
1071+ end
1072+
1073+ assert_equal %w[ oversized-record x ] , driver . events . map { |event | event [ 2 ] [ 'message' ] } . sort
1074+ ensure
1075+ continue_large << true if continue_large && !released
1076+ tail_input . send ( :remove_const , :WORKER_QUEUE_BYTES ) if tail_input . const_defined? ( :WORKER_QUEUE_BYTES , false )
1077+ tail_input . const_set ( :WORKER_QUEUE_BYTES , original_byte_limit ) if original_byte_limit
1078+ end
1079+
10341080 def test_worker_queue_saturation_drains_rotated_file
10351081 paths = %w[ tail-a.txt tail-b.txt tail-c.txt tail-d.txt tail-e.txt tail-f.txt ] . map { |name | "#{ @tmp_dir } /#{ name } " }
10361082 paths . each { |path | Fluent ::FileWrapper . open ( path , 'wb' ) { } }
0 commit comments