Merge pull request #4 from clue-labs/multiline

Support parsing multiline CSV values with newlines in cell values

Author: clue

README.md

@@ -64,14 +64,15 @@ Carol,40
 Dave,30
 ```
 
-CSV allows handling field values that contain spaces or the delimiting comma
-(think of URLs or user-provided descriptions) by enclosing them with quotes like
-this:
+CSV allows handling field values that contain spaces, the delimiting comma or
+even newline characters (think of URLs or user-provided descriptions) by
+enclosing them with quotes like this:
 
 ```
 name,comment
 Alice,"Yes, I like cheese"
-Bob,"Hello World!"
+Bob,"Hello
+World!"
 ```
 
 > Note that these more advanced parsing rules are often handled inconsistently

src/Decoder.php

@@ -13,14 +13,14 @@
 class Decoder extends EventEmitter implements ReadableStreamInterface
 {
     private $input;
-    private $temp = false;
 
     private $delimiter;
     private $enclosure;
     private $escapeChar;
     private $maxlength;
 
     private $buffer = '';
+    private $offset = 0;
     private $closed = false;
 
     public function __construct(ReadableStreamInterface $input, $delimiter = ',', $enclosure = '"', $escapeChar = '\\', $maxlength = 65536)
@@ -35,8 +35,6 @@ public function __construct(ReadableStreamInterface $input, $delimiter = ',', $e
             return $this->close();
         }
 
-        $this->temp = fopen('php://memory', 'r+');
-
         $this->input->on('data', array($this, 'handleData'));
         $this->input->on('end', array($this, 'handleEnd'));
         $this->input->on('error', array($this, 'handleError'));
@@ -56,12 +54,6 @@ public function close()
 
         $this->closed = true;
         $this->buffer = '';
-
-        if ($this->temp !== false) {
-            fclose($this->temp);
-            $this->temp = false;
-        }
-
         $this->input->close();
 
         $this->emit('close');
@@ -91,21 +83,34 @@ public function handleData($data)
         $this->buffer .= $data;
 
         // keep parsing while a newline has been found
-        while (($newline = strpos($this->buffer, "\n")) !== false && $newline <= $this->maxlength) {
-            // read data up until newline and remove from buffer
-            ftruncate($this->temp, 0);
-            fwrite($this->temp, (string)substr($this->buffer, 0, $newline));
-            rewind($this->temp);
-            $this->buffer = (string)substr($this->buffer, $newline + 1);
-
-            $data = fgetcsv($this->temp, 0, $this->delimiter, $this->enclosure, $this->escapeChar);
-
-            // abort stream if decoding failed
-            if ($data === false) {
+        while (($newline = \strpos($this->buffer, "\n", $this->offset)) !== false && $newline <= $this->maxlength) {
+            // read data up until newline and try to parse
+            $data = \str_getcsv(
+                \substr($this->buffer, 0, $newline + 1),
+                $this->delimiter,
+                $this->enclosure,
+                $this->escapeChar
+            );
+
+            // unable to decode? abort
+            if ($data === false || \end($data) === null) {
                 $this->handleError(new \RuntimeException('Unable to decode CSV'));
                 return;
             }
 
+            // the last parsed cell value ends with a newline and the buffer does not end with end quote?
+            // this looks like a multiline value, so only remember offset and wait for next newline
+            $last = \substr(\end($data), -1);
+            \reset($data);
+            if ($last === "\n" && ($newline === 1 || $this->buffer[$newline - 1] !== $this->enclosure)) {
+                $this->offset = $newline + 1;
+                continue;
+            }
+
+            // parsing successful => remove from buffer and emit
+            $this->buffer = (string)\substr($this->buffer, $newline + 1);
+            $this->offset = 0;
+
             $this->emit('data', array($data));
         }
 
@@ -121,6 +126,10 @@ public function handleEnd()
             $this->handleData("\n");
         }
 
+        if ($this->buffer !== '') {
+            $this->handleError(new \RuntimeException('Unable to decode CSV'));
+        }
+
         if (!$this->closed) {
             $this->emit('end');
             $this->close();

tests/DecoderTest.php

@@ -60,6 +60,34 @@ public function testEmitDataStringWillForward()
         $this->input->emit('data', array("\"hello\"\n"));
     }
 
+    public function testEmitDataStringWithNewlineWillForward()
+    {
+        $this->decoder->on('data', $this->expectCallableOnceWith(array("hello" . "\n" . "world")));
+
+        $this->input->emit('data', array("\"hello\nworld\"\n"));
+    }
+
+    public function testEmitDataStringWithMultiNewlineWillForward()
+    {
+        $this->decoder->on('data', $this->expectCallableOnceWith(array("hello" . "\n\n" . "world")));
+
+        $this->input->emit('data', array("\"hello\n\nworld\"\n"));
+    }
+
+    public function testEmitDataStringEndsWithNewlineWillForward()
+    {
+        $this->decoder->on('data', $this->expectCallableOnceWith(array("hello" . "\n")));
+
+        $this->input->emit('data', array("\"hello\n\"\n"));
+    }
+
+    public function testEmitDataStringOnlyNewlineWillForward()
+    {
+        $this->decoder->on('data', $this->expectCallableOnceWith(array("\n")));
+
+        $this->input->emit('data', array("\"\n\"\n"));
+    }
+
     public function testEmitDataWithoutNewlineWillNotForward()
     {
         $this->decoder->on('data', $this->expectCallableNever());
@@ -90,6 +118,7 @@ public function testEmitDataErrorWillForwardError()
         $this->decoder->on('error', $this->expectCallableOnce());
 
         $this->input->emit('data', array("\"hello\\\"test\n"));
+        $this->input->emit('end');
     }
 
     public function testEmitDataErrorInMultipleChunksWillForwardError()
@@ -99,6 +128,7 @@ public function testEmitDataErrorInMultipleChunksWillForwardError()
 
         $this->input->emit('data', array("\"hello"));
         $this->input->emit('data', array("\\\"test\n"));
+        $this->input->emit('end');
     }
 
     public function testEmitDataWithExactBufferSizeWillForward()

tests/EncoderTest.php

@@ -62,6 +62,17 @@ public function testWriteArrayWithStringWithSpacesUsesEnclosing()
         $this->encoder->write(array('hello world'));
     }
 
+    public function testWriteArrayWithStringWithNewlineUsesEnclosing()
+    {
+        $this->output = $this->getMockBuilder('React\Stream\WritableStreamInterface')->getMock();
+        $this->output->expects($this->once())->method('isWritable')->willReturn(true);
+        $this->encoder = new Encoder($this->output);
+
+        $this->output->expects($this->once())->method('write')->with("\"hello\nworld\"\n");
+
+        $this->encoder->write(array("hello\nworld"));
+    }
+
     public function testWriteArrayWithSpecialStringUsesEnclosing()
     {
         $this->output = $this->getMockBuilder('React\Stream\WritableStreamInterface')->getMock();