Merge pull request #43 from keboola/zajca-bom

utf8 bom detection

Author: zajca

.travis.yml

@@ -7,12 +7,16 @@ php:
   - 7.3
   - 7.4
 
-before_script: 
+env:
+  global:
+    - XDEBUG_MODE=coverage
+
+before_script:
   - composer install
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
   - chmod +x ./cc-test-reporter
   - export GIT_COMMIT_SHA=$TRAVIS_COMMIT
-  - export GIT_BRANCH=$TRAVIS_BRANCH  
+  - export GIT_BRANCH=$TRAVIS_BRANCH
   - ./cc-test-reporter before-build
 
 script:
@@ -22,6 +26,6 @@ after_success:
   - ./cc-test-reporter after-build --exit-code 0 --debug
 
 notifications:
-  email: false  
+  email: false
   slack:
     secure: WVnUU0fkZS75md3mm7B08SxhP3HDeHbJ8GTPR1DUVjK3MHAmKeSah/plNNxn9I/TdlXnHzQO5WBN33nUq0ODGGT4WFzFa66YTX2tb+bNSmewBOv82hEoITTI1PI9SLq0WNtcamHWCM3Rt1XtiZb3DQk/OcUfiWrrn74q4PPX+VY=

src/CsvReader.php

@@ -58,7 +58,7 @@ public function __construct(
         $this->validateLineBreak();
 
         rewind($this->filePointer);
-        $this->header = $this->readLine();
+        $this->header = UTF8BOMHelper::detectAndRemoveBOM($this->readLine());
         $this->rewind();
     }
 

src/UTF8BOMHelper.php

@@ -0,0 +1,37 @@
+<?php
+
+namespace Keboola\Csv;
+
+class UTF8BOMHelper
+{
+    /**
+     * @param array $header
+     * @return array
+     */
+    public static function detectAndRemoveBOM($header)
+    {
+        if (!is_array($header)) {
+            return $header;
+        }
+        $utf32BigEndianBom = chr(0x00) . chr(0x00) . chr(0xFE) . chr(0xFF);
+        $utf32LittleEndianBom = chr(0xFF) . chr(0xFE) . chr(0x00) . chr(0x00);
+        $utf16BigEndianBom = chr(0xFE) . chr(0xFF);
+        $utf16LittleEndianBom = chr(0xFF) . chr(0xFE);
+        $utf8Bom = chr(0xEF) . chr(0xBB) . chr(0xBF);
+
+        foreach ([
+                     $utf32BigEndianBom,
+                     $utf32LittleEndianBom,
+                     $utf16BigEndianBom,
+                     $utf16LittleEndianBom,
+                     $utf8Bom,
+                 ] as $bomString) {
+            if (strpos($header[0], $bomString) === 0) {
+                $header[0] = trim(substr($header[0], strlen($bomString)), '"');
+                break;
+            }
+        }
+
+        return $header;
+    }
+}

tests/CsvReadTest.php

@@ -137,6 +137,26 @@ public function testParseEscapedBy()
         self::assertEquals($expected, iterator_to_array($csvFile));
     }
 
+    /**
+     * @dataProvider bomProvider
+     */
+    public function testUtf8BOM($bomFile)
+    {
+        $csvFile = new CsvReader(__DIR__ . '/data/bom/' . $bomFile . '.csv');
+        self::assertEquals(['id', 'name',], $csvFile->getHeader());
+    }
+
+    public function bomProvider()
+    {
+        return [
+            ['utf32BigEndianBom'],
+            ['utf32LittleEndianBom'],
+            ['utf16BigEndianBom'],
+            ['utf16LittleEndianBom'],
+            ['utf8Bom'],
+        ];
+    }
+
     public function testParseMacLineEndsInField()
     {
         $csvFile = new CsvReader(__DIR__ . '/data/test-input.lineBreaks.csv', ",", '"', '\\');

tests/UTF8BOMHelperTest.php

@@ -0,0 +1,34 @@
+<?php
+
+namespace Keboola\Csv\Tests;
+
+use Keboola\Csv\CsvReader;
+use Keboola\Csv\UTF8BOMHelper;
+use PHPUnit\Framework\TestCase;
+
+class UTF8BOMHelperTest extends TestCase
+{
+    /**
+     * @dataProvider bomProvider
+     * @param string $bomFile
+     */
+    public function testDetectAndRemoveBOM($bomFile)
+    {
+        $file = __DIR__ . '/data/bom/' . $bomFile . '.csv';
+        $reader = new CsvReader($file);
+        $firstLine = $reader->current();
+        $this->assertNotSame(['id', 'name'], $firstLine);
+        $this->assertSame(['id', 'name'], UTF8BOMHelper::detectAndRemoveBOM($firstLine));
+    }
+
+    public function bomProvider()
+    {
+        return [
+            ['utf32BigEndianBom'],
+            ['utf32LittleEndianBom'],
+            ['utf16BigEndianBom'],
+            ['utf16LittleEndianBom'],
+            ['utf8Bom'],
+        ];
+    }
+}

tests/data/bom/utf16BigEndianBom.csv

@@ -0,0 +1 @@
+��"id","name"

tests/data/bom/utf16LittleEndianBom.csv

@@ -0,0 +1 @@
+��"id","name"

tests/data/bom/utf32BigEndianBom.csv

@@ -0,0 +1 @@
+"id","name"