From e81b4f88bc225202f87e92ab7efa3825a065fce8 Mon Sep 17 00:00:00 2001 From: "Andrei.Ovcharenko" Date: Sun, 18 Oct 2026 08:39:45 +0300 Subject: [PATCH] fix: normalize a leading UTF-8 BOM before CSV conversion --- src/MarkdownTableCore.cpp | 2 +- test-fixtures/markdown-table-core-golden.json | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/MarkdownTableCore.cpp b/src/MarkdownTableCore.cpp index 63cca6c..2c4867d 100644 --- a/src/MarkdownTableCore.cpp +++ b/src/MarkdownTableCore.cpp @@ -2824,7 +2824,7 @@ EditResult convertDelimitedToTable(const std::string &text) { EditResult result; // Tabs at the edges delimit empty TSV cells and must reach the parser. - const std::string &value = text; + const std::string value = text.compare(0, 3, "\xEF\xBB\xBF") == 0 ? text.substr(3) : text; const char delimiter = detectDelimiter(value); if (!hasDelimitedStructure(value, delimiter)) { diff --git a/test-fixtures/markdown-table-core-golden.json b/test-fixtures/markdown-table-core-golden.json index 284c7fe..7b8d08b 100644 --- a/test-fixtures/markdown-table-core-golden.json +++ b/test-fixtures/markdown-table-core-golden.json @@ -1,6 +1,41 @@ { "schemaVersion": 2, "conversion": [ + { + "name": "leading bom quoted csv", + "input": "\uFEFF\"Name\",\"City\"\n\"Bob\",\"Paris\"", + "lines": ["| Name | City |", "| ---- | ----- |", "| Bob | Paris |"] + }, + { + "name": "leading bom quoted tsv crlf", + "input": "\uFEFF\"Name\"\t\"City\"\r\n\"Bob\"\t\"Paris\"", + "lines": ["| Name | City |", "| ---- | ----- |", "| Bob | Paris |"] + }, + { + "name": "leading bom multiline quoted csv header", + "input": "\uFEFF\"First\nName\",City\nBob,Paris", + "lines": ["| First Name | City |", "| ---------- | ----- |", "| Bob | Paris |"] + }, + { + "name": "leading bom empty tsv edge cells", + "input": "\uFEFF\tName\t\n\tBob\t", + "lines": ["| | Name | |", "| --- | ---- | --- |", "| | Bob | |"] + }, + { + "name": "leading bom preserves internal bom", + "input": "\uFEFFName,City\nBo\uFEFFb,Paris", + "lines": ["| Name | City |", "| ---- | ----- |", "| Bo\uFEFFb | Paris |"] + }, + { + "name": "leading bom quoted csv tab is data", + "input": "\uFEFF\"Name\tAlias\",City\nBob,Paris", + "lines": ["| Name\tAlias | City |", "| ---------- | ----- |", "| Bob | Paris |"] + }, + { + "name": "leading bom escaped pipe contract", + "input": "\uFEFFName,Note\nBob,\"a|b\\|c\"", + "lines": ["| Name | Note |", "| ---- | ------- |", "| Bob | a\\|b\\|c |"] + }, { "name": "quoted tabs in later records preserve established tsv", "input": "Title, notes\nA\tfoo\nB\tbar\n\t\"x\ty\"",