fix(cli): infer delimiters for TSV inputs

README.md

@@ -75,7 +75,8 @@ datacompy compare \
 
 It exits `0` when the datasets match, `1` when they differ, and `2` on error.
 CSV, Parquet, and JSON inputs are supported on the pandas, polars, and Spark
-backends, and Snowflake tables can be compared in place.
+backends, including tab separated CSV via a `.tsv` extension, and Snowflake
+tables can be compared in place.
 
 ## Programmatic Report Access
 

datacompy/cli/backends.py

@@ -46,22 +46,30 @@
 from datacompy.cli.errors import BadArgsError, LoadError, MissingExtraError
 from datacompy.cli.parser import OPTIONS
 
-#: File extension to canonical format name.
+#: File extension to canonical format name and field delimiter.
 #:
-#: ``.tsv`` is deliberately absent. Mapping it to ``csv`` would pick the right
-#: reader but not the right delimiter, which stays comma unless
-#: ``--csv-delimiter`` says otherwise, so a ``.tsv`` file would be recognised
-#: and then misparsed. Until the delimiter is inferred per file, a tab
-#: separated file is read with an explicit ``--input-format csv``.
-_EXTENSION_FORMATS = {
-    ".csv": "csv",
-    ".parquet": "parquet",
-    ".pq": "parquet",
-    ".json": "json",
-    ".jsonl": "json",
-    ".ndjson": "json",
+#: Format and delimiter live in one table on purpose. Mapping an extension to
+#: ``csv`` without deciding its delimiter picks the right reader but not the
+#: right separator, so the file is recognised and then misparsed into a single
+#: column. Keeping the two in separate tables makes that mistake possible again
+#: every time an extension is added; here the second element cannot be
+#: forgotten. It is ``None`` for formats that have no delimiter.
+_EXTENSIONS: dict[str, tuple[str, str | None]] = {
+    ".csv": ("csv", ","),
+    ".tsv": ("csv", "\t"),
+    ".parquet": ("parquet", None),
+    ".pq": ("parquet", None),
+    ".json": ("json", None),
+    ".jsonl": ("json", None),
+    ".ndjson": ("json", None),
 }
 
+#: Delimiter for CSV input whose extension does not imply one.
+_DEFAULT_DELIMITER = ","
+
+#: Delimiters worth naming when an input parses into a single column.
+_PLAUSIBLE_DELIMITERS = (",", "\t", ";", "|")
+
 _NDJSON_EXTENSIONS = frozenset({".jsonl", ".ndjson"})
 
 #: A two or three part dotted Snowflake identifier, e.g. ``DB.SCHEMA.TABLE``.
@@ -92,14 +100,96 @@ def infer_format(ref: str, override: str | None) -> str:
         return override
     extension = Path(ref).suffix.lower()
     try:
-        return _EXTENSION_FORMATS[extension]
+        return _EXTENSIONS[extension][0]
     except KeyError:
         raise BadArgsError(
             f"cannot infer the format of {ref!r} from its extension "
             f"{extension or '(none)'!r}. Pass --input-format csv|parquet|json."
         ) from None
 
 
+def infer_delimiter(ref: str, override: str | None) -> str:
+    """Return the field delimiter for *ref*.
+
+    Parameters
+    ----------
+    ref : str
+        File path or URI.
+    override : str, optional
+        Explicit ``--csv-delimiter`` value. Returned as is when provided, and
+        applied to both inputs. This is also how a comma is forced for a comma
+        delimited file that happens to be named ``.tsv``, because
+        ``--input-format`` selects the reader and says nothing about the
+        delimiter.
+
+    Returns
+    -------
+    str
+        The delimiter implied by the extension, or a comma for anything else.
+    """
+    if override is not None:
+        return override
+    _, delimiter = _EXTENSIONS.get(Path(ref).suffix.lower(), ("", None))
+    return delimiter or _DEFAULT_DELIMITER
+
+
+def suspect_delimiter(
+    ref: str, namespace: argparse.Namespace, frame: Any
+) -> str | None:
+    """Return a warning when *frame* looks like it was read with the wrong delimiter.
+
+    A CSV input read with the wrong delimiter collapses every row into one
+    column whose name is the entire header line. Nothing fails at that point:
+    the symptom surfaces later as a missing join column, or not at all under
+    ``--on-index``, where two mangled strings are compared instead. This is the
+    last place where the reference, the resolved format and the resolved
+    delimiter are all still in hand, so the guess is made here.
+
+    Parameters
+    ----------
+    ref : str
+        The ``--left`` or ``--right`` reference *frame* was read from.
+    namespace : argparse.Namespace
+        Parsed arguments, for the format and delimiter overrides.
+    frame : Any
+        The loaded DataFrame. Only ``columns`` is read, which every file
+        backend exposes without touching the data.
+
+    Returns
+    -------
+    str or None
+        A warning message, or ``None`` when the parse looks fine. Only a single
+        column CSV whose one column name contains a delimiter other than the
+        one used is reported, so a genuine single column file and a plain typo
+        in ``--on`` stay quiet.
+    """
+    try:
+        if infer_format(ref, namespace.input_format) != "csv":
+            return None
+    except BadArgsError:
+        # Not a file this module recognises, for example a Snowflake table
+        # reference. Nothing to say about its delimiter.
+        return None
+
+    # A pandas Index has no usable truth value, so test for None explicitly.
+    frame_columns = getattr(frame, "columns", None)
+    if frame_columns is None:
+        return None
+    columns = list(frame_columns)
+    if len(columns) != 1:
+        return None
+
+    name = str(columns[0])
+    used = infer_delimiter(ref, namespace.csv_delimiter)
+    if not any(char != used and char in name for char in _PLAUSIBLE_DELIMITERS):
+        return None
+    return (
+        f"{ref} parsed into a single column named {name!r}. It was read with "
+        f"{used!r}, which may be the wrong delimiter. Pass --csv-delimiter to "
+        "override the extension based default."
+    )
+
+
 def _is_ndjson(ref: str) -> bool:
     """Return ``True`` when *ref* looks like newline delimited JSON."""
     return Path(ref).suffix.lower() in _NDJSON_EXTENSIONS
@@ -203,7 +293,9 @@ def load(
         fmt = infer_format(ref, namespace.input_format)
         try:
             if fmt == "csv":
-                return pd.read_csv(ref, sep=namespace.csv_delimiter)
+                return pd.read_csv(
+                    ref, sep=infer_delimiter(ref, namespace.csv_delimiter)
+                )
             if fmt == "parquet":
                 return pd.read_parquet(ref)
             return pd.read_json(ref, lines=_is_ndjson(ref))
@@ -227,7 +319,9 @@ def load(
         fmt = infer_format(ref, namespace.input_format)
         try:
             if fmt == "csv":
-                return pl.read_csv(ref, separator=namespace.csv_delimiter)
+                return pl.read_csv(
+                    ref, separator=infer_delimiter(ref, namespace.csv_delimiter)
+                )
             if fmt == "parquet":
                 return pl.read_parquet(ref)
             if _is_ndjson(ref):
@@ -296,7 +390,7 @@ def load(self, session: Any, ref: str, namespace: argparse.Namespace) -> Any:
                     ref,
                     header=True,
                     inferSchema=True,
-                    sep=namespace.csv_delimiter,
+                    sep=infer_delimiter(ref, namespace.csv_delimiter),
                 )
             if fmt == "parquet":
                 return session.read.parquet(ref)

datacompy/cli/compare.py

@@ -18,9 +18,9 @@
 import argparse
 from contextlib import ExitStack
 
-from datacompy.cli.backends import BACKENDS
+from datacompy.cli.backends import BACKENDS, suspect_delimiter
 from datacompy.cli.errors import BadArgsError
-from datacompy.cli.output import emit
+from datacompy.cli.output import emit, print_warning
 from datacompy.cli.parser import OPTIONS, OPTIONS_BY_FLAG, fill_defaults
 from datacompy.report import ReportData
 
@@ -52,6 +52,15 @@ def run_compare(namespace: argparse.Namespace) -> int:
         session = backend.open_session(namespace, stack)
         left = backend.load(session, namespace.left, namespace)
         right = backend.load(session, namespace.right, namespace)
+
+        # Warn before building, so a wrong delimiter is named ahead of the
+        # missing join column it causes, and is still reported under
+        # --on-index, where nothing fails at all.
+        for ref, frame in ((namespace.left, left), (namespace.right, right)):
+            suspect = suspect_delimiter(ref, namespace, frame)
+            if suspect is not None:
+                print_warning(suspect)
+
         try:
             comparison = backend.build(namespace, session, left, right)
         except ValueError as exc:

datacompy/cli/output.py

@@ -103,3 +103,12 @@ def emit(
 def print_error(message: str) -> None:
     """Write *message* to stderr with a ``datacompy:`` prefix."""
     print(f"datacompy: {message}", file=sys.stderr)
+
+
+def print_warning(message: str) -> None:
+    """Write *message* to stderr with a ``datacompy: warning:`` prefix.
+
+    Warnings are diagnostics rather than output, so they ignore ``--quiet``,
+    which suppresses the report itself.
+    """
+    print(f"datacompy: warning: {message}", file=sys.stderr)

datacompy/cli/parser.py

@@ -291,12 +291,14 @@ def _resolve_dataset_name(
     Opt(
         flags=("--csv-delimiter",),
         help=(
-            "Field delimiter for CSV input (default: comma). "
-            r"Use '\t' for tab separated files."
+            "Field delimiter for CSV input. Inferred from each file extension, "
+            "a tab for .tsv and a comma otherwise. This flag overrides "
+            r"inference for both inputs: use '\t' for a tab separated file "
+            "with an unusual extension, or ',' to force a comma for a comma "
+            "separated file named .tsv."
         ),
         group=GROUP_INPUT,
         backends=FILE_BACKENDS,
-        default=",",
         options={"type": single_char, "metavar": "CHAR"},
     ),
     Opt(

docs/source/cli.rst

@@ -55,8 +55,8 @@ Inputs
 ------
 
 ``--left`` and ``--right`` accept local paths and cloud URIs. CSV, Parquet, and
-JSON are supported, including newline delimited JSON via a ``.jsonl`` or
-``.ndjson`` extension.
+JSON are supported, including tab separated CSV via a ``.tsv`` extension and
+newline delimited JSON via a ``.jsonl`` or ``.ndjson`` extension.
 
 The format is inferred per file from its extension, so mixed inputs work without
 any extra flags:
@@ -65,22 +65,32 @@ any extra flags:
 
     datacompy compare --left snapshot.csv --right snapshot.parquet --on id
 
-The extensions recognised are ``.csv``, ``.parquet``, ``.json``, ``.jsonl``,
-and ``.ndjson``. Use ``--input-format`` when the extension is missing or
-unusual, and ``--csv-delimiter`` for anything other than a comma:
+The extensions recognised are ``.csv``, ``.tsv``, ``.parquet``, ``.pq``,
+``.json``, ``.jsonl``, and ``.ndjson``. The delimiter is inferred the same way,
+a tab for ``.tsv`` and a comma for everything else, so a tab separated file
+compares against a comma separated one without any extra flags.
+
+Use ``--input-format`` when the extension is missing or unusual. It selects the
+reader only and says nothing about the delimiter, so pair it with
+``--csv-delimiter``, which overrides inference for both inputs:
 
 .. code-block:: bash
 
     datacompy compare --left extract.dat --right extract2.dat --on id \
         --input-format csv --csv-delimiter '\t'
 
-.. note::
+``--csv-delimiter`` is also the way to correct a misleading extension. A comma
+separated file named ``.tsv`` would otherwise be read with tabs, so force the
+comma explicitly:
+
+.. code-block:: bash
+
+    datacompy compare --left export.tsv --right export2.tsv --on id \
+        --csv-delimiter ','
 
-    ``--csv-delimiter`` applies to both datasets, and ``.tsv`` is not inferred
-    as a format for that reason. Read tab separated files with an explicit
-    ``--input-format csv --csv-delimiter '\t'``, which requires both sides to
-    use the same delimiter. Comparing a comma separated file against a tab
-    separated one is not currently supported.
+A file read with the wrong delimiter collapses into a single column, which
+surfaces as a missing join column. The CLI warns on stderr when it sees that,
+naming the file and the delimiter it used.
 
 Cloud URIs such as ``s3://``, ``gs://``, and ``abfs://`` are handed straight to
 the underlying reader, so they work once the matching filesystem library