diff --git a/README.md b/README.md index 3b6ba1ed..1a6369ed 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,8 @@ datacompy compare \ It exits `0` when the datasets match, `1` when they differ, and `2` on error. CSV, Parquet, and JSON inputs are supported on the pandas, polars, and Spark -backends, and Snowflake tables can be compared in place. +backends, including tab separated CSV via a `.tsv` extension, and Snowflake +tables can be compared in place. ## Programmatic Report Access diff --git a/datacompy/cli/backends.py b/datacompy/cli/backends.py index fe61e05d..ebb03c2f 100644 --- a/datacompy/cli/backends.py +++ b/datacompy/cli/backends.py @@ -46,22 +46,30 @@ from datacompy.cli.errors import BadArgsError, LoadError, MissingExtraError from datacompy.cli.parser import OPTIONS -#: File extension to canonical format name. +#: File extension to canonical format name and field delimiter. #: -#: ``.tsv`` is deliberately absent. Mapping it to ``csv`` would pick the right -#: reader but not the right delimiter, which stays comma unless -#: ``--csv-delimiter`` says otherwise, so a ``.tsv`` file would be recognised -#: and then misparsed. Until the delimiter is inferred per file, a tab -#: separated file is read with an explicit ``--input-format csv``. -_EXTENSION_FORMATS = { - ".csv": "csv", - ".parquet": "parquet", - ".pq": "parquet", - ".json": "json", - ".jsonl": "json", - ".ndjson": "json", +#: Format and delimiter live in one table on purpose. Mapping an extension to +#: ``csv`` without deciding its delimiter picks the right reader but not the +#: right separator, so the file is recognised and then misparsed into a single +#: column. Keeping the two in separate tables makes that mistake possible again +#: every time an extension is added; here the second element cannot be +#: forgotten. It is ``None`` for formats that have no delimiter. +_EXTENSIONS: dict[str, tuple[str, str | None]] = { + ".csv": ("csv", ","), + ".tsv": ("csv", "\t"), + ".parquet": ("parquet", None), + ".pq": ("parquet", None), + ".json": ("json", None), + ".jsonl": ("json", None), + ".ndjson": ("json", None), } +#: Delimiter for CSV input whose extension does not imply one. +_DEFAULT_DELIMITER = "," + +#: Delimiters worth naming when an input parses into a single column. +_PLAUSIBLE_DELIMITERS = (",", "\t", ";", "|") + _NDJSON_EXTENSIONS = frozenset({".jsonl", ".ndjson"}) #: A two or three part dotted Snowflake identifier, e.g. ``DB.SCHEMA.TABLE``. @@ -92,7 +100,7 @@ def infer_format(ref: str, override: str | None) -> str: return override extension = Path(ref).suffix.lower() try: - return _EXTENSION_FORMATS[extension] + return _EXTENSIONS[extension][0] except KeyError: raise BadArgsError( f"cannot infer the format of {ref!r} from its extension " @@ -100,6 +108,88 @@ def infer_format(ref: str, override: str | None) -> str: ) from None +def infer_delimiter(ref: str, override: str | None) -> str: + """Return the field delimiter for *ref*. + + Parameters + ---------- + ref : str + File path or URI. + override : str, optional + Explicit ``--csv-delimiter`` value. Returned as is when provided, and + applied to both inputs. This is also how a comma is forced for a comma + delimited file that happens to be named ``.tsv``, because + ``--input-format`` selects the reader and says nothing about the + delimiter. + + Returns + ------- + str + The delimiter implied by the extension, or a comma for anything else. + """ + if override is not None: + return override + _, delimiter = _EXTENSIONS.get(Path(ref).suffix.lower(), ("", None)) + return delimiter or _DEFAULT_DELIMITER + + +def suspect_delimiter( + ref: str, namespace: argparse.Namespace, frame: Any +) -> str | None: + """Return a warning when *frame* looks like it was read with the wrong delimiter. + + A CSV input read with the wrong delimiter collapses every row into one + column whose name is the entire header line. Nothing fails at that point: + the symptom surfaces later as a missing join column, or not at all under + ``--on-index``, where two mangled strings are compared instead. This is the + last place where the reference, the resolved format and the resolved + delimiter are all still in hand, so the guess is made here. + + Parameters + ---------- + ref : str + The ``--left`` or ``--right`` reference *frame* was read from. + namespace : argparse.Namespace + Parsed arguments, for the format and delimiter overrides. + frame : Any + The loaded DataFrame. Only ``columns`` is read, which every file + backend exposes without touching the data. + + Returns + ------- + str or None + A warning message, or ``None`` when the parse looks fine. Only a single + column CSV whose one column name contains a delimiter other than the + one used is reported, so a genuine single column file and a plain typo + in ``--on`` stay quiet. + """ + try: + if infer_format(ref, namespace.input_format) != "csv": + return None + except BadArgsError: + # Not a file this module recognises, for example a Snowflake table + # reference. Nothing to say about its delimiter. + return None + + # A pandas Index has no usable truth value, so test for None explicitly. + frame_columns = getattr(frame, "columns", None) + if frame_columns is None: + return None + columns = list(frame_columns) + if len(columns) != 1: + return None + + name = str(columns[0]) + used = infer_delimiter(ref, namespace.csv_delimiter) + if not any(char != used and char in name for char in _PLAUSIBLE_DELIMITERS): + return None + return ( + f"{ref} parsed into a single column named {name!r}. It was read with " + f"{used!r}, which may be the wrong delimiter. Pass --csv-delimiter to " + "override the extension based default." + ) + + def _is_ndjson(ref: str) -> bool: """Return ``True`` when *ref* looks like newline delimited JSON.""" return Path(ref).suffix.lower() in _NDJSON_EXTENSIONS @@ -203,7 +293,9 @@ def load( fmt = infer_format(ref, namespace.input_format) try: if fmt == "csv": - return pd.read_csv(ref, sep=namespace.csv_delimiter) + return pd.read_csv( + ref, sep=infer_delimiter(ref, namespace.csv_delimiter) + ) if fmt == "parquet": return pd.read_parquet(ref) return pd.read_json(ref, lines=_is_ndjson(ref)) @@ -227,7 +319,9 @@ def load( fmt = infer_format(ref, namespace.input_format) try: if fmt == "csv": - return pl.read_csv(ref, separator=namespace.csv_delimiter) + return pl.read_csv( + ref, separator=infer_delimiter(ref, namespace.csv_delimiter) + ) if fmt == "parquet": return pl.read_parquet(ref) if _is_ndjson(ref): @@ -296,7 +390,7 @@ def load(self, session: Any, ref: str, namespace: argparse.Namespace) -> Any: ref, header=True, inferSchema=True, - sep=namespace.csv_delimiter, + sep=infer_delimiter(ref, namespace.csv_delimiter), ) if fmt == "parquet": return session.read.parquet(ref) diff --git a/datacompy/cli/compare.py b/datacompy/cli/compare.py index 2fd73375..5d2c8a4c 100644 --- a/datacompy/cli/compare.py +++ b/datacompy/cli/compare.py @@ -18,9 +18,9 @@ import argparse from contextlib import ExitStack -from datacompy.cli.backends import BACKENDS +from datacompy.cli.backends import BACKENDS, suspect_delimiter from datacompy.cli.errors import BadArgsError -from datacompy.cli.output import emit +from datacompy.cli.output import emit, print_warning from datacompy.cli.parser import OPTIONS, OPTIONS_BY_FLAG, fill_defaults from datacompy.report import ReportData @@ -52,6 +52,15 @@ def run_compare(namespace: argparse.Namespace) -> int: session = backend.open_session(namespace, stack) left = backend.load(session, namespace.left, namespace) right = backend.load(session, namespace.right, namespace) + + # Warn before building, so a wrong delimiter is named ahead of the + # missing join column it causes, and is still reported under + # --on-index, where nothing fails at all. + for ref, frame in ((namespace.left, left), (namespace.right, right)): + suspect = suspect_delimiter(ref, namespace, frame) + if suspect is not None: + print_warning(suspect) + try: comparison = backend.build(namespace, session, left, right) except ValueError as exc: diff --git a/datacompy/cli/output.py b/datacompy/cli/output.py index 4de904de..4fb07ca6 100644 --- a/datacompy/cli/output.py +++ b/datacompy/cli/output.py @@ -103,3 +103,12 @@ def emit( def print_error(message: str) -> None: """Write *message* to stderr with a ``datacompy:`` prefix.""" print(f"datacompy: {message}", file=sys.stderr) + + +def print_warning(message: str) -> None: + """Write *message* to stderr with a ``datacompy: warning:`` prefix. + + Warnings are diagnostics rather than output, so they ignore ``--quiet``, + which suppresses the report itself. + """ + print(f"datacompy: warning: {message}", file=sys.stderr) diff --git a/datacompy/cli/parser.py b/datacompy/cli/parser.py index bce17b66..3729d969 100644 --- a/datacompy/cli/parser.py +++ b/datacompy/cli/parser.py @@ -291,12 +291,14 @@ def _resolve_dataset_name( Opt( flags=("--csv-delimiter",), help=( - "Field delimiter for CSV input (default: comma). " - r"Use '\t' for tab separated files." + "Field delimiter for CSV input. Inferred from each file extension, " + "a tab for .tsv and a comma otherwise. This flag overrides " + r"inference for both inputs: use '\t' for a tab separated file " + "with an unusual extension, or ',' to force a comma for a comma " + "separated file named .tsv." ), group=GROUP_INPUT, backends=FILE_BACKENDS, - default=",", options={"type": single_char, "metavar": "CHAR"}, ), Opt( diff --git a/docs/source/cli.rst b/docs/source/cli.rst index 406a7823..65229107 100644 --- a/docs/source/cli.rst +++ b/docs/source/cli.rst @@ -55,8 +55,8 @@ Inputs ------ ``--left`` and ``--right`` accept local paths and cloud URIs. CSV, Parquet, and -JSON are supported, including newline delimited JSON via a ``.jsonl`` or -``.ndjson`` extension. +JSON are supported, including tab separated CSV via a ``.tsv`` extension and +newline delimited JSON via a ``.jsonl`` or ``.ndjson`` extension. The format is inferred per file from its extension, so mixed inputs work without any extra flags: @@ -65,22 +65,32 @@ any extra flags: datacompy compare --left snapshot.csv --right snapshot.parquet --on id -The extensions recognised are ``.csv``, ``.parquet``, ``.json``, ``.jsonl``, -and ``.ndjson``. Use ``--input-format`` when the extension is missing or -unusual, and ``--csv-delimiter`` for anything other than a comma: +The extensions recognised are ``.csv``, ``.tsv``, ``.parquet``, ``.pq``, +``.json``, ``.jsonl``, and ``.ndjson``. The delimiter is inferred the same way, +a tab for ``.tsv`` and a comma for everything else, so a tab separated file +compares against a comma separated one without any extra flags. + +Use ``--input-format`` when the extension is missing or unusual. It selects the +reader only and says nothing about the delimiter, so pair it with +``--csv-delimiter``, which overrides inference for both inputs: .. code-block:: bash datacompy compare --left extract.dat --right extract2.dat --on id \ --input-format csv --csv-delimiter '\t' -.. note:: +``--csv-delimiter`` is also the way to correct a misleading extension. A comma +separated file named ``.tsv`` would otherwise be read with tabs, so force the +comma explicitly: + +.. code-block:: bash + + datacompy compare --left export.tsv --right export2.tsv --on id \ + --csv-delimiter ',' - ``--csv-delimiter`` applies to both datasets, and ``.tsv`` is not inferred - as a format for that reason. Read tab separated files with an explicit - ``--input-format csv --csv-delimiter '\t'``, which requires both sides to - use the same delimiter. Comparing a comma separated file against a tab - separated one is not currently supported. +A file read with the wrong delimiter collapses into a single column, which +surfaces as a missing join column. The CLI warns on stderr when it sees that, +naming the file and the delimiter it used. Cloud URIs such as ``s3://``, ``gs://``, and ``abfs://`` are handed straight to the underlying reader, so they work once the matching filesystem library diff --git a/tests/cli/test_backends.py b/tests/cli/test_backends.py new file mode 100644 index 00000000..3e208a76 --- /dev/null +++ b/tests/cli/test_backends.py @@ -0,0 +1,178 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for delimiter inference and misparse detection. + +``tests/cli/test_compare.py`` drives both helpers through ``main()`` on real +files, which is the right level for the paths a user actually walks. The cases +here are the ones an end to end run cannot reach or does not pin down: a +Snowflake table reference, a frame that exposes no columns, a quoted header, +and which delimiter the warning names. +""" + +import argparse + +import pandas as pd +import polars as pl +import pytest +from datacompy.cli.backends import infer_delimiter, suspect_delimiter + +TAB = "\t" + + +def _namespace( + *, input_format: str | None = None, csv_delimiter: str | None = None +) -> argparse.Namespace: + """Return the only two attributes ``suspect_delimiter`` reads.""" + return argparse.Namespace(input_format=input_format, csv_delimiter=csv_delimiter) + + +@pytest.fixture(params=["pandas", "polars"]) +def frame_of(request): + """Return a builder for a frame with the given column names. + + Parametrised over both in memory backends because they disagree on the type + of ``columns``: pandas returns an ``Index``, whose truth value raises rather + than answering, and polars returns a plain list. ``suspect_delimiter`` has + to accept both without testing either for truthiness. + """ + + def _build(*names: str): + data = {name: ["x", "y"] for name in names} + return pd.DataFrame(data) if request.param == "pandas" else pl.DataFrame(data) + + return _build + + +# --------------------------------------------------------------------------- +# infer_delimiter +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("ref", "expected"), + [ + ("data.csv", ","), + ("data.tsv", TAB), + ("s3://bucket/nightly/data.tsv", TAB), + ("extract.dat", ","), + ("extract", ","), + ], +) +def test_the_delimiter_comes_from_the_extension(ref, expected): + assert infer_delimiter(ref, None) == expected + + +def test_extension_matching_is_case_insensitive(): + """Nothing downstream lowercases the path, so this helper has to.""" + assert infer_delimiter("DATA.TSV", None) == TAB + + +@pytest.mark.parametrize("ref", ["data.json", "data.parquet", "data.ndjson"]) +def test_a_non_csv_extension_read_as_csv_gets_a_comma(ref): + """``--input-format csv`` can point the CSV reader at any extension. + + Those rows carry ``None`` in the extension table because their format has + no delimiter, which must resolve to the default rather than reaching the + reader as ``None``. + """ + assert infer_delimiter(ref, None) == "," + + +@pytest.mark.parametrize("ref", ["data.csv", "data.tsv", "data.parquet", "extract"]) +def test_an_explicit_delimiter_wins_over_every_extension(ref): + assert infer_delimiter(ref, ";") == ";" + + +# --------------------------------------------------------------------------- +# suspect_delimiter +# --------------------------------------------------------------------------- + + +def test_a_snowflake_table_reference_is_not_diagnosed(frame_of): + """The warning loop runs for every backend, including Snowflake. + + ``PROD.ANALYTICS.SALES`` has an extension as far as ``Path`` is concerned, + and it is not one this module knows, so format inference raises. That has + to stay contained here: a diagnostic helper must not turn a working + Snowflake comparison into an argument error. + """ + frame = frame_of("id,name,amount") + assert suspect_delimiter("PROD.ANALYTICS.SALES", _namespace(), frame) is None + + +def test_a_non_csv_input_is_not_diagnosed(frame_of): + """A one column Parquet file is a fact about the file, not a parse failure.""" + frame = frame_of("id,name,amount") + assert suspect_delimiter("data.parquet", _namespace(), frame) is None + + +def test_a_frame_without_columns_is_not_diagnosed(): + """The helper is best effort, so an unfamiliar object is skipped, not read.""" + + class Opaque: + pass + + assert suspect_delimiter("data.csv", _namespace(), Opaque()) is None + + +def test_a_quoted_header_is_not_reported_as_a_misparse(frame_of): + """A single column genuinely named ``a,b`` survives a comma delimited read. + + The comma in that name is the delimiter the file was read with, so it + cannot be evidence that the wrong one was chosen. Without that check the + only correctly parsed file in the world with a comma in a column name gets + warned about on every run. + """ + frame = frame_of("a,b") + assert suspect_delimiter("data.csv", _namespace(), frame) is None + + +def test_a_genuine_single_column_frame_is_not_reported(frame_of): + assert suspect_delimiter("data.csv", _namespace(), frame_of("id")) is None + + +def test_a_multi_column_frame_is_not_reported(frame_of): + frame = frame_of("id", "name", "amount") + assert suspect_delimiter("data.csv", _namespace(), frame) is None + + +def test_the_warning_names_the_delimiter_the_file_was_read_with(frame_of): + """A comma delimited file named ``.tsv`` is read with a tab and collapses.""" + message = suspect_delimiter("data.tsv", _namespace(), frame_of("id,name,amount")) + assert message is not None + assert "data.tsv" in message + assert repr(TAB) in message + assert "--csv-delimiter" in message + + +def test_the_warning_names_an_overridden_delimiter(frame_of): + """The delimiter reported is the one actually used, not the inferred one.""" + message = suspect_delimiter( + "data.csv", _namespace(csv_delimiter=";"), frame_of("id,name,amount") + ) + assert message is not None + assert repr(";") in message + + +def test_only_the_misparsed_input_is_reported(frame_of): + """Inference is per file, so one bad input does not implicate the other.""" + namespace = _namespace() + good = suspect_delimiter("left.csv", namespace, frame_of("id", "name", "amount")) + bad = suspect_delimiter("right.tsv", namespace, frame_of("id,name,amount")) + + assert good is None + assert bad is not None + assert "right.tsv" in bad diff --git a/tests/cli/test_compare.py b/tests/cli/test_compare.py index 0719ea23..0a80e9f7 100644 --- a/tests/cli/test_compare.py +++ b/tests/cli/test_compare.py @@ -586,40 +586,198 @@ def test_custom_csv_delimiter(tmp_path, left_frame, backend, capsys): ) -def test_tsv_extension_is_not_inferred(cli, tmp_path, left_frame, backend, capsys): - """``.tsv`` is not in the extension table, so it asks for an explicit format. - - Inferring it as CSV would pick the right reader and the wrong delimiter, - since ``--csv-delimiter`` applies to both sides at once and defaults to a - comma. Failing with a message that names the flag beats parsing the file - into a single mangled column. - """ +def test_tsv_extension_is_inferred(tmp_path, left_frame, backend, capsys): left = tmp_path / "left.tsv" right = tmp_path / "right.tsv" left_frame.to_csv(left, index=False, sep="\t") left_frame.to_csv(right, index=False, sep="\t") assert ( - cli("--left", str(left), "--right", str(right), "--backend", backend) == ERROR + main( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + backend, + ] + ) + == MATCH ) - assert "--input-format" in capsys.readouterr().err - # Forcing both the format and the delimiter still works. + +def test_mixed_csv_and_tsv_delimiters_are_inferred_per_file( + tmp_path, left_frame, backend, capsys +): + left = tmp_path / "left.csv" + right = tmp_path / "right.tsv" + left_frame.to_csv(left, index=False) + left_frame.to_csv(right, index=False, sep="\t") + assert ( - cli( + main( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + backend, + ] + ) + == MATCH + ) + + +def test_tab_extension_is_not_inferred(tmp_path, left_frame, backend, capsys): + """``.tab`` is deliberately unmapped: some tools mean "tabular" by it.""" + left = tmp_path / "left.tab" + right = tmp_path / "right.tab" + left_frame.to_csv(left, index=False, sep="\t") + left_frame.to_csv(right, index=False, sep="\t") + + assert ( + main( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + backend, + ] + ) + == ERROR + ) + assert "cannot infer the format" in capsys.readouterr().err + + +def test_explicit_delimiter_overrides_the_extension( + tmp_path, left_frame, backend, capsys +): + """The escape hatch for a comma separated file that carries a ``.tsv`` name. + + Without this the two return statements in ``infer_delimiter`` could be + reordered and every other delimiter test would still pass. + """ + left = tmp_path / "left.tsv" + right = tmp_path / "right.tsv" + left_frame.to_csv(left, index=False) + left_frame.to_csv(right, index=False) + + argv = [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + backend, + ] + + assert main([*argv, "--csv-delimiter", ","]) == MATCH + # The extension is wrong about this file, so inference alone cannot read it. + assert main(argv) == ERROR + + +def test_wrong_delimiter_warns_before_the_join_column_error( + tmp_path, left_frame, backend, capsys +): + left = tmp_path / "left.tsv" + right = tmp_path / "right.tsv" + left_frame.to_csv(left, index=False) + left_frame.to_csv(right, index=False) + + assert ( + main( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + backend, + ] + ) + == ERROR + ) + + stderr = capsys.readouterr().err + assert "parsed into a single column" in stderr + assert "--csv-delimiter" in stderr + assert str(left) in stderr + assert "must have all columns from join_columns" in stderr + + +def test_wrong_delimiter_warns_on_the_on_index_path(tmp_path, left_frame, capsys): + """``--on-index`` raises nothing, so the warning is the only signal.""" + left = tmp_path / "left.tsv" + right = tmp_path / "right.tsv" + left_frame.to_csv(left, index=False) + left_frame.to_csv(right, index=False) + + main( + [ + "compare", "--left", str(left), "--right", str(right), - "--input-format", - "csv", - "--csv-delimiter", - r"\t", + "--on-index", "--backend", - backend, + "pandas", + ] + ) + + assert "parsed into a single column" in capsys.readouterr().err + + +def test_a_correctly_parsed_input_is_not_warned_about(cli, backend, capsys): + assert cli("--backend", backend) == MISMATCH + assert "warning" not in capsys.readouterr().err + + +def test_a_genuine_single_column_file_is_not_warned_about( + tmp_path, left_frame, backend, capsys +): + """One column and no delimiter in its name is an ordinary file, not a misparse.""" + left = tmp_path / "left.csv" + right = tmp_path / "right.csv" + left_frame[["id"]].to_csv(left, index=False) + left_frame[["id"]].to_csv(right, index=False) + + assert ( + main( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + backend, + ] ) == MATCH ) + assert "warning" not in capsys.readouterr().err # --------------------------------------------------------------------------- diff --git a/tests/cli/test_parser.py b/tests/cli/test_parser.py index b3ecf74e..5abd606f 100644 --- a/tests/cli/test_parser.py +++ b/tests/cli/test_parser.py @@ -19,7 +19,7 @@ import inspect import pytest -from datacompy.cli.backends import BACKENDS, compare_kwargs +from datacompy.cli.backends import _EXTENSIONS, BACKENDS, compare_kwargs from datacompy.cli.errors import BadArgsError, MissingExtraError from datacompy.cli.parser import ( ALL_BACKENDS, @@ -260,6 +260,35 @@ def test_repeating_a_bare_tolerance_is_rejected(): OPTIONS_BY_FLAG["--rel-tol"].resolved(namespace) +# --------------------------------------------------------------------------- +# Input format and delimiter tables +# --------------------------------------------------------------------------- + + +def test_every_csv_extension_has_a_delimiter_and_no_other_format_does(): + """The invariant that keeps a recognised file from being misparsed. + + An extension mapped to ``csv`` with no delimiter behind it reaches the right + reader and is then split on the wrong character, which is why format and + delimiter share one table. + """ + for extension, (fmt, delimiter) in _EXTENSIONS.items(): + if fmt == "csv": + assert delimiter, f"{extension} maps to csv with no delimiter" + else: + assert delimiter is None, ( + f"{extension} is not csv but carries {delimiter!r}" + ) + + +def test_csv_delimiter_help_describes_inference_not_a_comma_default(): + """The help is the only place the escape hatch is documented.""" + help_text = OPTIONS_BY_FLAG["--csv-delimiter"].help + assert "default: comma" not in help_text + assert "Inferred from each file extension" in help_text + assert "','" in help_text + + # --------------------------------------------------------------------------- # Other type callables # --------------------------------------------------------------------------- diff --git a/tests/cli/test_spark.py b/tests/cli/test_spark.py index f1e140b9..9566ba84 100644 --- a/tests/cli/test_spark.py +++ b/tests/cli/test_spark.py @@ -98,6 +98,29 @@ def test_spark_matches_identical_files(left_csv, tmp_path, left_frame, capsys): assert exit_code == MATCH +def test_spark_infers_mixed_csv_and_tsv_delimiters(tmp_path, left_frame, capsys): + csv_path = tmp_path / "left.csv" + tsv_path = tmp_path / "right.tsv" + left_frame.to_csv(csv_path, index=False) + left_frame.to_csv(tsv_path, index=False, sep="\t") + + exit_code = main( + [ + "compare", + "--left", + str(csv_path), + "--right", + str(tsv_path), + "--on", + "id", + "--backend", + "spark", + ] + ) + + assert exit_code == MATCH + + def test_spark_session_is_stopped_even_when_loading_fails( no_borrowed_session, tmp_path, capsys ):