From 8e60bff241e63562ba2feecf433e6a5a439afd34 Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 10:40:38 -0500 Subject: [PATCH 01/15] refactor: add py3.12 and py3.13 to tests --- .github/workflows/test-package.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index 47416a93..e88e74c8 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/tox.ini b/tox.ini index caf70d43..9b729481 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py39, py310, py311, pypi-description, manifest, precom +envlist = py310, py311, py312, py313, pypi-description, manifest, precom [testenv] From 9cab92c006e97475561adf89773b428294c8af6a Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 10:56:20 -0500 Subject: [PATCH 02/15] fix: commit setting --- .github/workflows/test-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index e88e74c8..752cb232 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -17,9 +17,9 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v5.6.0 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From 23b158faed9707dccf5e65171cff266ed0bbd9f8 Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 11:33:32 -0500 Subject: [PATCH 03/15] fix: mypy issues --- dataprofiler/_version.py | 3 +- dataprofiler/data_readers/csv_data.py | 6 ++-- dataprofiler/data_readers/data_utils.py | 10 +++--- .../data_readers/filepath_or_buffer.py | 2 +- dataprofiler/labelers/base_data_labeler.py | 2 +- dataprofiler/labelers/column_name_model.py | 2 +- dataprofiler/labelers/data_labelers.py | 2 +- dataprofiler/labelers/data_processing.py | 8 ++--- dataprofiler/labelers/regex_model.py | 7 ++-- .../profilers/float_column_profile.py | 3 +- dataprofiler/profilers/int_column_profile.py | 4 ++- .../profilers/numerical_column_stats.py | 3 +- dataprofiler/profilers/profile_builder.py | 36 ++++++++++--------- dataprofiler/profilers/profiler_options.py | 2 +- dataprofiler/profilers/text_column_profile.py | 3 +- dataprofiler/reports/graphs.py | 12 +++---- dataprofiler/tests/plugins/test_plugins.py | 8 ++--- .../test_numeric_stats_mixin_profile.py | 2 +- 18 files changed, 64 insertions(+), 51 deletions(-) diff --git a/dataprofiler/_version.py b/dataprofiler/_version.py index 66995988..e26962f4 100644 --- a/dataprofiler/_version.py +++ b/dataprofiler/_version.py @@ -14,6 +14,7 @@ import re import subprocess import sys +from typing import Any, Callable def get_keywords(): @@ -52,7 +53,7 @@ class NotThisMethod(Exception): LONG_VERSION_PY = {} # type: ignore -HANDLERS = {} +HANDLERS: dict[str, dict[str, Callable[..., Any]]] = {} def register_vcs_handler(vcs, method): # decorator diff --git a/dataprofiler/data_readers/csv_data.py b/dataprofiler/data_readers/csv_data.py index cb1a2e2d..5f026258 100644 --- a/dataprofiler/data_readers/csv_data.py +++ b/dataprofiler/data_readers/csv_data.py @@ -674,8 +674,10 @@ def is_match(cls, file_path: str, options: Optional[Dict] = None) -> bool: empty_line_count = 0 delimiter_count = dict() - delimiter_regex = data_utils.get_delimiter_regex(delimiter, quotechar) - space_regex = data_utils.get_delimiter_regex(" ", quotechar) + delimiter_regex = data_utils.get_delimiter_regex( + delimiter or ",", quotechar or '"' + ) + space_regex = data_utils.get_delimiter_regex(" ", quotechar or '"') # Count the possible delimiters for line in data_as_str.split("\n")[header:]: diff --git a/dataprofiler/data_readers/data_utils.py b/dataprofiler/data_readers/data_utils.py index 833d650e..81856efc 100644 --- a/dataprofiler/data_readers/data_utils.py +++ b/dataprofiler/data_readers/data_utils.py @@ -277,7 +277,7 @@ def read_json( return lines -def reservoir(file: TextIOWrapper, sample_nrows: int) -> list: +def reservoir(file: TextIOWrapper | StringIO, sample_nrows: int) -> list: """ Implement the mathematical logic of Reservoir sampling. @@ -344,7 +344,9 @@ def reservoir(file: TextIOWrapper, sample_nrows: int) -> list: return values -def rsample(file_path: TextIOWrapper, sample_nrows: int, args: dict) -> StringIO: +def rsample( + file_path: TextIOWrapper | StringIO, sample_nrows: int, args: dict +) -> StringIO: """ Implement Reservoir Sampling to sample n rows out of a total of M rows. @@ -369,7 +371,7 @@ def rsample(file_path: TextIOWrapper, sample_nrows: int, args: dict) -> StringIO def read_csv_df( - file_path: Union[str, BytesIO, TextIOWrapper], + file_path: Union[str, StringIO, BytesIO, TextIOWrapper], delimiter: Optional[str], header: Optional[int], sample_nrows: Optional[int] = None, @@ -425,7 +427,7 @@ def read_csv_df( file_path = open(file_path, encoding=encoding) is_file_open = True - file_data = file_path + file_data: TextIOWrapper | StringIO = file_path if sample_nrows: file_data = rsample(file_path, sample_nrows, args) fo = pd.read_csv(file_data, **args) diff --git a/dataprofiler/data_readers/filepath_or_buffer.py b/dataprofiler/data_readers/filepath_or_buffer.py index 201f690e..860eb037 100644 --- a/dataprofiler/data_readers/filepath_or_buffer.py +++ b/dataprofiler/data_readers/filepath_or_buffer.py @@ -93,7 +93,7 @@ def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None: TextIOWrapper, self._filepath_or_buffer ) # guaranteed by self._is_wrapped wrapper = self._filepath_or_buffer - self._filepath_or_buffer = wrapper.buffer + self._filepath_or_buffer = cast(BytesIO, wrapper.buffer) wrapper.detach() if isinstance(self._filepath_or_buffer, (StringIO, BytesIO)): diff --git a/dataprofiler/labelers/base_data_labeler.py b/dataprofiler/labelers/base_data_labeler.py index f9a4a0ab..4547b592 100644 --- a/dataprofiler/labelers/base_data_labeler.py +++ b/dataprofiler/labelers/base_data_labeler.py @@ -17,7 +17,7 @@ from . import data_processing, utils from .base_model import BaseModel -default_labeler_dir = utils.find_resources_dir("labelers") +default_labeler_dir = str(utils.find_resources_dir("labelers")) class BaseDataLabeler: diff --git a/dataprofiler/labelers/column_name_model.py b/dataprofiler/labelers/column_name_model.py index 1732983c..5801d5f0 100644 --- a/dataprofiler/labelers/column_name_model.py +++ b/dataprofiler/labelers/column_name_model.py @@ -176,7 +176,7 @@ def _reconstruct_model(self) -> None: pass def _need_to_reconstruct_model(self) -> bool: - pass + return False def reset_weights(self) -> None: """Reset weights function.""" diff --git a/dataprofiler/labelers/data_labelers.py b/dataprofiler/labelers/data_labelers.py index 961b45e6..94283d5b 100644 --- a/dataprofiler/labelers/data_labelers.py +++ b/dataprofiler/labelers/data_labelers.py @@ -12,7 +12,7 @@ from .base_model import BaseModel from .data_processing import BaseDataPostprocessor, BaseDataPreprocessor -default_labeler_dir = utils.find_resources_dir("labelers") +default_labeler_dir = str(utils.find_resources_dir("labelers")) def train_structured_labeler( diff --git a/dataprofiler/labelers/data_processing.py b/dataprofiler/labelers/data_processing.py index 70c980c3..8c7b9901 100644 --- a/dataprofiler/labelers/data_processing.py +++ b/dataprofiler/labelers/data_processing.py @@ -19,7 +19,7 @@ from . import utils -default_labeler_dir = utils.find_resources_dir("labelers") +default_labeler_dir = str(utils.find_resources_dir("labelers")) Processor = TypeVar("Processor", bound="BaseDataProcessor") @@ -491,7 +491,7 @@ def gen_none() -> Generator[None, None, None]: for start, end, label in label_set: label_index = label_mapping[label] label_buffer[start:end] = label_index - label_buffer = label_buffer.tolist() + label_buffer_list = label_buffer.tolist() # loop until the buffer is empty and placed as requested buffer_ind = 0 @@ -543,7 +543,7 @@ def gen_none() -> Generator[None, None, None]: batch_data["samples"].append(sample_buffer[buffer_ind:separate_ind]) if label_set is not None: batch_data["labels"].append( - label_buffer[buffer_ind:separate_ind] + label_buffer_list[buffer_ind:separate_ind] + [label_mapping[pad_label]] * pad_len ) @@ -610,7 +610,7 @@ def gen_none() -> Generator[None, None, None]: if label_set is not None: flattened_entities.extend( - label_buffer[buffer_ind:separate_ind] + label_buffer_list[buffer_ind:separate_ind] ) buffer_ind = separate_ind diff --git a/dataprofiler/labelers/regex_model.py b/dataprofiler/labelers/regex_model.py index dd74da71..80458acf 100644 --- a/dataprofiler/labelers/regex_model.py +++ b/dataprofiler/labelers/regex_model.py @@ -7,6 +7,7 @@ import os import re import sys +from typing import Any import numpy as np @@ -167,7 +168,7 @@ def _reconstruct_model(self) -> None: pass def _need_to_reconstruct_model(self) -> bool: - pass + return False def reset_weights(self) -> None: """Reset weights.""" @@ -225,7 +226,7 @@ def predict( # Construct array initial regex predictions where background is # predicted. - predictions = [np.empty((0,))] * 100 + predictions: list[Any] = [np.empty((0,))] * 100 i = 0 for i, input_string in enumerate(data): @@ -247,7 +248,7 @@ def predict( if verbose: sys.stdout.flush() sys.stdout.write(f"\rData Samples Processed: {i + 1:d} ") - predictions[i] = pred + predictions[i] = pred.tolist() if verbose: logger.info(f"\rData Samples Processed: {i + 1:d} ") diff --git a/dataprofiler/profilers/float_column_profile.py b/dataprofiler/profilers/float_column_profile.py index 3d6ede32..b75425b7 100644 --- a/dataprofiler/profilers/float_column_profile.py +++ b/dataprofiler/profilers/float_column_profile.py @@ -4,6 +4,7 @@ import copy import re +from typing import cast import numpy as np import pandas as pd @@ -201,7 +202,7 @@ def profile(self) -> dict: :return: """ - profile = NumericStatsMixin.profile(self) + profile = cast(dict, NumericStatsMixin.profile.__get__(self, type(self))) profile.update( dict( precision=dict( diff --git a/dataprofiler/profilers/int_column_profile.py b/dataprofiler/profilers/int_column_profile.py index ae4ed575..d0194d8f 100644 --- a/dataprofiler/profilers/int_column_profile.py +++ b/dataprofiler/profilers/int_column_profile.py @@ -2,6 +2,8 @@ from __future__ import annotations +from typing import cast + import numpy as np import pandas as pd @@ -99,7 +101,7 @@ def profile(self) -> dict: :return: """ - return NumericStatsMixin.profile(self) + return cast(dict, NumericStatsMixin.profile.__get__(self, type(self))) @property def data_type_ratio(self) -> float | None: diff --git a/dataprofiler/profilers/numerical_column_stats.py b/dataprofiler/profilers/numerical_column_stats.py index 9ec2190a..4b7ef21e 100644 --- a/dataprofiler/profilers/numerical_column_stats.py +++ b/dataprofiler/profilers/numerical_column_stats.py @@ -366,6 +366,7 @@ def _add_helper( other1._median_abs_dev_is_enabled and other2._median_abs_dev_is_enabled ) + @property def profile(self) -> dict: """ Return profile of the column. @@ -408,7 +409,7 @@ def report(self, remove_disabled_flag: bool = False) -> dict: :rtype: Profile """ calcs_dict_keys = self._NumericStatsMixin__calculations.keys() - profile = self.profile() + profile = self.profile if remove_disabled_flag: profile_keys = list(profile.keys()) diff --git a/dataprofiler/profilers/profile_builder.py b/dataprofiler/profilers/profile_builder.py index 7d904b6a..6c87bea5 100644 --- a/dataprofiler/profilers/profile_builder.py +++ b/dataprofiler/profilers/profile_builder.py @@ -12,7 +12,7 @@ from collections import OrderedDict, defaultdict from datetime import datetime from multiprocessing.pool import Pool -from typing import Any, Generator, List, Optional, TypeVar, cast +from typing import Any, Generator, List, Optional, Sized, TypeVar, cast import networkx as nx import numpy as np @@ -738,7 +738,7 @@ def __init__( self._min_sample_size: int = 5000 # assign data labeler - data_labeler_options = self.options.data_labeler + data_labeler_options = cast(Any, self.options).data_labeler if ( data_labeler_options.is_enabled and data_labeler_options.data_labeler_object is None @@ -751,11 +751,13 @@ def __init__( dirpath=data_labeler_options.data_labeler_dirpath, load_options=None, ) - self.options.set({"data_labeler.data_labeler_object": data_labeler}) + cast(Any, self.options).set( + {"data_labeler.data_labeler_object": data_labeler} + ) except Exception as e: profiler_utils.warn_on_profile("data_labeler", e) - self.options.set({"data_labeler.is_enabled": False}) + cast(Any, self.options).set({"data_labeler.is_enabled": False}) def _add_error_checks(self, other: BaseProfiler) -> None: """ @@ -976,7 +978,7 @@ def update_profile( f"one of the following: {self._allowed_external_data_types}" ) - if not len(data): + if not len(cast(Sized, data)): warnings.warn( "The passed dataset was empty, hence no data was " "profiled." ) @@ -2674,18 +2676,18 @@ def _update_null_replication_metrics(self, clean_samples: dict) -> None: mean_not_null = sum_not_null / true_count # Convert numpy arrays to lists (serializable) - sum_null = sum_null.tolist() - sum_not_null = sum_not_null.tolist() + sum_null_list = sum_null.tolist() + sum_not_null_list = sum_not_null.tolist() - mean_null = mean_null.tolist() - mean_not_null = mean_not_null.tolist() + mean_null_list = mean_null.tolist() + mean_not_null_list = mean_not_null.tolist() # Array index serves as class label # 0 indicates not null, 1 indicates null self._null_replication_metrics[col_id] = { "class_prior": [prior_not_null, prior_null], - "class_sum": [sum_not_null, sum_null], - "class_mean": [mean_not_null, mean_null], + "class_sum": [sum_not_null_list, sum_null_list], + "class_mean": [mean_not_null_list, mean_null_list], } def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: @@ -2776,18 +2778,18 @@ def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: mean_not_null = sum_not_null / true_count # Convert numpy arrays to lists (serializable) - sum_null = sum_null.tolist() - sum_not_null = sum_not_null.tolist() + sum_null_list = sum_null.tolist() + sum_not_null_list = sum_not_null.tolist() - mean_null = mean_null.tolist() - mean_not_null = mean_not_null.tolist() + mean_null_list = mean_null.tolist() + mean_not_null_list = mean_not_null.tolist() merged_properties[col_id] = { # Array index serves as class label # 0 indicates not null, 1 indicates null "class_prior": [prior_not_null, prior_null], - "class_sum": [sum_not_null, sum_null], - "class_mean": [mean_not_null, mean_null], + "class_sum": [sum_not_null_list, sum_null_list], + "class_mean": [mean_not_null_list, mean_null_list], } return merged_properties diff --git a/dataprofiler/profilers/profiler_options.py b/dataprofiler/profilers/profiler_options.py index 038acf80..6fb3eeaf 100644 --- a/dataprofiler/profilers/profiler_options.py +++ b/dataprofiler/profilers/profiler_options.py @@ -9,7 +9,7 @@ from typing import Any, Generic, TypeVar, cast from ..labelers.base_data_labeler import BaseDataLabeler -from ..plugins.__init__ import get_plugins +from ..plugins import get_plugins from . import profiler_utils from .json_decoder import load_option diff --git a/dataprofiler/profilers/text_column_profile.py b/dataprofiler/profilers/text_column_profile.py index eb79643f..48333759 100644 --- a/dataprofiler/profilers/text_column_profile.py +++ b/dataprofiler/profilers/text_column_profile.py @@ -3,6 +3,7 @@ from __future__ import annotations import itertools +from typing import cast import numpy as np import pandas as pd @@ -91,7 +92,7 @@ def profile(self) -> dict: :return: """ - profile = NumericStatsMixin.profile(self) + profile = cast(dict, NumericStatsMixin.profile.__get__(self, type(self))) # remove num_zeros and num_negative updated from numeric profile profile.pop("num_zeros") profile.pop("num_negatives") diff --git a/dataprofiler/reports/graphs.py b/dataprofiler/reports/graphs.py index 4e630a1c..4d2672df 100644 --- a/dataprofiler/reports/graphs.py +++ b/dataprofiler/reports/graphs.py @@ -33,7 +33,7 @@ def plot_histograms( profiler: StructuredProfiler, column_names: list[int | str] | None = None, column_inds: list[int] | None = None, -) -> matplotlib.pyplot.figure: +) -> matplotlib.figure.Figure | None: """ Plot the histograms of column names that are int or float columns. @@ -109,7 +109,7 @@ def is_index_graphable_column(ind_to_graph: int) -> bool: "No plots were constructed" " because no int or float columns were found in columns" ) - return + return None # get proper tile format for graph n = len(inds_to_graph) @@ -192,7 +192,7 @@ def plot_missing_values_matrix( profiler: StructuredProfiler, ax: matplotlib.axes.Axes | None = None, title: str | None = None, -) -> matplotlib.pyplot.figure: +) -> matplotlib.figure.Figure | None: """ Generate matrix of bar graphs for missing value locations in cols of struct dataset. @@ -216,7 +216,7 @@ def plot_col_missing_values( col_profiler_list: list[StructuredColProfiler], ax: matplotlib.axes.Axes | None = None, title: str | None = None, -) -> matplotlib.pyplot.figure: +) -> matplotlib.figure.Figure | None: """ Generate bar graph of missing value locations within a col. @@ -244,7 +244,7 @@ def plot_col_missing_values( warnings.warn( "There was no data in the profiles to plot missing " "column values." ) - return + return None # bar width settings and height settings for each null value # width = 1, height = 1 would be no gaps @@ -263,7 +263,7 @@ def plot_col_missing_values( ax = fig.add_subplot(111) is_own_fig = True # in case user passed their own axes - fig = ax.figure + fig = cast(matplotlib.figure.Figure, ax.figure) # loop through eac column plotting their null values for col_id, col_profiler in enumerate(col_profiler_list): diff --git a/dataprofiler/tests/plugins/test_plugins.py b/dataprofiler/tests/plugins/test_plugins.py index 9368975d..91677e5c 100644 --- a/dataprofiler/tests/plugins/test_plugins.py +++ b/dataprofiler/tests/plugins/test_plugins.py @@ -2,7 +2,7 @@ from collections import defaultdict from unittest import mock -from dataprofiler.plugins.__init__ import get_plugins, load_plugins +from dataprofiler.plugins import get_plugins, load_plugins from dataprofiler.plugins.decorators import plugin_decorator, plugins_dict @@ -24,9 +24,9 @@ def test_plugin(): test_get_dict = get_plugins("test") self.assertDictEqual({"mock_test": test_plugin}, test_get_dict) - @mock.patch("dataprofiler.plugins.__init__.importlib.util") - @mock.patch("dataprofiler.plugins.__init__.os.path.isdir") - @mock.patch("dataprofiler.plugins.__init__.os.listdir") + @mock.patch("dataprofiler.plugins.importlib.util") + @mock.patch("dataprofiler.plugins.os.path.isdir") + @mock.patch("dataprofiler.plugins.os.listdir") def test_load_plugin(self, mock_listdir, mock_isdir, mock_importlib_util): mock_listdir.side_effect = lambda folder_dir: ( ["__pycache__", "py"] diff --git a/dataprofiler/tests/profilers/test_numeric_stats_mixin_profile.py b/dataprofiler/tests/profilers/test_numeric_stats_mixin_profile.py index e112781a..5dd26244 100644 --- a/dataprofiler/tests/profilers/test_numeric_stats_mixin_profile.py +++ b/dataprofiler/tests/profilers/test_numeric_stats_mixin_profile.py @@ -759,7 +759,7 @@ def test_profile(self): # Validate that the times dictionary is empty self.assertEqual(defaultdict(float), num_profiler.times) - profile = num_profiler.profile() + profile = num_profiler.profile # pop out the histogram and quartiles to test separately from the # rest of the dict as we need comparison with some precision histogram = profile.pop("histogram") From ae1dc4b3d22311ce5f5bbdb25e3dacd095dc1368 Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 11:42:18 -0500 Subject: [PATCH 04/15] fix: typing --- dataprofiler/labelers/data_processing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dataprofiler/labelers/data_processing.py b/dataprofiler/labelers/data_processing.py index 8c7b9901..a9c6aca1 100644 --- a/dataprofiler/labelers/data_processing.py +++ b/dataprofiler/labelers/data_processing.py @@ -482,6 +482,7 @@ def gen_none() -> Generator[None, None, None]: sample_buffer = str(sample_buffer) # buffer is empty, add sample to the buffer. sample_len = len(sample_buffer) + label_buffer_list: list[int] = [] if label_set is not None: # Create an entity buffer for sample, assign the default entity From 91a4a24c86633eb4ef03a5945ca51436b95b3349 Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 11:51:40 -0500 Subject: [PATCH 05/15] refactor: update mypy --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cd3dc8be..49978a12 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,7 +40,7 @@ repos: # Mypy: Optional static type checking # https://github.com/pre-commit/mirrors-mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.982 + rev: v1.11.2 hooks: - id: mypy exclude: (^dataprofiler/tests/|^resources/|^examples|venv*/|versioneer.py|dataprofiler/_version.py|_docs/) From 8caaba884d28a7feb7f54707b151caab2f86ccaf Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 12:53:01 -0500 Subject: [PATCH 06/15] fix: issues with mypy --- .pre-commit-config.yaml | 4 +++- dataprofiler/data_readers/data_utils.py | 3 ++- dataprofiler/data_readers/parquet_data.py | 2 +- dataprofiler/labelers/classification_report_utils.py | 1 + dataprofiler/labelers/utils.py | 2 +- dataprofiler/profilers/numerical_column_stats.py | 2 +- setup.cfg | 3 +++ 7 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49978a12..7506c0aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,9 @@ repos: rev: v1.11.2 hooks: - id: mypy - exclude: (^dataprofiler/tests/|^resources/|^examples|venv*/|versioneer.py|dataprofiler/_version.py|_docs/) + # Let mypy own target selection and exclusions via setup.cfg. + args: [--config-file=setup.cfg, dataprofiler] + pass_filenames: false language_version: python3 additional_dependencies: # Keep up-to-date with the respective requirement files [ diff --git a/dataprofiler/data_readers/data_utils.py b/dataprofiler/data_readers/data_utils.py index 81856efc..baf26840 100644 --- a/dataprofiler/data_readers/data_utils.py +++ b/dataprofiler/data_readers/data_utils.py @@ -6,6 +6,7 @@ import random import re import urllib +import urllib.parse from collections import OrderedDict from io import BytesIO, StringIO, TextIOWrapper from itertools import islice @@ -676,7 +677,7 @@ def _decode_is_valid(encoding): # If no encoding is still found, default to utf-8 if not encoding: encoding = "utf-8" - return encoding.lower() + return str(encoding).lower() def detect_cell_type(cell: str) -> str: diff --git a/dataprofiler/data_readers/parquet_data.py b/dataprofiler/data_readers/parquet_data.py index b679431b..2e66d1a0 100644 --- a/dataprofiler/data_readers/parquet_data.py +++ b/dataprofiler/data_readers/parquet_data.py @@ -68,7 +68,7 @@ def __init__( self._load_data(data) @property - def file_encoding(self) -> None: + def file_encoding(self) -> Optional[str]: """Set file encoding to None since not detected for avro.""" return None diff --git a/dataprofiler/labelers/classification_report_utils.py b/dataprofiler/labelers/classification_report_utils.py index 840c236f..7ed8be0e 100644 --- a/dataprofiler/labelers/classification_report_utils.py +++ b/dataprofiler/labelers/classification_report_utils.py @@ -210,6 +210,7 @@ def precision_recall_fscore_support( support: np.ndarray | None = true_sum if average == "weighted": weights = true_sum + assert weights is not None if weights.sum() == 0: return np.array([0.0]), np.array([0.0]), np.array([0.0]), None elif average == "samples": diff --git a/dataprofiler/labelers/utils.py b/dataprofiler/labelers/utils.py index 99553869..b6e4a09c 100644 --- a/dataprofiler/labelers/utils.py +++ b/dataprofiler/labelers/utils.py @@ -66,7 +66,7 @@ def find_resources_dir(resource_path: str | Path | None = None) -> Traversable: """Return the path to the package resources.""" resource = importlib.resources.files("dataprofiler") / "resources" if resource_path: - resource /= resource_path + resource = resource / str(resource_path) if not (resource.is_file() or resource.is_dir()): raise FileNotFoundError(f"Resource not found: {resource_path}") diff --git a/dataprofiler/profilers/numerical_column_stats.py b/dataprofiler/profilers/numerical_column_stats.py index 4b7ef21e..2710152f 100644 --- a/dataprofiler/profilers/numerical_column_stats.py +++ b/dataprofiler/profilers/numerical_column_stats.py @@ -565,7 +565,7 @@ def median(self) -> float: :rtype: float """ if not self._has_histogram or not self._median_is_enabled: - return np.nan + return float(np.nan) return self._get_percentile([50])[0] @property diff --git a/setup.cfg b/setup.cfg index 8ad51bd8..181a009a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,6 +21,9 @@ use_parentheses=True line_length=88 [mypy] +files = dataprofiler +exclude = (^|.*/)dataprofiler/tests/ +implicit_optional = True warn_return_any = True warn_unused_configs = True ignore_missing_imports = True From 776607705c9de1176dab2fe629ae57b156bd3d48 Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 13:00:10 -0500 Subject: [PATCH 07/15] fix: cnn typing --- dataprofiler/labelers/character_level_cnn_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataprofiler/labelers/character_level_cnn_model.py b/dataprofiler/labelers/character_level_cnn_model.py index 7d78900a..e6fed328 100644 --- a/dataprofiler/labelers/character_level_cnn_model.py +++ b/dataprofiler/labelers/character_level_cnn_model.py @@ -911,8 +911,8 @@ def predict( ) # Pre-allocate space for predictions confidences: list | np.ndarray = [] - sentence_lengths = np.zeros((batch_size,), dtype=int) - predictions = np.zeros((batch_size, self._parameters["max_length"])) + sentence_lengths: np.ndarray = np.zeros((batch_size,), dtype=int) + predictions: np.ndarray = np.zeros((batch_size, self._parameters["max_length"])) if show_confidences: confidences = np.zeros( (batch_size, self._parameters["max_length"], self.num_labels) From 1cc50b5eada913348b708fe64fbc05a021ba5ff5 Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 13:34:27 -0500 Subject: [PATCH 08/15] fix: typing again --- dataprofiler/data_readers/base_data.py | 2 +- dataprofiler/data_readers/data_utils.py | 4 +- .../labelers/character_level_cnn_model.py | 6 +- .../labelers/classification_report_utils.py | 16 ++-- dataprofiler/labelers/column_name_model.py | 2 +- dataprofiler/labelers/data_processing.py | 75 +++++++++++++++---- dataprofiler/labelers/labeler_utils.py | 20 +++-- 7 files changed, 90 insertions(+), 35 deletions(-) diff --git a/dataprofiler/data_readers/base_data.py b/dataprofiler/data_readers/base_data.py index e6e85d3d..3dd32b21 100644 --- a/dataprofiler/data_readers/base_data.py +++ b/dataprofiler/data_readers/base_data.py @@ -155,7 +155,7 @@ def get_batch_generator( ) -> Generator[Union[pd.DataFrame, List], None, None]: """Get batch generator.""" data_length = len(self.data) - indices = np.random.permutation(data_length) + indices: np.ndarray = np.random.permutation(data_length) for i in range(0, data_length, batch_size): if isinstance(self.data, pd.DataFrame): yield self.data.iloc[indices[i : i + batch_size], :] diff --git a/dataprofiler/data_readers/data_utils.py b/dataprofiler/data_readers/data_utils.py index baf26840..7ac1fbe2 100644 --- a/dataprofiler/data_readers/data_utils.py +++ b/dataprofiler/data_readers/data_utils.py @@ -506,10 +506,10 @@ def sample_parquet( # sample n_rows = parquet_table.num_rows if n_rows > sample_nrows: - sample_index = np.array([False] * n_rows) + sample_index = cast(np.ndarray, np.array([False] * n_rows)) sample_index[random.sample(range(n_rows), sample_nrows)] = True else: - sample_index = np.array([True] * n_rows) + sample_index = cast(np.ndarray, np.array([True] * n_rows)) sample_df = parquet_table.filter(sample_index).to_pandas() # Convert all the unicode columns to utf-8 diff --git a/dataprofiler/labelers/character_level_cnn_model.py b/dataprofiler/labelers/character_level_cnn_model.py index e6fed328..bc30dd03 100644 --- a/dataprofiler/labelers/character_level_cnn_model.py +++ b/dataprofiler/labelers/character_level_cnn_model.py @@ -8,6 +8,7 @@ import sys import time from collections import defaultdict +from typing import cast import numpy as np import tensorflow as tf @@ -608,8 +609,9 @@ def _construct_model(self) -> None: _file_dir, "embeddings/glove-reduced-{}D.txt".format(self._parameters["dim_embed"]), ) - embedding_matrix = np.zeros( - (max_char_encoding_id + 2, self._parameters["dim_embed"]) + embedding_matrix = cast( + np.ndarray, + np.zeros((max_char_encoding_id + 2, self._parameters["dim_embed"])), ) embedding_dict = build_embd_dictionary(embed_file) diff --git a/dataprofiler/labelers/classification_report_utils.py b/dataprofiler/labelers/classification_report_utils.py index 7ed8be0e..59b64433 100644 --- a/dataprofiler/labelers/classification_report_utils.py +++ b/dataprofiler/labelers/classification_report_utils.py @@ -32,25 +32,27 @@ def convert_confusion_matrix_to_MCM(conf_matrix: list | np.ndarray) -> np.ndarra """ if not isinstance(conf_matrix, np.ndarray): conf_matrix = np.array(conf_matrix) + conf_matrix = cast(np.ndarray, conf_matrix) num_labels = len(conf_matrix) num_samples: int = int(np.sum(conf_matrix)) - MCM = np.zeros((num_labels, 2, 2), dtype=np.int64) + MCM = cast(np.ndarray, np.zeros((num_labels, 2, 2), dtype=np.int64)) # True Positives MCM[:, 1, 1] = np.sum(conf_matrix * np.eye(num_labels), axis=1) # False Negatives - MCM[:, 1, 0] = np.sum( - conf_matrix * (np.ones(num_labels) - np.eye(num_labels)), axis=1 + non_diagonal_mask = cast( + np.ndarray, np.ones((num_labels, num_labels)) - np.eye(num_labels) ) + MCM[:, 1, 0] = np.sum(conf_matrix * non_diagonal_mask, axis=1) # False Positives - MCM[:, 0, 1] = np.sum( - conf_matrix.T * (np.ones(num_labels) - np.eye(num_labels)), axis=1 - ) + MCM[:, 0, 1] = np.sum(conf_matrix.T * non_diagonal_mask, axis=1) # True Negatives - MCM[:, 0, 0] = num_samples - MCM[:, 1, 0] - MCM[:, 0, 1] - MCM[:, 1, 1] + MCM[:, 0, 0] = cast( + np.ndarray, num_samples - MCM[:, 1, 0] - MCM[:, 0, 1] - MCM[:, 1, 1] + ) return MCM diff --git a/dataprofiler/labelers/column_name_model.py b/dataprofiler/labelers/column_name_model.py index 5801d5f0..b0cd5fa7 100644 --- a/dataprofiler/labelers/column_name_model.py +++ b/dataprofiler/labelers/column_name_model.py @@ -191,7 +191,7 @@ def _model( scorer: Callable, include_label: bool = False, ) -> list: - scores = [] + scores: list[list[float | int]] = [] check_values_list = [dict["attribute"] for dict in check_values_dict] diff --git a/dataprofiler/labelers/data_processing.py b/dataprofiler/labelers/data_processing.py index a9c6aca1..bfd23fee 100644 --- a/dataprofiler/labelers/data_processing.py +++ b/dataprofiler/labelers/data_processing.py @@ -15,7 +15,6 @@ from typing import Any, Generator, Iterable, TypeVar, cast import numpy as np -import numpy.typing as npt from . import utils @@ -382,7 +381,16 @@ def _find_nearest_sentence_break_before_ind( sentence: str, start_ind: int, min_ind: int = 0, - separators: tuple[str, ...] = (" ", "\n", ",", "\t", "\r", "\x00", "\x01", ";"), + separators: tuple[str, ...] = ( + " ", + "\n", + ",", + "\t", + "\r", + "\x00", + "\x01", + ";", + ), ) -> int: """ Find nearest separator before the start_ind and return the index. @@ -486,13 +494,16 @@ def gen_none() -> Generator[None, None, None]: if label_set is not None: # Create an entity buffer for sample, assign the default entity - label_buffer = np.full(sample_len, label_mapping[default_label]) + label_buffer: np.ndarray = cast( + np.ndarray, + np.full(sample_len, label_mapping[default_label], dtype=int), + ) # Map the entity to the corresponding character for start, end, label in label_set: label_index = label_mapping[label] label_buffer[start:end] = label_index - label_buffer_list = label_buffer.tolist() + label_buffer_list = cast(list[int], label_buffer.tolist()) # loop until the buffer is empty and placed as requested buffer_ind = 0 @@ -537,7 +548,8 @@ def gen_none() -> Generator[None, None, None]: # pad the data until fits maximum length pad_len = max( - max_length - separate_ind + buffer_ind, max_length - sample_len + max_length - separate_ind + buffer_ind, + max_length - sample_len, ) # Only add the buffer up until maximum length @@ -897,7 +909,17 @@ def __init__( flatten_separator: str = " ", use_word_level_argmax: bool = False, output_format: str = "character_argmax", - separators: tuple[str, ...] = (" ", ",", ";", "'", '"', ":", "\n", "\t", "."), + separators: tuple[str, ...] = ( + " ", + ",", + ";", + "'", + '"', + ":", + "\n", + "\t", + ".", + ), word_level_min_percent: float = 0.75, ) -> None: """ @@ -1191,7 +1213,11 @@ def convert_to_NER_format( if begin_idx != -1: # Add last sample sample_output.append( - (begin_idx, curr_idx + 1, reverse_label_mapping[(int(curr_label))]) + ( + begin_idx, + curr_idx + 1, + reverse_label_mapping[(int(curr_label))], + ) ) # Add to total output list output_result.append(sample_output) @@ -1200,7 +1226,10 @@ def convert_to_NER_format( @staticmethod def match_sentence_lengths( - data: np.ndarray, results: dict, flatten_separator: str, inplace: bool = True + data: np.ndarray, + results: dict, + flatten_separator: str, + inplace: bool = True, ) -> dict: """ Convert results from model into same ragged data shapes as original data. @@ -1219,7 +1248,9 @@ def match_sentence_lengths( pred_buffer: np.ndarray = np.array([]) conf_buffer: np.ndarray = np.array([]) result_ind = 0 - buffer_add_inds: list[int] = np.cumsum(list(map(len, results["pred"]))).tolist() + buffer_add_inds = cast( + list[int], np.cumsum(list(map(len, results["pred"]))).tolist() + ) separator_len = len(flatten_separator) if not inplace: @@ -1409,7 +1440,7 @@ def get_parameters(self, param_list: list[str] | None = None) -> dict: return params def convert_to_unstructured_format( - self, data: np.ndarray, labels: list[str] | npt.NDArray[np.str_] | None + self, data: np.ndarray, labels: list[str] | np.ndarray | None ) -> tuple[str, list[tuple[int, int, str]] | None]: """ Convert data samples list to StructCharPreprocessor required input data format. @@ -1501,7 +1532,7 @@ def process( # with rework, can be tuned to be batches > size 1 for ind in range(len(data)): batch_data: np.ndarray = data[ind : ind + 1] - batch_labels: npt.NDArray[np.str_] | list[str] | None = ( + batch_labels: np.ndarray | list[str] | None = ( None if labels is None else labels[ind : ind + 1] ) ( @@ -1522,7 +1553,10 @@ def process( np_unstruct_labels = None return super().process( - np.array(unstructured_data), np_unstruct_labels, label_mapping, batch_size + np.array(unstructured_data), + np_unstruct_labels, + label_mapping, + batch_size, ) @@ -1668,7 +1702,10 @@ def help(cls) -> None: @staticmethod def match_sentence_lengths( - data: np.ndarray, results: dict, flatten_separator: str, inplace: bool = True + data: np.ndarray, + results: dict, + flatten_separator: str, + inplace: bool = True, ) -> dict: """ Convert results from model into same ragged data shapes as original data. @@ -1687,7 +1724,9 @@ def match_sentence_lengths( pred_buffer: np.ndarray = np.array([]) conf_buffer: np.ndarray = np.array([]) result_ind = 0 - buffer_add_inds: list[int] = np.cumsum(list(map(len, results["pred"]))).tolist() + buffer_add_inds = cast( + list[int], np.cumsum(list(map(len, results["pred"]))).tolist() + ) separator_len = len(flatten_separator) if not inplace: @@ -1775,9 +1814,13 @@ def convert_to_structured_analysis( ignore_value = label_mapping[pad_label] num_labels = max(label_mapping.values()) + 1 - labels_out = np.ones((len(results["pred"]),)) + labels_out: np.ndarray = cast( + np.ndarray, np.full((len(results["pred"]),), None, dtype=object) + ) if "conf" in results: - confs_out = np.zeros((len(results["pred"]), num_labels)) + confs_out: np.ndarray = cast( + np.ndarray, np.zeros((len(results["pred"]), num_labels)) + ) for i, label_samples in enumerate(zip(results["pred"], sentences)): column_labels, sample = label_samples diff --git a/dataprofiler/labelers/labeler_utils.py b/dataprofiler/labelers/labeler_utils.py index 3a1097ce..04b0abf0 100644 --- a/dataprofiler/labelers/labeler_utils.py +++ b/dataprofiler/labelers/labeler_utils.py @@ -120,16 +120,22 @@ def evaluate_accuracy( if x[1] not in omitted_labels ] - predicted_entities = [np.asarray(row) for row in predicted_entities_in_index] - true_entities = [np.asarray(row) for row in true_entities_in_index] + predicted_entities: list[np.ndarray] = [ + np.asarray(row) for row in predicted_entities_in_index + ] + true_entities: list[np.ndarray] = [ + np.asarray(row) for row in true_entities_in_index + ] max_len = len(predicted_entities[0]) - true_labels_padded = np.zeros((len(true_entities), max_len)) + true_labels_padded: np.ndarray = cast( + np.ndarray, np.zeros((len(true_entities), max_len)) + ) for i, true_labels_row in enumerate(true_entities): true_labels_padded[i][: len(true_labels_row)] = true_labels_row - true_labels_flatten = np.hstack(true_labels_padded) # type: ignore - predicted_labels_flatten = np.hstack(predicted_entities) + true_labels_flatten: np.ndarray = np.hstack(true_labels_padded) # type: ignore + predicted_labels_flatten: np.ndarray = np.hstack(predicted_entities) all_labels: list[str] = [] if entity_rev_dict: @@ -139,7 +145,9 @@ def evaluate_accuracy( # By definition a confusion matrix :math:`C` is such that :math:`C_{i, j}` # is equal to the number of observations known to be in group :math:`i` but # predicted to be in group :math:`j`. - conf_mat = np.zeros((num_labels, num_labels), dtype=np.int64) + conf_mat: np.ndarray = cast( + np.ndarray, np.zeros((num_labels, num_labels), dtype=np.int64) + ) batch_size = min(2**20, len(true_labels_flatten)) for batch_ind in range(len(true_labels_flatten) // batch_size + 1): true_label_batch = true_labels_flatten[ From 11454a3416f0803e5fa6c6cd35b72a9fb4569725 Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 13:47:06 -0500 Subject: [PATCH 09/15] fix: numpy typing --- dataprofiler/labelers/classification_report_utils.py | 2 +- dataprofiler/labelers/column_name_model.py | 5 +++-- dataprofiler/labelers/data_processing.py | 10 ++++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/dataprofiler/labelers/classification_report_utils.py b/dataprofiler/labelers/classification_report_utils.py index 59b64433..66ee2c65 100644 --- a/dataprofiler/labelers/classification_report_utils.py +++ b/dataprofiler/labelers/classification_report_utils.py @@ -42,7 +42,7 @@ def convert_confusion_matrix_to_MCM(conf_matrix: list | np.ndarray) -> np.ndarra # False Negatives non_diagonal_mask = cast( - np.ndarray, np.ones((num_labels, num_labels)) - np.eye(num_labels) + np.ndarray, np.logical_not(np.eye(num_labels, dtype=bool)).astype(np.int64) ) MCM[:, 1, 0] = np.sum(conf_matrix * non_diagonal_mask, axis=1) diff --git a/dataprofiler/labelers/column_name_model.py b/dataprofiler/labelers/column_name_model.py index b0cd5fa7..fd431c76 100644 --- a/dataprofiler/labelers/column_name_model.py +++ b/dataprofiler/labelers/column_name_model.py @@ -4,7 +4,7 @@ import json import os -from typing import Any, Callable +from typing import Any, Callable, cast import numpy as np @@ -199,7 +199,8 @@ def _model( list_of_column_names, check_values_list, processor=processor, scorer=scorer ) - for iter_value, ngram_match_results in enumerate(model_outputs): + for _, ngram_match_results in enumerate(model_outputs): + ngram_match_results = cast(np.ndarray, ngram_match_results) column_result = [np.max(ngram_match_results)] if include_label: index_max_result = ngram_match_results.argmax(axis=0) diff --git a/dataprofiler/labelers/data_processing.py b/dataprofiler/labelers/data_processing.py index bfd23fee..794e4526 100644 --- a/dataprofiler/labelers/data_processing.py +++ b/dataprofiler/labelers/data_processing.py @@ -1248,9 +1248,10 @@ def match_sentence_lengths( pred_buffer: np.ndarray = np.array([]) conf_buffer: np.ndarray = np.array([]) result_ind = 0 - buffer_add_inds = cast( - list[int], np.cumsum(list(map(len, results["pred"]))).tolist() + cumulative_lengths = cast( + np.ndarray, np.cumsum(list(map(len, results["pred"]))) ) + buffer_add_inds = cast(list[int], cumulative_lengths.tolist()) separator_len = len(flatten_separator) if not inplace: @@ -1724,9 +1725,10 @@ def match_sentence_lengths( pred_buffer: np.ndarray = np.array([]) conf_buffer: np.ndarray = np.array([]) result_ind = 0 - buffer_add_inds = cast( - list[int], np.cumsum(list(map(len, results["pred"]))).tolist() + cumulative_lengths = cast( + np.ndarray, np.cumsum(list(map(len, results["pred"]))) ) + buffer_add_inds = cast(list[int], cumulative_lengths.tolist()) separator_len = len(flatten_separator) if not inplace: From 20320cdd99c467c57a55071b25ba122670a6d2e8 Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 14:09:29 -0500 Subject: [PATCH 10/15] fix: typing more --- dataprofiler/data_readers/csv_data.py | 4 ++-- dataprofiler/profilers/numerical_column_stats.py | 13 +++++++------ dataprofiler/profilers/profiler_utils.py | 14 +++++++------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/dataprofiler/data_readers/csv_data.py b/dataprofiler/data_readers/csv_data.py index 5f026258..30795d80 100644 --- a/dataprofiler/data_readers/csv_data.py +++ b/dataprofiler/data_readers/csv_data.py @@ -730,9 +730,9 @@ def is_match(cls, file_path: str, options: Optional[Dict] = None) -> bool: if not count_percent.size: return False - max_count_index = count_percent.argmax() + max_count_index = int(count_percent.argmax()) max_count_value = list(delimiter_count.keys())[max_count_index] - max_count_percent = count_percent[max_count_index] + max_count_percent = float(count_percent[max_count_index]) # Inferred the file was a CSV if (max_count_value > 0 or delimiter is None) and ( diff --git a/dataprofiler/profilers/numerical_column_stats.py b/dataprofiler/profilers/numerical_column_stats.py index 2710152f..12dea86d 100644 --- a/dataprofiler/profilers/numerical_column_stats.py +++ b/dataprofiler/profilers/numerical_column_stats.py @@ -1108,11 +1108,12 @@ def _estimate_mode_from_histogram(self) -> list[float]: elif bin_counts[i] == cur_max and count < self._top_k_modes: highest_idxs.append(i) count += 1 - highest_idxs = np.array(highest_idxs) # type: ignore + highest_idx_array = cast(np.ndarray, np.array(highest_idxs)) - mode: npt.NDArray[np.float64] = ( - bin_edges[highest_idxs] + bin_edges[highest_idxs + 1] # type: ignore - ) / 2 + mode = cast( + np.ndarray, + (bin_edges[highest_idx_array] + bin_edges[highest_idx_array + 1]) / 2, + ) return cast(List[float], mode.tolist()) def _estimate_stats_from_histogram(self) -> np.float64: @@ -1135,9 +1136,9 @@ def _total_histogram_bin_variance( bin_edges = bin_edges.copy() bin_edges[-1] += 1e-3 - inds = np.digitize(input_array, bin_edges) + inds = cast(np.ndarray, np.digitize(input_array, bin_edges)) sum_var = 0 - non_zero_bins = np.where(bin_counts)[0] + 1 + non_zero_bins = cast(np.ndarray, np.where(bin_counts)[0] + 1) for i in non_zero_bins: elements_in_bin = input_array[inds == i] bin_var = elements_in_bin.var() diff --git a/dataprofiler/profilers/profiler_utils.py b/dataprofiler/profilers/profiler_utils.py index 7986cec0..8d903abe 100644 --- a/dataprofiler/profilers/profiler_utils.py +++ b/dataprofiler/profilers/profiler_utils.py @@ -44,7 +44,7 @@ def as_float_scalar( value: int | float | np.integer | np.floating | np.ndarray | list[float], ) -> float: """Convert a scalar-like value to a Python float.""" - array_value = np.asarray(value) + array_value = cast(np.ndarray, np.asarray(value)) if array_value.ndim == 0: return float(array_value) if array_value.size == 1: @@ -135,13 +135,13 @@ def shuffle_in_chunks( values = [-1] * true_chunk_size # Generate random list of indexes - lower_bound_list = np.array(range(j, j + true_chunk_size)) - random_list = rng.integers(lower_bound_list, data_length) + lower_bound_list = cast(np.ndarray, np.array(range(j, j + true_chunk_size))) + random_list = cast(np.ndarray, rng.integers(lower_bound_list, data_length)) # shuffle the indexes for count in range(true_chunk_size): # get a random index to swap and swap it with j - k = random_list[count] + k = int(random_list[count]) indices[j], indices[k] = indices[k], indices[j] # set the swapped value to the output @@ -610,11 +610,11 @@ def find_diff_of_matrices( :rtype: list(list(float)) """ if matrix1 is not None and matrix2 is not None: - mat1 = np.array(matrix1, dtype=np.float64) - mat2 = np.array(matrix2, dtype=np.float64) + mat1 = cast(np.ndarray, np.array(matrix1, dtype=np.float64)) + mat2 = cast(np.ndarray, np.array(matrix2, dtype=np.float64)) if np.shape(mat1) == np.shape(mat2): - diff: np.ndarray = mat1 - mat2 + diff: np.ndarray = cast(np.ndarray, mat1 - mat2) if ((diff == 0) | np.isnan(diff)).all(): return "unchanged" return diff From f1305c6d6387087c5986bea9026c4389172e38d4 Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 14:20:55 -0500 Subject: [PATCH 11/15] fix: typing more --- dataprofiler/data_readers/csv_data.py | 8 ++- dataprofiler/data_readers/data_utils.py | 4 +- dataprofiler/labelers/column_name_model.py | 5 +- .../profilers/data_labeler_column_profile.py | 21 +++++-- dataprofiler/profilers/graph_profiler.py | 17 ++++-- .../profilers/numerical_column_stats.py | 45 ++++++++------ dataprofiler/profilers/profile_builder.py | 58 ++++++++++--------- 7 files changed, 96 insertions(+), 62 deletions(-) diff --git a/dataprofiler/data_readers/csv_data.py b/dataprofiler/data_readers/csv_data.py index 30795d80..49d1c6a7 100644 --- a/dataprofiler/data_readers/csv_data.py +++ b/dataprofiler/data_readers/csv_data.py @@ -724,8 +724,12 @@ def is_match(cls, file_path: str, options: Optional[Dict] = None) -> bool: active_line_count - max_deviation_count ) / active_line_count - delimiter_count_values = np.array(list(delimiter_count.values())) - count_percent = delimiter_count_values / np.sum(delimiter_count_values) + delimiter_count_values = cast( + np.ndarray, np.array(list(delimiter_count.values())) + ) + count_percent = cast( + np.ndarray, delimiter_count_values / np.sum(delimiter_count_values) + ) if not count_percent.size: return False diff --git a/dataprofiler/data_readers/data_utils.py b/dataprofiler/data_readers/data_utils.py index 7ac1fbe2..99aae412 100644 --- a/dataprofiler/data_readers/data_utils.py +++ b/dataprofiler/data_readers/data_utils.py @@ -26,12 +26,12 @@ import boto3 import botocore -import dateutil import numpy as np import pandas as pd import pyarrow.parquet as pq import requests from chardet.universaldetector import UniversalDetector +from dateutil import parser as dateutil_parser # type: ignore[import-untyped] from typing_extensions import TypeGuard from .. import dp_logging, rng_utils @@ -694,7 +694,7 @@ def detect_cell_type(cell: str) -> str: try: # need to ingore type bc https://github.com/python/mypy/issues/8878 - if dateutil.parser.parse(cell, fuzzy=False): # type:ignore + if dateutil_parser.parse(cell, fuzzy=False): # type: ignore[attr-defined] cell_type = "date" except (ValueError, OverflowError, TypeError): pass diff --git a/dataprofiler/labelers/column_name_model.py b/dataprofiler/labelers/column_name_model.py index fd431c76..15d999db 100644 --- a/dataprofiler/labelers/column_name_model.py +++ b/dataprofiler/labelers/column_name_model.py @@ -198,9 +198,10 @@ def _model( model_outputs = rapidfuzz.process.cdist( list_of_column_names, check_values_list, processor=processor, scorer=scorer ) + model_outputs = cast(np.ndarray, model_outputs) - for _, ngram_match_results in enumerate(model_outputs): - ngram_match_results = cast(np.ndarray, ngram_match_results) + for i in range(len(model_outputs)): + ngram_match_results: np.ndarray = cast(np.ndarray, model_outputs[i]) column_result = [np.max(ngram_match_results)] if include_label: index_max_result = ngram_match_results.argmax(axis=0) diff --git a/dataprofiler/profilers/data_labeler_column_profile.py b/dataprofiler/profilers/data_labeler_column_profile.py index 3ce6257f..15375c0b 100644 --- a/dataprofiler/profilers/data_labeler_column_profile.py +++ b/dataprofiler/profilers/data_labeler_column_profile.py @@ -251,18 +251,27 @@ def data_label(self) -> str | None: return None ranks_items = self.rank_distribution.items() - ordered_top_k_rank = np.array( - sorted(ranks_items, key=operator.itemgetter(1), reverse=True) + ordered_top_k_rank = sorted( + ranks_items, key=operator.itemgetter(1), reverse=True )[: self._top_k_labels] - top_k_probabilities = np.fromiter( - map(operator.itemgetter(1), ordered_top_k_rank), dtype=float - ) / sum(self.rank_distribution.values()) + top_k_probabilities = cast( + np.ndarray, + np.fromiter(map(operator.itemgetter(1), ordered_top_k_rank), dtype=float) + / sum(self.rank_distribution.values()), + ) is_value_close = ( top_k_probabilities - top_k_probabilities[0] >= -self._min_prob_differential ) data_label = "|".join( - map(operator.itemgetter(0), ordered_top_k_rank[is_value_close]) + map( + operator.itemgetter(0), + [ + rank_item + for rank_item, keep_value in zip(ordered_top_k_rank, is_value_close) + if keep_value + ], + ) ) top_label = ordered_top_k_rank[0][0] if cast(Dict, self.label_representation)[top_label] < self._min_top_label_prob: diff --git a/dataprofiler/profilers/graph_profiler.py b/dataprofiler/profilers/graph_profiler.py index 345a0f2e..fd7e7453 100644 --- a/dataprofiler/profilers/graph_profiler.py +++ b/dataprofiler/profilers/graph_profiler.py @@ -456,10 +456,13 @@ def _get_categorical_distribution( for attribute in attributes: if attribute in categorical_attributes: data_as_list = self._attribute_data_as_list(graph, attribute) - hist, edges = np.histogram(data_as_list, bins="auto", density=False) + hist, edges = cast( + tuple[np.ndarray, np.ndarray], + np.histogram(data_as_list, bins="auto", density=False), + ) categorical_distributions[attribute] = { - "bin_counts": list(hist), - "bin_edges": list(edges), + "bin_counts": cast(list[float], hist.tolist()), + "bin_edges": cast(list[float], edges.tolist()), } else: categorical_distributions[attribute] = None @@ -493,9 +496,11 @@ def _get_categorical_and_continuous_attributes( @staticmethod def _find_all_attributes(graph: nx.Graph) -> list[str]: """Compute the number of attributes for each edge.""" - attribute_list = set( - np.array([list(graph.edges[n].keys()) for n in graph.edges()]).flatten() - ) + attribute_list = { + attribute + for edge in graph.edges() + for attribute in graph.edges[edge].keys() + } return list(attribute_list) def _attribute_data_as_list(self, graph: nx.Graph, attribute: str) -> list: diff --git a/dataprofiler/profilers/numerical_column_stats.py b/dataprofiler/profilers/numerical_column_stats.py index 12dea86d..1283f399 100644 --- a/dataprofiler/profilers/numerical_column_stats.py +++ b/dataprofiler/profilers/numerical_column_stats.py @@ -9,7 +9,6 @@ from typing import Any, Callable, Dict, List, TypeAlias, TypeVar, cast import numpy as np -import numpy.typing as npt import pandas as pd import scipy.stats @@ -1216,9 +1215,9 @@ def _select_method_for_histogram( self.histogram_methods[method]["current_loss"] = self._histogram_loss( current_diff_var[method_id], current_avg_diff_var, - current_total_var[method_id], + cast(float, current_total_var[method_id]), current_avg_total_var, - current_run_time[method_id], + cast(float, current_run_time[method_id]), current_avg_run_time, ) self.histogram_methods[method]["total_loss"] += self.histogram_methods[ @@ -1319,7 +1318,9 @@ def _get_histogram( ] = suggested_bin_count # calculate the stored histogram bins - bin_counts, bin_edges = np.histogram(values, bins=n_equal_bins) + bin_counts, bin_edges = cast( + tuple[np.ndarray, np.ndarray], np.histogram(values, bins=n_equal_bins) + ) return bin_counts, bin_edges def _merge_histogram(self, values: np.ndarray | pd.Series) -> None: @@ -1571,7 +1572,7 @@ def _get_percentile(self, percentiles: np.ndarray | list[float]) -> list[float]: bin_counts = bin_counts.astype(float) normalized_bin_counts = bin_counts / np.sum(bin_counts) - cumsum_bin_counts = np.cumsum(normalized_bin_counts) + cumsum_bin_counts = cast(np.ndarray, np.cumsum(normalized_bin_counts)) median_value = None median_bin_inds = np.abs(cumsum_bin_counts - 0.5) < 1e-10 @@ -1579,6 +1580,7 @@ def _get_percentile(self, percentiles: np.ndarray | list[float]) -> list[float]: median_value = np.mean(bin_edges[np.append([False], median_bin_inds)]) # use the floor by slightly increasing cases where no bin exist. + cumsum_bin_counts = cast(np.ndarray, cumsum_bin_counts.copy()) cumsum_bin_counts[zero_inds] += 1e-15 # add initial zero bin @@ -1703,18 +1705,24 @@ def median_abs_deviation(self) -> float | np.float64: np.append([True], np.diff(bin_edges_impose) > 1e-14) ] - bin_counts_impose_pos: npt.NDArray[np.float64] = np.interp( - bin_edges_impose, - bin_edges_pos, - np.cumsum(np.append([0], bin_counts_pos)), + bin_counts_impose_pos = cast( + np.ndarray, + np.interp( + bin_edges_impose, + bin_edges_pos, + np.cumsum(np.append([0], bin_counts_pos)), + ), ) - bin_counts_impose_neg: npt.NDArray[np.float64] = np.interp( - bin_edges_impose, - bin_edges_neg, - np.cumsum(np.append([0], bin_counts_neg)), + bin_counts_impose_neg = cast( + np.ndarray, + np.interp( + bin_edges_impose, + bin_edges_neg, + np.cumsum(np.append([0], bin_counts_neg)), + ), ) - bin_counts_impose: npt.NDArray[np.float64] = ( - bin_counts_impose_pos + bin_counts_impose_neg + bin_counts_impose = cast( + np.ndarray, bin_counts_impose_pos + bin_counts_impose_neg ) median_inds = np.abs(bin_counts_impose - 0.5) < 1e-10 @@ -1729,9 +1737,10 @@ def _get_quantiles(self) -> None: :return: list of quantiles """ - percentiles: np.ndarray = np.linspace(0, 100, (self._num_quantiles - 1) + 2)[ - 1:-1 - ] + percentile_range = cast( + np.ndarray, np.linspace(0, 100, (self._num_quantiles - 1) + 2) + ) + percentiles: np.ndarray = percentile_range[1:-1] self.quantiles = self._get_percentile(percentiles=percentiles) def _update_helper(self, df_series_clean: pd.Series, profile: dict) -> None: diff --git a/dataprofiler/profilers/profile_builder.py b/dataprofiler/profilers/profile_builder.py index 6c87bea5..fbc212a7 100644 --- a/dataprofiler/profilers/profile_builder.py +++ b/dataprofiler/profilers/profile_builder.py @@ -612,7 +612,7 @@ def clean_data_and_get_base_stats( ) else: sample_ind_generator = profiler_utils.partition( - sample_ids[0], chunk_size=sample_size + list(sample_ids[0]), chunk_size=sample_size ) na_columns: dict = dict() @@ -2346,11 +2346,11 @@ def _get_correlation( # fill correlation matrix with nan initially n_cols = len(self._profile) - corr_mat = np.full((n_cols, n_cols), np.nan) + corr_mat = cast(np.ndarray, np.full((n_cols, n_cols), np.nan)) # then, fill in the correlations for valid columns - rows = [[id] for id in clean_column_ids] - corr_mat[rows, clean_column_ids] = np.corrcoef(data, rowvar=False) + corr_rows = np.ix_(clean_column_ids, clean_column_ids) + corr_mat[corr_rows] = np.corrcoef(data, rowvar=False) return corr_mat @@ -2399,26 +2399,28 @@ def _merge_correlation(self, other: StructuredProfiler) -> pd.DataFrame: return None # get column indices without nan - col_ids1 = np.where(~np.isnan(corr_mat1).all(axis=0))[0] - col_ids2 = np.where(~np.isnan(corr_mat2).all(axis=0))[0] + col_ids1 = cast(np.ndarray, np.where(~np.isnan(corr_mat1).all(axis=0))[0]) + col_ids2 = cast(np.ndarray, np.where(~np.isnan(corr_mat2).all(axis=0))[0]) if len(col_ids1) != len(col_ids2) or len(col_ids1) <= 1: return None - if (col_ids1 != col_ids2).any(): + col_ids1_list = cast(list[int], col_ids1.tolist()) + col_ids2_list = cast(list[int], col_ids2.tolist()) + if col_ids1_list != col_ids2_list: return None mean1 = np.array( [ self._profile[idx].profile["statistics"]["mean"] for idx in range(len(self._profile)) - if idx in col_ids1 + if idx in col_ids1_list ] ) std1 = np.array( [ self._profile[idx].profile["statistics"]["stddev"] for idx in range(len(self._profile)) - if idx in col_ids1 + if idx in col_ids1_list ] ) @@ -2426,14 +2428,14 @@ def _merge_correlation(self, other: StructuredProfiler) -> pd.DataFrame: [ other._profile[idx].profile["statistics"]["mean"] for idx in range(len(self._profile)) - if idx in col_ids2 + if idx in col_ids2_list ] ) std2 = np.array( [ other._profile[idx].profile["statistics"]["stddev"] for idx in range(len(self._profile)) - if idx in col_ids2 + if idx in col_ids2_list ] ) return self._merge_correlation_helper( @@ -2561,7 +2563,7 @@ def _update_chi2(self) -> np.ndarray: """ n_cols = len(self._profile) # Fill matrix with nan initially - chi2_mat = np.full((n_cols, n_cols), np.nan) + chi2_mat = cast(np.ndarray, np.full((n_cols, n_cols), np.nan)) # Compute chi_sq for each for i in range(n_cols): data_stats_compiler1 = self._profile[i].profiles["data_stats_profile"] @@ -2570,7 +2572,7 @@ def _update_chi2(self) -> np.ndarray: continue for j in range(i, n_cols): if i == j: - chi2_mat[i][j] = 1 + chi2_mat[i, j] = 1 continue data_stats_compiler2 = self._profile[j].profiles["data_stats_profile"] profiler2 = data_stats_compiler2._profiles["category"] @@ -2583,8 +2585,8 @@ def _update_chi2(self) -> np.ndarray: profiler2.categorical_counts, profiler2.sample_size, ) - chi2_mat[i][j] = results["p-value"] - chi2_mat[j][i] = results["p-value"] + chi2_mat[i, j] = results["p-value"] + chi2_mat[j, i] = results["p-value"] return chi2_mat @@ -2676,11 +2678,12 @@ def _update_null_replication_metrics(self, clean_samples: dict) -> None: mean_not_null = sum_not_null / true_count # Convert numpy arrays to lists (serializable) - sum_null_list = sum_null.tolist() - sum_not_null_list = sum_not_null.tolist() + sum_null_list = cast(list[float], sum_null.tolist()) + sum_not_null_list = cast(list[float], sum_not_null.tolist()) - mean_null_list = mean_null.tolist() - mean_not_null_list = mean_not_null.tolist() + mean_null_list = cast(list[float], mean_null.tolist()) + mean_not_null_array = cast(np.ndarray, mean_not_null) + mean_not_null_list = cast(list[float], mean_not_null_array.tolist()) # Array index serves as class label # 0 indicates not null, 1 indicates null @@ -2726,7 +2729,9 @@ def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: for profile in other._profile ] ) - total_row_sum: np.ndarray = self_row_sum + other_row_sum + self_row_sum = cast(np.ndarray, self_row_sum) + other_row_sum = cast(np.ndarray, other_row_sum) + total_row_sum = cast(np.ndarray, self_row_sum + other_row_sum) merged_properties: dict = defaultdict(dict) for col_id in range(len(self._profile)): self_profile = self._profile[col_id] @@ -2757,7 +2762,7 @@ def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: else None ) # Initialize zeros array of size (number of columns - 1) - sum_null = np.zeros(len(self._profile) - 1) + sum_null = cast(np.ndarray, np.zeros(len(self._profile) - 1)) # Add sum_nulls if they exist # Guarantees that at least one of self_sum_null, other_sum_null != None @@ -2767,7 +2772,7 @@ def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: if other_sum_null is not None: sum_null += np.asarray(other_sum_null) - sum_not_null = np.delete(total_row_sum, col_id) - sum_null + sum_not_null = cast(np.ndarray, np.delete(total_row_sum, col_id) - sum_null) mean_null = sum_null / null_count @@ -2778,11 +2783,12 @@ def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: mean_not_null = sum_not_null / true_count # Convert numpy arrays to lists (serializable) - sum_null_list = sum_null.tolist() - sum_not_null_list = sum_not_null.tolist() + sum_null_list = cast(list[float], sum_null.tolist()) + sum_not_null_list = cast(list[float], sum_not_null.tolist()) - mean_null_list = mean_null.tolist() - mean_not_null_list = mean_not_null.tolist() + mean_null_list = cast(list[float], mean_null.tolist()) + mean_not_null_array = cast(np.ndarray, mean_not_null) + mean_not_null_list = cast(list[float], mean_not_null_array.tolist()) merged_properties[col_id] = { # Array index serves as class label From ae11f3b864a27d593b4f4583046c69ad5948be4f Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 15:50:04 -0500 Subject: [PATCH 12/15] fix: req issue --- .pre-commit-config.yaml | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7506c0aa..8de323b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,7 +56,7 @@ repos: 'pandas>=1.1.2,<3.0.0', python-dateutil>=2.7.5, pytz>=2020.1, - pyarrow>=1.0.1, + 'pyarrow>=1.0.1,<24.0.0', 'chardet>=3.0.4,<7.0.0', fastavro>=1.1.0, python-snappy>=0.7.1, @@ -112,7 +112,7 @@ repos: hooks: - id: check-manifest additional_dependencies: ['h5py', 'wheel', 'future', 'numpy>=1.22.0,<3.0.0', - 'pandas', 'python-dateutil', 'pytz', 'pyarrow', 'chardet', + 'pandas', 'python-dateutil', 'pytz', 'pyarrow<24.0.0', 'chardet', 'fastavro>=1.1.0', 'python-snappy', 'charset-normalizer', 'psutil', 'scipy>=1.10.0', 'requests>=2.32.4', 'networkx', 'typing-extensions', 'HLL', 'datasketches', 'packaging>=23.0', 'boto3>=1.37.15', diff --git a/requirements.txt b/requirements.txt index 3db3daad..d0d112cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ numpy>=1.22.0,<3.0.0 pandas>=1.1.2,<3.0.0 python-dateutil>=2.7.5 pytz>=2020.1 -pyarrow>=1.0.1 +pyarrow>=1.0.1,<24.0.0 chardet>=3.0.4,<7.0.0 fastavro>=1.1.0 python-snappy>=0.7.1 From 004ce6c442212961591e43699e0e5bca8a4769fc Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 15:50:49 -0500 Subject: [PATCH 13/15] fix: test failure --- .../tests/profilers/test_base_column_profilers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dataprofiler/tests/profilers/test_base_column_profilers.py b/dataprofiler/tests/profilers/test_base_column_profilers.py index 4ab7182c..2fc8e823 100644 --- a/dataprofiler/tests/profilers/test_base_column_profilers.py +++ b/dataprofiler/tests/profilers/test_base_column_profilers.py @@ -176,11 +176,13 @@ def test_cannot_instantiate(self): """showing we normally can't instantiate an abstract class""" with self.assertRaises(TypeError) as e: BaseColumnPrimitiveTypeProfiler() - self.assertEqual( - "Can't instantiate abstract class BaseColumnPrimitiveTypeProfiler " - "with abstract methods _update_helper, profile, report, update", - str(e.exception), + error_message = str(e.exception) + self.assertIn( + "Can't instantiate abstract class " "BaseColumnPrimitiveTypeProfiler", + error_message, ) + for abstract_method in ("_update_helper", "profile", "report", "update"): + self.assertIn(abstract_method, error_message) def test_combine_unqiue_sets(self): a = [1, 2, 3] From a9b83cb61ee07311279425e539a9aae420f9c63c Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 15:57:31 -0500 Subject: [PATCH 14/15] fix: bug in test --- .../tests/profilers/test_column_profile_compilers.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dataprofiler/tests/profilers/test_column_profile_compilers.py b/dataprofiler/tests/profilers/test_column_profile_compilers.py index 7cb6b57f..7ae1aa07 100644 --- a/dataprofiler/tests/profilers/test_column_profile_compilers.py +++ b/dataprofiler/tests/profilers/test_column_profile_compilers.py @@ -24,11 +24,12 @@ def test_cannot_instantiate(self): """showing we normally can't instantiate an abstract class""" with self.assertRaises(TypeError) as e: col_pro_compilers.BaseCompiler() - self.assertRegex( - str(e.exception), - "Can't instantiate abstract class BaseCompiler with " - "abstract methods? report", + error_message = str(e.exception) + self.assertIn( + "Can't instantiate abstract class BaseCompiler", + error_message, ) + self.assertIn("report", error_message) @mock.patch.multiple( col_pro_compilers.BaseCompiler, From f5a238bd3f33f19330c5cb6ec37532c2fff68ea3 Mon Sep 17 00:00:00 2001 From: Jeremy Goodsitt Date: Mon, 6 Jul 2026 16:10:58 -0500 Subject: [PATCH 15/15] fix: assertalmostequal --- .../profilers/test_text_column_profile.py | 201 ++++++++++-------- 1 file changed, 110 insertions(+), 91 deletions(-) diff --git a/dataprofiler/tests/profilers/test_text_column_profile.py b/dataprofiler/tests/profilers/test_text_column_profile.py index 699e35cb..a51f808f 100644 --- a/dataprofiler/tests/profilers/test_text_column_profile.py +++ b/dataprofiler/tests/profilers/test_text_column_profile.py @@ -696,106 +696,125 @@ def test_json_encode_after_update(self, time): # popping vocab and comparing as set below since order is random serialized_vocab = serialized_dict["data"].pop("vocab") - serialized = json.dumps(serialized_dict) - - expected = json.dumps( - { - "class": "TextColumn", - "data": { - "min": 1.0, - "max": 12.0, - "_top_k_modes": 5, - "sum": 38.0, - "_biased_variance": 9.33884297520661, - "_biased_skewness": 1.8025833203700588, - "_biased_kurtosis": 2.7208317017777395, - "_median_is_enabled": True, - "_median_abs_dev_is_enabled": True, - "max_histogram_bin": 100000, - "min_histogram_bin": 1000, - "histogram_bin_method_names": ["custom"], - "histogram_selection": None, - "user_set_histogram_bin": 5, - "bias_correction": True, - "_mode_is_enabled": True, - "num_zeros": 0, - "num_negatives": 0, - "_num_quantiles": 1000, - "histogram_methods": { - "custom": { - "total_loss": 0.0, - "current_loss": 0.0, - "suggested_bin_count": 5, - "histogram": { - "bin_counts": None, - "bin_edges": None, - }, - } - }, - "_stored_histogram": { - "total_loss": 7.63, - "current_loss": 7.63, - "suggested_bin_count": 1000, + expected_dict = { + "class": "TextColumn", + "data": { + "min": 1.0, + "max": 12.0, + "_top_k_modes": 5, + "sum": 38.0, + "_biased_variance": 9.33884297520661, + "_biased_skewness": 1.8025833203700588, + "_biased_kurtosis": 2.7208317017777395, + "_median_is_enabled": True, + "_median_abs_dev_is_enabled": True, + "max_histogram_bin": 100000, + "min_histogram_bin": 1000, + "histogram_bin_method_names": ["custom"], + "histogram_selection": None, + "user_set_histogram_bin": 5, + "bias_correction": True, + "_mode_is_enabled": True, + "num_zeros": 0, + "num_negatives": 0, + "_num_quantiles": 1000, + "histogram_methods": { + "custom": { + "total_loss": 0.0, + "current_loss": 0.0, + "suggested_bin_count": 5, "histogram": { - "bin_counts": [6, 4, 0, 0, 1], - "bin_edges": [ - 1.0, - 3.2, - 5.4, - 7.6000000000000005, - 9.8, - 12.0, - ], + "bin_counts": None, + "bin_edges": None, }, + } + }, + "_stored_histogram": { + "total_loss": 7.63, + "current_loss": 7.63, + "suggested_bin_count": 1000, + "histogram": { + "bin_counts": [6, 4, 0, 0, 1], + "bin_edges": [ + 1.0, + 3.2, + 5.4, + 7.6000000000000005, + 9.8, + 12.0, + ], }, - "_batch_history": [ - { - "match_count": 11, - "sample_size": 11, - "min": 1.0, - "max": 12.0, - "sum": 38.0, - "biased_variance": 9.33884297520661, - "mean": 3.4545454545454546, - "biased_skewness": 1.8025833203700588, - "biased_kurtosis": 2.7208317017777395, - } - ], - "_NumericStatsMixin__calculations": { - "min": "_get_min", - "max": "_get_max", - "sum": "_get_sum", - "variance": "_get_variance", - "skewness": "_get_skewness", - "kurtosis": "_get_kurtosis", - "histogram_and_quantiles": "_get_histogram_and_quantiles", - }, - "name": None, - "col_index": np.nan, - "sample_size": 11, - "metadata": {}, - "times": { - "vocab": 1.0, + }, + "_batch_history": [ + { + "match_count": 11, + "sample_size": 11, "min": 1.0, - "max": 1.0, - "sum": 1.0, - "variance": 1.0, - "skewness": 1.0, - "kurtosis": 1.0, - "histogram_and_quantiles": 1.0, - }, - "thread_safe": True, - "match_count": 11, - "_TextColumn__calculations": {"vocab": "_update_vocab"}, - "type": "string", + "max": 12.0, + "sum": 38.0, + "biased_variance": 9.33884297520661, + "mean": 3.4545454545454546, + "biased_skewness": 1.8025833203700588, + "biased_kurtosis": 2.7208317017777395, + } + ], + "_NumericStatsMixin__calculations": { + "min": "_get_min", + "max": "_get_max", + "sum": "_get_sum", + "variance": "_get_variance", + "skewness": "_get_skewness", + "kurtosis": "_get_kurtosis", + "histogram_and_quantiles": "_get_histogram_and_quantiles", }, - } - ) + "name": None, + "col_index": np.nan, + "sample_size": 11, + "metadata": {}, + "times": { + "vocab": 1.0, + "min": 1.0, + "max": 1.0, + "sum": 1.0, + "variance": 1.0, + "skewness": 1.0, + "kurtosis": 1.0, + "histogram_and_quantiles": 1.0, + }, + "thread_safe": True, + "match_count": 11, + "_TextColumn__calculations": {"vocab": "_update_vocab"}, + "type": "string", + }, + } + + for field in ("_biased_variance", "_biased_skewness", "_biased_kurtosis"): + self.assertAlmostEqual( + serialized_dict["data"][field], expected_dict["data"][field] + ) + serialized_dict["data"][field] = expected_dict["data"][field] + + for field in ( + "min", + "max", + "sum", + "biased_variance", + "mean", + "biased_skewness", + "biased_kurtosis", + ): + self.assertAlmostEqual( + serialized_dict["data"]["_batch_history"][0][field], + expected_dict["data"]["_batch_history"][0][field], + ) + serialized_dict["data"]["_batch_history"][0][field] = expected_dict["data"][ + "_batch_history" + ][0][field] expected_vocab = profiler.vocab expected_quantiles = profiler.quantiles - self.assertEqual(serialized, expected) + self.assertEqual(json.dumps(serialized_dict), json.dumps(expected_dict)) self.assertSetEqual(set(serialized_vocab), set(expected_vocab)) self.assertListEqual(serialized_quantiles, expected_quantiles)