refactor: add py3.12 and py3.13 to tests

.github/workflows/test-package.yml

@@ -14,12 +14,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4.3.1
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v5.6.0
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

dataprofiler/_version.py

@@ -14,6 +14,7 @@
 import re
 import subprocess
 import sys
+from typing import Any, Callable
 
 
 def get_keywords():
@@ -52,7 +53,7 @@ class NotThisMethod(Exception):
 
 
 LONG_VERSION_PY = {} # type: ignore
-HANDLERS = {}
+HANDLERS: dict[str, dict[str, Callable[..., Any]]] = {}
 
 
 def register_vcs_handler(vcs, method):  # decorator

dataprofiler/data_readers/csv_data.py

@@ -674,8 +674,10 @@ def is_match(cls, file_path: str, options: Optional[Dict] = None) -> bool:
         empty_line_count = 0
         delimiter_count = dict()
 
-        delimiter_regex = data_utils.get_delimiter_regex(delimiter, quotechar)
-        space_regex = data_utils.get_delimiter_regex(" ", quotechar)
+        delimiter_regex = data_utils.get_delimiter_regex(
+            delimiter or ",", quotechar or '"'
+        )
+        space_regex = data_utils.get_delimiter_regex(" ", quotechar or '"')
 
         # Count the possible delimiters
         for line in data_as_str.split("\n")[header:]:

dataprofiler/data_readers/data_utils.py

@@ -277,7 +277,7 @@ def read_json(
     return lines
 
 
-def reservoir(file: TextIOWrapper, sample_nrows: int) -> list:
+def reservoir(file: TextIOWrapper | StringIO, sample_nrows: int) -> list:
     """
     Implement the mathematical logic of Reservoir sampling.
 
@@ -344,7 +344,9 @@ def reservoir(file: TextIOWrapper, sample_nrows: int) -> list:
     return values
 
 
-def rsample(file_path: TextIOWrapper, sample_nrows: int, args: dict) -> StringIO:
+def rsample(
+    file_path: TextIOWrapper | StringIO, sample_nrows: int, args: dict
+) -> StringIO:
     """
     Implement Reservoir Sampling to sample n rows out of a total of M rows.
 
@@ -369,7 +371,7 @@ def rsample(file_path: TextIOWrapper, sample_nrows: int, args: dict) -> StringIO
 
 
 def read_csv_df(
-    file_path: Union[str, BytesIO, TextIOWrapper],
+    file_path: Union[str, StringIO, BytesIO, TextIOWrapper],
     delimiter: Optional[str],
     header: Optional[int],
     sample_nrows: Optional[int] = None,
@@ -425,7 +427,7 @@ def read_csv_df(
         file_path = open(file_path, encoding=encoding)
         is_file_open = True
 
-    file_data = file_path
+    file_data: TextIOWrapper | StringIO = file_path
     if sample_nrows:
         file_data = rsample(file_path, sample_nrows, args)
     fo = pd.read_csv(file_data, **args)

dataprofiler/data_readers/filepath_or_buffer.py

@@ -93,7 +93,7 @@ def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None:
                 TextIOWrapper, self._filepath_or_buffer
             )  # guaranteed by self._is_wrapped
             wrapper = self._filepath_or_buffer
-            self._filepath_or_buffer = wrapper.buffer
+            self._filepath_or_buffer = cast(BytesIO, wrapper.buffer)
             wrapper.detach()
 
         if isinstance(self._filepath_or_buffer, (StringIO, BytesIO)):

dataprofiler/labelers/base_data_labeler.py

@@ -17,7 +17,7 @@
 from . import data_processing, utils
 from .base_model import BaseModel
 
-default_labeler_dir = utils.find_resources_dir("labelers")
+default_labeler_dir = str(utils.find_resources_dir("labelers"))
 
 
 class BaseDataLabeler:

dataprofiler/labelers/column_name_model.py

@@ -176,7 +176,7 @@ def _reconstruct_model(self) -> None:
         pass
 
     def _need_to_reconstruct_model(self) -> bool:
-        pass
+        return False
 
     def reset_weights(self) -> None:
         """Reset weights function."""

dataprofiler/labelers/data_labelers.py

@@ -12,7 +12,7 @@
 from .base_model import BaseModel
 from .data_processing import BaseDataPostprocessor, BaseDataPreprocessor
 
-default_labeler_dir = utils.find_resources_dir("labelers")
+default_labeler_dir = str(utils.find_resources_dir("labelers"))
 
 
 def train_structured_labeler(

dataprofiler/labelers/data_processing.py

@@ -19,7 +19,7 @@
 
 from . import utils
 
-default_labeler_dir = utils.find_resources_dir("labelers")
+default_labeler_dir = str(utils.find_resources_dir("labelers"))
 
 
 Processor = TypeVar("Processor", bound="BaseDataProcessor")
@@ -491,7 +491,7 @@ def gen_none() -> Generator[None, None, None]:
                 for start, end, label in label_set:
                     label_index = label_mapping[label]
                     label_buffer[start:end] = label_index
-                label_buffer = label_buffer.tolist()
+                label_buffer_list = label_buffer.tolist()
 
             # loop until the buffer is empty and placed as requested
             buffer_ind = 0
@@ -543,7 +543,7 @@ def gen_none() -> Generator[None, None, None]:
                     batch_data["samples"].append(sample_buffer[buffer_ind:separate_ind])
                     if label_set is not None:
                         batch_data["labels"].append(
-                            label_buffer[buffer_ind:separate_ind]
+                            label_buffer_list[buffer_ind:separate_ind]
                             + [label_mapping[pad_label]] * pad_len
                         )
 
@@ -610,7 +610,7 @@ def gen_none() -> Generator[None, None, None]:
 
                         if label_set is not None:
                             flattened_entities.extend(
-                                label_buffer[buffer_ind:separate_ind]
+                                label_buffer_list[buffer_ind:separate_ind]
                             )
 
                         buffer_ind = separate_ind

dataprofiler/labelers/regex_model.py

@@ -7,6 +7,7 @@
 import os
 import re
 import sys
+from typing import Any
 
 import numpy as np
 
@@ -167,7 +168,7 @@ def _reconstruct_model(self) -> None:
         pass
 
     def _need_to_reconstruct_model(self) -> bool:
-        pass
+        return False
 
     def reset_weights(self) -> None:
         """Reset weights."""
@@ -225,7 +226,7 @@ def predict(
 
         # Construct array initial regex predictions where background is
         # predicted.
-        predictions = [np.empty((0,))] * 100
+        predictions: list[Any] = [np.empty((0,))] * 100
         i = 0
         for i, input_string in enumerate(data):
 
@@ -247,7 +248,7 @@ def predict(
             if verbose:
                 sys.stdout.flush()
                 sys.stdout.write(f"\rData Samples Processed: {i + 1:d}   ")
-            predictions[i] = pred
+            predictions[i] = pred.tolist()
 
         if verbose:
             logger.info(f"\rData Samples Processed: {i + 1:d}   ")

dataprofiler/profilers/float_column_profile.py

@@ -4,6 +4,7 @@
 
 import copy
 import re
+from typing import cast
 
 import numpy as np
 import pandas as pd
@@ -201,7 +202,7 @@ def profile(self) -> dict:
 
         :return:
         """
-        profile = NumericStatsMixin.profile(self)
+        profile = cast(dict, NumericStatsMixin.profile.__get__(self, type(self)))
         profile.update(
             dict(
                 precision=dict(

dataprofiler/profilers/int_column_profile.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+from typing import cast
+
 import numpy as np
 import pandas as pd
 
@@ -99,7 +101,7 @@ def profile(self) -> dict:
 
         :return:
         """
-        return NumericStatsMixin.profile(self)
+        return cast(dict, NumericStatsMixin.profile.__get__(self, type(self)))
 
     @property
     def data_type_ratio(self) -> float | None:

dataprofiler/profilers/numerical_column_stats.py

@@ -366,6 +366,7 @@ def _add_helper(
             other1._median_abs_dev_is_enabled and other2._median_abs_dev_is_enabled
         )
 
+    @property
     def profile(self) -> dict:
         """
         Return profile of the column.
@@ -408,7 +409,7 @@ def report(self, remove_disabled_flag: bool = False) -> dict:
         :rtype: Profile
         """
         calcs_dict_keys = self._NumericStatsMixin__calculations.keys()
-        profile = self.profile()
+        profile = self.profile
 
         if remove_disabled_flag:
             profile_keys = list(profile.keys())

dataprofiler/profilers/profile_builder.py

@@ -12,7 +12,7 @@
 from collections import OrderedDict, defaultdict
 from datetime import datetime
 from multiprocessing.pool import Pool
-from typing import Any, Generator, List, Optional, TypeVar, cast
+from typing import Any, Generator, List, Optional, Sized, TypeVar, cast
 
 import networkx as nx
 import numpy as np
@@ -738,7 +738,7 @@ def __init__(
         self._min_sample_size: int = 5000
 
         # assign data labeler
-        data_labeler_options = self.options.data_labeler
+        data_labeler_options = cast(Any, self.options).data_labeler
         if (
             data_labeler_options.is_enabled
             and data_labeler_options.data_labeler_object is None
@@ -751,11 +751,13 @@ def __init__(
                     dirpath=data_labeler_options.data_labeler_dirpath,
                     load_options=None,
                 )
-                self.options.set({"data_labeler.data_labeler_object": data_labeler})
+                cast(Any, self.options).set(
+                    {"data_labeler.data_labeler_object": data_labeler}
+                )
 
             except Exception as e:
                 profiler_utils.warn_on_profile("data_labeler", e)
-                self.options.set({"data_labeler.is_enabled": False})
+                cast(Any, self.options).set({"data_labeler.is_enabled": False})
 
     def _add_error_checks(self, other: BaseProfiler) -> None:
         """
@@ -976,7 +978,7 @@ def update_profile(
                 f"one of the following: {self._allowed_external_data_types}"
             )
 
-        if not len(data):
+        if not len(cast(Sized, data)):
             warnings.warn(
                 "The passed dataset was empty, hence no data was " "profiled."
             )
@@ -2674,18 +2676,18 @@ def _update_null_replication_metrics(self, clean_samples: dict) -> None:
                 mean_not_null = sum_not_null / true_count
 
             # Convert numpy arrays to lists (serializable)
-            sum_null = sum_null.tolist()
-            sum_not_null = sum_not_null.tolist()
+            sum_null_list = sum_null.tolist()
+            sum_not_null_list = sum_not_null.tolist()
 
-            mean_null = mean_null.tolist()
-            mean_not_null = mean_not_null.tolist()
+            mean_null_list = mean_null.tolist()
+            mean_not_null_list = mean_not_null.tolist()
 
             # Array index serves as class label
             # 0 indicates not null, 1 indicates null
             self._null_replication_metrics[col_id] = {
                 "class_prior": [prior_not_null, prior_null],
-                "class_sum": [sum_not_null, sum_null],
-                "class_mean": [mean_not_null, mean_null],
+                "class_sum": [sum_not_null_list, sum_null_list],
+                "class_mean": [mean_not_null_list, mean_null_list],
             }
 
     def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict:
@@ -2776,18 +2778,18 @@ def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict:
                 mean_not_null = sum_not_null / true_count
 
             # Convert numpy arrays to lists (serializable)
-            sum_null = sum_null.tolist()
-            sum_not_null = sum_not_null.tolist()
+            sum_null_list = sum_null.tolist()
+            sum_not_null_list = sum_not_null.tolist()
 
-            mean_null = mean_null.tolist()
-            mean_not_null = mean_not_null.tolist()
+            mean_null_list = mean_null.tolist()
+            mean_not_null_list = mean_not_null.tolist()
 
             merged_properties[col_id] = {
                 # Array index serves as class label
                 # 0 indicates not null, 1 indicates null
                 "class_prior": [prior_not_null, prior_null],
-                "class_sum": [sum_not_null, sum_null],
-                "class_mean": [mean_not_null, mean_null],
+                "class_sum": [sum_not_null_list, sum_null_list],
+                "class_mean": [mean_not_null_list, mean_null_list],
             }
 
         return merged_properties

dataprofiler/profilers/profiler_options.py

@@ -9,7 +9,7 @@
 from typing import Any, Generic, TypeVar, cast
 
 from ..labelers.base_data_labeler import BaseDataLabeler
-from ..plugins.__init__ import get_plugins
+from ..plugins import get_plugins
 from . import profiler_utils
 from .json_decoder import load_option
 

dataprofiler/profilers/text_column_profile.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import itertools
+from typing import cast
 
 import numpy as np
 import pandas as pd
@@ -91,7 +92,7 @@ def profile(self) -> dict:
 
         :return:
         """
-        profile = NumericStatsMixin.profile(self)
+        profile = cast(dict, NumericStatsMixin.profile.__get__(self, type(self)))
         # remove num_zeros and num_negative updated from numeric profile
         profile.pop("num_zeros")
         profile.pop("num_negatives")