refactor: add py3.12 and py3.13 to tests

.github/workflows/test-package.yml

@@ -14,12 +14,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4.3.1
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v5.6.0
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

.pre-commit-config.yaml

@@ -40,10 +40,12 @@ repos:
   # Mypy: Optional static type checking
   # https://github.com/pre-commit/mirrors-mypy
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.982
+    rev: v1.11.2
     hooks:
       - id: mypy
-        exclude: (^dataprofiler/tests/|^resources/|^examples|venv*/|versioneer.py|dataprofiler/_version.py|_docs/)
+        # Let mypy own target selection and exclusions via setup.cfg.
+        args: [--config-file=setup.cfg, dataprofiler]
+        pass_filenames: false
         language_version: python3
         additional_dependencies: # Keep up-to-date with the respective requirement files
           [

dataprofiler/_version.py

@@ -14,6 +14,7 @@
 import re
 import subprocess
 import sys
+from typing import Any, Callable
 
 
 def get_keywords():
@@ -52,7 +53,7 @@ class NotThisMethod(Exception):
 
 
 LONG_VERSION_PY = {} # type: ignore
-HANDLERS = {}
+HANDLERS: dict[str, dict[str, Callable[..., Any]]] = {}
 
 
 def register_vcs_handler(vcs, method):  # decorator

dataprofiler/data_readers/base_data.py

@@ -155,7 +155,7 @@ def get_batch_generator(
     ) -> Generator[Union[pd.DataFrame, List], None, None]:
         """Get batch generator."""
         data_length = len(self.data)
-        indices = np.random.permutation(data_length)
+        indices: np.ndarray = np.random.permutation(data_length)
         for i in range(0, data_length, batch_size):
             if isinstance(self.data, pd.DataFrame):
                 yield self.data.iloc[indices[i : i + batch_size], :]

dataprofiler/data_readers/csv_data.py

@@ -674,8 +674,10 @@ def is_match(cls, file_path: str, options: Optional[Dict] = None) -> bool:
         empty_line_count = 0
         delimiter_count = dict()
 
-        delimiter_regex = data_utils.get_delimiter_regex(delimiter, quotechar)
-        space_regex = data_utils.get_delimiter_regex(" ", quotechar)
+        delimiter_regex = data_utils.get_delimiter_regex(
+            delimiter or ",", quotechar or '"'
+        )
+        space_regex = data_utils.get_delimiter_regex(" ", quotechar or '"')
 
         # Count the possible delimiters
         for line in data_as_str.split("\n")[header:]:
@@ -722,15 +724,19 @@ def is_match(cls, file_path: str, options: Optional[Dict] = None) -> bool:
             active_line_count - max_deviation_count
         ) / active_line_count
 
-        delimiter_count_values = np.array(list(delimiter_count.values()))
-        count_percent = delimiter_count_values / np.sum(delimiter_count_values)
+        delimiter_count_values = cast(
+            np.ndarray, np.array(list(delimiter_count.values()))
+        )
+        count_percent = cast(
+            np.ndarray, delimiter_count_values / np.sum(delimiter_count_values)
+        )
 
         if not count_percent.size:
             return False
 
-        max_count_index = count_percent.argmax()
+        max_count_index = int(count_percent.argmax())
         max_count_value = list(delimiter_count.keys())[max_count_index]
-        max_count_percent = count_percent[max_count_index]
+        max_count_percent = float(count_percent[max_count_index])
 
         # Inferred the file was a CSV
         if (max_count_value > 0 or delimiter is None) and (

dataprofiler/data_readers/data_utils.py

@@ -6,6 +6,7 @@
 import random
 import re
 import urllib
+import urllib.parse
 from collections import OrderedDict
 from io import BytesIO, StringIO, TextIOWrapper
 from itertools import islice
@@ -25,12 +26,12 @@
 
 import boto3
 import botocore
-import dateutil
 import numpy as np
 import pandas as pd
 import pyarrow.parquet as pq
 import requests
 from chardet.universaldetector import UniversalDetector
+from dateutil import parser as dateutil_parser  # type: ignore[import-untyped]
 from typing_extensions import TypeGuard
 
 from .. import dp_logging, rng_utils
@@ -277,7 +278,7 @@ def read_json(
     return lines
 
 
-def reservoir(file: TextIOWrapper, sample_nrows: int) -> list:
+def reservoir(file: TextIOWrapper | StringIO, sample_nrows: int) -> list:
     """
     Implement the mathematical logic of Reservoir sampling.
 
@@ -344,7 +345,9 @@ def reservoir(file: TextIOWrapper, sample_nrows: int) -> list:
     return values
 
 
-def rsample(file_path: TextIOWrapper, sample_nrows: int, args: dict) -> StringIO:
+def rsample(
+    file_path: TextIOWrapper | StringIO, sample_nrows: int, args: dict
+) -> StringIO:
     """
     Implement Reservoir Sampling to sample n rows out of a total of M rows.
 
@@ -369,7 +372,7 @@ def rsample(file_path: TextIOWrapper, sample_nrows: int, args: dict) -> StringIO
 
 
 def read_csv_df(
-    file_path: Union[str, BytesIO, TextIOWrapper],
+    file_path: Union[str, StringIO, BytesIO, TextIOWrapper],
     delimiter: Optional[str],
     header: Optional[int],
     sample_nrows: Optional[int] = None,
@@ -425,7 +428,7 @@ def read_csv_df(
         file_path = open(file_path, encoding=encoding)
         is_file_open = True
 
-    file_data = file_path
+    file_data: TextIOWrapper | StringIO = file_path
     if sample_nrows:
         file_data = rsample(file_path, sample_nrows, args)
     fo = pd.read_csv(file_data, **args)
@@ -503,10 +506,10 @@ def sample_parquet(
     # sample
     n_rows = parquet_table.num_rows
     if n_rows > sample_nrows:
-        sample_index = np.array([False] * n_rows)
+        sample_index = cast(np.ndarray, np.array([False] * n_rows))
         sample_index[random.sample(range(n_rows), sample_nrows)] = True
     else:
-        sample_index = np.array([True] * n_rows)
+        sample_index = cast(np.ndarray, np.array([True] * n_rows))
     sample_df = parquet_table.filter(sample_index).to_pandas()
 
     # Convert all the unicode columns to utf-8
@@ -674,7 +677,7 @@ def _decode_is_valid(encoding):
     # If no encoding is still found, default to utf-8
     if not encoding:
         encoding = "utf-8"
-    return encoding.lower()
+    return str(encoding).lower()
 
 
 def detect_cell_type(cell: str) -> str:
@@ -691,7 +694,7 @@ def detect_cell_type(cell: str) -> str:
 
         try:
             # need to ingore type bc https://github.com/python/mypy/issues/8878
-            if dateutil.parser.parse(cell, fuzzy=False):  # type:ignore
+            if dateutil_parser.parse(cell, fuzzy=False):  # type: ignore[attr-defined]
                 cell_type = "date"
         except (ValueError, OverflowError, TypeError):
             pass

dataprofiler/data_readers/filepath_or_buffer.py

@@ -93,7 +93,7 @@ def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None:
                 TextIOWrapper, self._filepath_or_buffer
             )  # guaranteed by self._is_wrapped
             wrapper = self._filepath_or_buffer
-            self._filepath_or_buffer = wrapper.buffer
+            self._filepath_or_buffer = cast(BytesIO, wrapper.buffer)
             wrapper.detach()
 
         if isinstance(self._filepath_or_buffer, (StringIO, BytesIO)):

dataprofiler/data_readers/parquet_data.py

@@ -68,7 +68,7 @@ def __init__(
             self._load_data(data)
 
     @property
-    def file_encoding(self) -> None:
+    def file_encoding(self) -> Optional[str]:
         """Set file encoding to None since not detected for avro."""
         return None
 

dataprofiler/labelers/base_data_labeler.py

@@ -17,7 +17,7 @@
 from . import data_processing, utils
 from .base_model import BaseModel
 
-default_labeler_dir = utils.find_resources_dir("labelers")
+default_labeler_dir = str(utils.find_resources_dir("labelers"))
 
 
 class BaseDataLabeler:

dataprofiler/labelers/character_level_cnn_model.py

@@ -8,6 +8,7 @@
 import sys
 import time
 from collections import defaultdict
+from typing import cast
 
 import numpy as np
 import tensorflow as tf
@@ -608,8 +609,9 @@ def _construct_model(self) -> None:
             _file_dir,
             "embeddings/glove-reduced-{}D.txt".format(self._parameters["dim_embed"]),
         )
-        embedding_matrix = np.zeros(
-            (max_char_encoding_id + 2, self._parameters["dim_embed"])
+        embedding_matrix = cast(
+            np.ndarray,
+            np.zeros((max_char_encoding_id + 2, self._parameters["dim_embed"])),
         )
         embedding_dict = build_embd_dictionary(embed_file)
 
@@ -911,8 +913,8 @@ def predict(
             )
         # Pre-allocate space for predictions
         confidences: list | np.ndarray = []
-        sentence_lengths = np.zeros((batch_size,), dtype=int)
-        predictions = np.zeros((batch_size, self._parameters["max_length"]))
+        sentence_lengths: np.ndarray = np.zeros((batch_size,), dtype=int)
+        predictions: np.ndarray = np.zeros((batch_size, self._parameters["max_length"]))
         if show_confidences:
             confidences = np.zeros(
                 (batch_size, self._parameters["max_length"], self.num_labels)

dataprofiler/labelers/classification_report_utils.py

@@ -32,25 +32,27 @@ def convert_confusion_matrix_to_MCM(conf_matrix: list | np.ndarray) -> np.ndarra
     """
     if not isinstance(conf_matrix, np.ndarray):
         conf_matrix = np.array(conf_matrix)
+    conf_matrix = cast(np.ndarray, conf_matrix)
     num_labels = len(conf_matrix)
     num_samples: int = int(np.sum(conf_matrix))
-    MCM = np.zeros((num_labels, 2, 2), dtype=np.int64)
+    MCM = cast(np.ndarray, np.zeros((num_labels, 2, 2), dtype=np.int64))
 
     # True Positives
     MCM[:, 1, 1] = np.sum(conf_matrix * np.eye(num_labels), axis=1)
 
     # False Negatives
-    MCM[:, 1, 0] = np.sum(
-        conf_matrix * (np.ones(num_labels) - np.eye(num_labels)), axis=1
+    non_diagonal_mask = cast(
+        np.ndarray, np.logical_not(np.eye(num_labels, dtype=bool)).astype(np.int64)
     )
+    MCM[:, 1, 0] = np.sum(conf_matrix * non_diagonal_mask, axis=1)
 
     # False Positives
-    MCM[:, 0, 1] = np.sum(
-        conf_matrix.T * (np.ones(num_labels) - np.eye(num_labels)), axis=1
-    )
+    MCM[:, 0, 1] = np.sum(conf_matrix.T * non_diagonal_mask, axis=1)
 
     # True Negatives
-    MCM[:, 0, 0] = num_samples - MCM[:, 1, 0] - MCM[:, 0, 1] - MCM[:, 1, 1]
+    MCM[:, 0, 0] = cast(
+        np.ndarray, num_samples - MCM[:, 1, 0] - MCM[:, 0, 1] - MCM[:, 1, 1]
+    )
 
     return MCM
 
@@ -210,6 +212,7 @@ def precision_recall_fscore_support(
     support: np.ndarray | None = true_sum
     if average == "weighted":
         weights = true_sum
+        assert weights is not None
         if weights.sum() == 0:
             return np.array([0.0]), np.array([0.0]), np.array([0.0]), None
     elif average == "samples":

dataprofiler/labelers/column_name_model.py

@@ -4,7 +4,7 @@
 
 import json
 import os
-from typing import Any, Callable
+from typing import Any, Callable, cast
 
 import numpy as np
 
@@ -176,7 +176,7 @@ def _reconstruct_model(self) -> None:
         pass
 
     def _need_to_reconstruct_model(self) -> bool:
-        pass
+        return False
 
     def reset_weights(self) -> None:
         """Reset weights function."""
@@ -191,15 +191,17 @@ def _model(
         scorer: Callable,
         include_label: bool = False,
     ) -> list:
-        scores = []
+        scores: list[list[float | int]] = []
 
         check_values_list = [dict["attribute"] for dict in check_values_dict]
 
         model_outputs = rapidfuzz.process.cdist(
             list_of_column_names, check_values_list, processor=processor, scorer=scorer
         )
+        model_outputs = cast(np.ndarray, model_outputs)
 
-        for iter_value, ngram_match_results in enumerate(model_outputs):
+        for i in range(len(model_outputs)):
+            ngram_match_results: np.ndarray = cast(np.ndarray, model_outputs[i])
             column_result = [np.max(ngram_match_results)]
             if include_label:
                 index_max_result = ngram_match_results.argmax(axis=0)

dataprofiler/labelers/data_labelers.py

@@ -12,7 +12,7 @@
 from .base_model import BaseModel
 from .data_processing import BaseDataPostprocessor, BaseDataPreprocessor
 
-default_labeler_dir = utils.find_resources_dir("labelers")
+default_labeler_dir = str(utils.find_resources_dir("labelers"))
 
 
 def train_structured_labeler(