diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c9f8adff3..8f2df6fcfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ This can also be enabled programmatically with `warnings.simplefilter('default', * support for SVG `` and `` elements - _cf._ [issue #1580](https://github.com/py-pdf/fpdf2/issues/1580) - thanks to @Ani07-05 * mypy and pyright checks in the CI pipeline to enforce strict typing * support WOFF and WOFF2 fonts - thanks to @BharathPESU +* Unicode font detection and enhanced error handling via `UnicodeFontManager` class for automatic font recommendations when working with non-Latin scripts (Cyrillic, Arabic, Chinese, etc.), providing helpful suggestions in encoding error messages - _cf._ [PR #1563](https://github.com/py-pdf/fpdf2/pull/1563) - thanks to @otrepid4github ### Fixed * the `A5` value that could be specified as page `format` to the `FPDF` constructor was slightly incorrect, and the corresponding page dimensions have been fixed. This could lead to a minor change in your documents dimensions if you used this `A5` page format. - _cf._ [issue #1699](https://github.com/py-pdf/fpdf2/issues/1699) * a bug when rendering empty tables with `INTERNAL` layout, that caused an extra border to be rendered due to an erroneous use of `list.index()` - _cf._ [issue #1669](https://github.com/py-pdf/fpdf2/issues/1669) diff --git a/fpdf/errors.py b/fpdf/errors.py index b571bdd3b4..9d55b6b660 100644 --- a/fpdf/errors.py +++ b/fpdf/errors.py @@ -35,22 +35,29 @@ def __str__(self) -> str: class FPDFUnicodeEncodingException(FPDFException): """Error is thrown when a character that cannot be encoded by the chosen encoder is provided""" - def __init__(self, text_index: int, character: str, font_name: str) -> None: + def __init__( + self, text_index: int, character: str, font_name: str, suggestion: str = None + ) -> None: super().__init__() self.text_index = text_index self.character = character self.font_name = font_name + self.suggestion = suggestion def __repr__(self) -> str: - return f"{self.__class__.__name__}({repr(self.text_index), repr(self.character), repr(self.font_name)})" + return f"{self.__class__.__name__}({repr(self.text_index)}, {repr(self.character)}, {repr(self.font_name)}, {repr(self.suggestion)})" def __str__(self) -> str: - return ( + base_message = ( f'Character "{self.character}" at index {self.text_index} in text is outside the range of characters' f' supported by the font used: "{self.font_name}".' - " Please consider using a Unicode font." ) + if self.suggestion: + return f"{base_message}\n\n{self.suggestion}" + else: + return f"{base_message} Please consider using a Unicode font." + class ComplianceError(FPDFException): """Base class for standards-compliance violations (PDF/A, PDF/X, etc.).""" diff --git a/fpdf/fpdf.py b/fpdf/fpdf.py index c09a2a87b6..66c038e4de 100644 --- a/fpdf/fpdf.py +++ b/fpdf/fpdf.py @@ -123,6 +123,7 @@ class Image: # type: ignore[no-redef] FPDFUnicodeEncodingException, PDFAComplianceError, ) +from .unicode_font_utils import suggest_unicode_font_for_error from .fonts import CORE_FONTS, CoreFont, FontFace, TextStyle, TitleStyle, TTFFont from .graphics_state import GraphicsStateMixin, StateStackType from .html import HTML2FPDF @@ -5639,10 +5640,13 @@ def normalize_text(self, text: str) -> str: try: return text.encode(self.core_fonts_encoding).decode("latin-1") except UnicodeEncodeError as error: + font_name = self.font_family + self.font_style + suggestion = suggest_unicode_font_for_error(text, font_name) raise FPDFUnicodeEncodingException( text_index=error.start, character=text[error.start], - font_name=self.font_family + self.font_style, + font_name=font_name, + suggestion=suggestion, ) from error return text diff --git a/fpdf/unicode_font_utils.py b/fpdf/unicode_font_utils.py new file mode 100644 index 0000000000..be48b94eb7 --- /dev/null +++ b/fpdf/unicode_font_utils.py @@ -0,0 +1,292 @@ +""" +Unicode font utilities for fpdf2. + +This module provides utilities for automatic Unicode font detection and management, +helping users work with non-Latin scripts like Cyrillic, Arabic, Chinese, etc. + +The contents of this module are internal to fpdf2, and not part of the public API. +They may change at any time without prior warning or any deprecation period, +in non-backward-compatible ways. +""" + +import os +import platform +from pathlib import Path +from typing import List, Optional, Tuple, Dict +import logging + +from .unicode_script import get_unicode_script, UnicodeScript + +LOGGER = logging.getLogger(__name__) + + +class UnicodeFontManager: + """ + Manages Unicode font detection and provides recommendations for different scripts. + """ + + def __init__(self): + self.system = platform.system().lower() + self.font_paths = self._get_system_font_paths() + self.available_fonts = self._scan_available_fonts() + + def _get_system_font_paths(self) -> List[Path]: + """Get common font paths for the current system.""" + paths = [] + + if self.system == "darwin": # macOS + paths.extend( + [ + Path("/System/Library/Fonts"), + Path("/Library/Fonts"), + Path.home() / "Library/Fonts", + ] + ) + elif self.system == "linux": + paths.extend( + [ + Path("/usr/share/fonts"), + Path("/usr/local/share/fonts"), + Path.home() / ".fonts", + Path.home() / ".local/share/fonts", + ] + ) + elif self.system == "windows": + paths.extend( + [ + Path("C:/Windows/Fonts"), + Path.home() / "AppData/Local/Microsoft/Windows/Fonts", + ] + ) + + return [p for p in paths if p.exists()] + + def _scan_available_fonts(self) -> Dict[str, Path]: + """Scan for available TrueType/OpenType fonts.""" + fonts = {} + + for font_dir in self.font_paths: + for font_file in font_dir.rglob("*.ttf"): + font_name = font_file.stem.lower() + if font_name not in fonts: # Prefer first found + fonts[font_name] = font_file + for font_file in font_dir.rglob("*.otf"): + font_name = font_file.stem.lower() + if font_name not in fonts: # Prefer first found + fonts[font_name] = font_file + for font_file in font_dir.rglob("*.ttc"): + font_name = font_file.stem.lower() + if font_name not in fonts: # Prefer first found + fonts[font_name] = font_file + + return fonts + + def get_recommended_fonts_for_script( + self, script: UnicodeScript + ) -> List[Tuple[str, str]]: + """ + Get recommended font names and their file paths for a specific Unicode script. + + Returns: + List of (font_name, file_path) tuples, ordered by preference. + """ + recommendations = [] + + # Define font recommendations by script + script_fonts = { + UnicodeScript.CYRILLIC: [ + "dejavusans", + "dejavuserif", + "dejavusanscondensed", + "arial", + "helvetica", + "liberationsans", + "liberationserif", + "notosans", + "notoserif", + "roboto", + "opensans", + ], + UnicodeScript.ARABIC: [ + "dejavusans", + "dejavuserif", + "notosansarabic", + "notoserifarabic", + "amiri", + "scheherazade", + "lateef", + ], + UnicodeScript.HAN: [ + "dejavusans", + "dejavuserif", + "notosanscjk", + "notoserifcjk", + "sourcehansans", + "sourcehanserif", + "fireflysung", + ], + UnicodeScript.HANGUL: [ + "dejavusans", + "dejavuserif", + "notosanskr", + "notoserifkr", + "nanumgothic", + "nanummyeongjo", + ], + UnicodeScript.HIRAGANA: [ + "dejavusans", + "dejavuserif", + "notosansjp", + "notoserifjp", + "sourcehansans", + "sourcehanserif", + ], + UnicodeScript.KATAKANA: [ + "dejavusans", + "dejavuserif", + "notosansjp", + "notoserifjp", + "sourcehansans", + "sourcehanserif", + ], + UnicodeScript.DEVANAGARI: [ + "dejavusans", + "dejavuserif", + "notosansdevanagari", + "notoserifdevanagari", + "gargi", + "lohitdevanagari", + ], + UnicodeScript.THAI: [ + "dejavusans", + "dejavuserif", + "notosansthai", + "notoserifthai", + "waree", + "garuda", + ], + UnicodeScript.HEBREW: [ + "dejavusans", + "dejavuserif", + "notosanshebrew", + "notoserifhebrew", + "frankruehl", + "david", + ], + } + + # Get fonts for the specific script + preferred_fonts = script_fonts.get(script, ["dejavusans", "dejavuserif"]) + + # Find available fonts from the preferred list + for font_name in preferred_fonts: + if font_name in self.available_fonts: + recommendations.append( + (font_name, str(self.available_fonts[font_name])) + ) + + # If no specific fonts found, recommend DejaVu fonts (most comprehensive Unicode support) + if not recommendations: + for fallback in ["dejavusans", "dejavuserif", "arial", "helvetica"]: + if fallback in self.available_fonts: + recommendations.append( + (fallback, str(self.available_fonts[fallback])) + ) + break + + return recommendations + + def detect_script_in_text(self, text: str) -> Optional[UnicodeScript]: + """ + Detect the primary Unicode script in the given text. + + Returns: + The most common non-Latin Unicode script in the text, or None if only Latin/Common scripts are found. + """ + script_counts = {} + + for char in text: + script = get_unicode_script(char) + if script != UnicodeScript.COMMON and script != UnicodeScript.LATIN: + script_counts[script] = script_counts.get(script, 0) + 1 + + if not script_counts: + return None + + return max(script_counts.items(), key=lambda x: x[1])[0] + + def get_font_recommendation_for_text(self, text: str) -> Optional[Tuple[str, str]]: + """ + Get a font recommendation for the given text based on its Unicode script. + + Returns: + (font_name, file_path) tuple for the recommended font, or None if no recommendation. + """ + script = self.detect_script_in_text(text) + if not script: + return None + + recommendations = self.get_recommended_fonts_for_script(script) + return recommendations[0] if recommendations else None + + def list_available_unicode_fonts(self) -> Dict[str, str]: + """ + List all available Unicode-capable fonts. + + Returns: + Dictionary mapping font names to file paths. + """ + return {name: str(path) for name, path in self.available_fonts.items()} + + +def get_unicode_font_recommendation(text: str) -> Optional[Tuple[str, str]]: + """ + Convenience function to get a Unicode font recommendation for text. + + Args: + text: The text to analyze for Unicode script detection. + + Returns: + (font_name, file_path) tuple for the recommended font, or None if no recommendation. + """ + manager = UnicodeFontManager() + return manager.get_font_recommendation_for_text(text) + + +def suggest_unicode_font_for_error(error_text: str, font_name: str) -> str: + """ + Generate a helpful error message suggesting Unicode fonts when encoding errors occur. + + Args: + error_text: The text that caused the encoding error. + font_name: The name of the font that failed. + + Returns: + A helpful error message with font suggestions. + """ + manager = UnicodeFontManager() + script = manager.detect_script_in_text(error_text) + + if not script: + return ( + f"The text contains characters that cannot be encoded with the '{font_name}' font. " + "Consider using a Unicode font like DejaVu Sans or Arial." + ) + + script_name = script.name.replace("_", " ").title() + recommendations = manager.get_recommended_fonts_for_script(script) + + if recommendations: + font_name_rec, font_path = recommendations[0] + message = ( + f"The text contains {script_name} characters that cannot be encoded with the '{font_name}' font. " + f"Consider using a Unicode font like '{font_name_rec}' instead.\n" + f"To use it, add the font with: pdf.add_font('{font_name_rec}', '', '{font_path}')" + ) + else: + message = ( + f"The text contains {script_name} characters that cannot be encoded with the '{font_name}' font. " + "Consider using a Unicode font like DejaVu Sans or Arial." + ) + + return message diff --git a/test/fonts/test_unicode_font_utils.py b/test/fonts/test_unicode_font_utils.py new file mode 100644 index 0000000000..736a6544cd --- /dev/null +++ b/test/fonts/test_unicode_font_utils.py @@ -0,0 +1,176 @@ +""" +Tests for Unicode font utilities. +""" + +import pytest +from pathlib import Path + +from fpdf.unicode_font_utils import ( + UnicodeFontManager, + get_unicode_font_recommendation, + suggest_unicode_font_for_error, +) +from fpdf.unicode_script import UnicodeScript + + +class TestUnicodeFontManager: + """Test the UnicodeFontManager class.""" + + def test_init(self): + """Test UnicodeFontManager initialization.""" + manager = UnicodeFontManager() + assert isinstance(manager.system, str) + assert isinstance(manager.font_paths, list) + assert isinstance(manager.available_fonts, dict) + + def test_detect_script_in_text_cyrillic(self): + """Test Cyrillic script detection.""" + manager = UnicodeFontManager() + + # Test Cyrillic text + cyrillic_text = "Привет, мир!" + script = manager.detect_script_in_text(cyrillic_text) + assert script == UnicodeScript.CYRILLIC + + # Test mixed text + mixed_text = "Hello Привет World" + script = manager.detect_script_in_text(mixed_text) + assert script == UnicodeScript.CYRILLIC # Should detect the most common non-Common script + + def test_detect_script_in_text_arabic(self): + """Test Arabic script detection.""" + manager = UnicodeFontManager() + + arabic_text = "مرحبا بالعالم" + script = manager.detect_script_in_text(arabic_text) + assert script == UnicodeScript.ARABIC + + def test_detect_script_in_text_latin_only(self): + """Test Latin-only text detection.""" + manager = UnicodeFontManager() + + latin_text = "Hello World" + script = manager.detect_script_in_text(latin_text) + assert script is None # Should return None for Latin-only text + + def test_get_recommended_fonts_for_script_cyrillic(self): + """Test font recommendations for Cyrillic script.""" + manager = UnicodeFontManager() + + recommendations = manager.get_recommended_fonts_for_script(UnicodeScript.CYRILLIC) + assert isinstance(recommendations, list) + + # Should return tuples of (font_name, file_path) + for font_name, file_path in recommendations: + assert isinstance(font_name, str) + assert isinstance(file_path, str) + + def test_get_font_recommendation_for_text(self): + """Test getting font recommendations for text.""" + manager = UnicodeFontManager() + + # Test with Cyrillic text + cyrillic_text = "Привет, мир!" + recommendation = manager.get_font_recommendation_for_text(cyrillic_text) + + if recommendation: # Only test if fonts are available + font_name, file_path = recommendation + assert isinstance(font_name, str) + assert isinstance(file_path, str) + assert Path(file_path).exists() + + def test_list_available_unicode_fonts(self): + """Test listing available Unicode fonts.""" + manager = UnicodeFontManager() + + fonts = manager.list_available_unicode_fonts() + assert isinstance(fonts, dict) + + # Check that all values are valid file paths + for font_name, file_path in fonts.items(): + assert isinstance(font_name, str) + assert isinstance(file_path, str) + assert Path(file_path).exists() + + +class TestConvenienceFunctions: + """Test convenience functions.""" + + def test_get_unicode_font_recommendation(self): + """Test the get_unicode_font_recommendation convenience function.""" + cyrillic_text = "Привет, мир!" + recommendation = get_unicode_font_recommendation(cyrillic_text) + + if recommendation: # Only test if fonts are available + font_name, file_path = recommendation + assert isinstance(font_name, str) + assert isinstance(file_path, str) + + def test_suggest_unicode_font_for_error_cyrillic(self): + """Test error suggestion for Cyrillic text.""" + error_text = "Привет, мир!" + font_name = "helvetica" + + suggestion = suggest_unicode_font_for_error(error_text, font_name) + assert isinstance(suggestion, str) + assert "Cyrillic" in suggestion + assert "helvetica" in suggestion + assert "Unicode font" in suggestion + + def test_suggest_unicode_font_for_error_arabic(self): + """Test error suggestion for Arabic text.""" + error_text = "مرحبا بالعالم" + font_name = "times" + + suggestion = suggest_unicode_font_for_error(error_text, font_name) + assert isinstance(suggestion, str) + assert "Arabic" in suggestion + assert "times" in suggestion + assert "Unicode font" in suggestion + + def test_suggest_unicode_font_for_error_no_script(self): + """Test error suggestion for text with no specific script.""" + error_text = "Hello World 123" + font_name = "courier" + + suggestion = suggest_unicode_font_for_error(error_text, font_name) + assert isinstance(suggestion, str) + assert "courier" in suggestion + assert "Unicode font" in suggestion + + +class TestIntegration: + """Integration tests with actual fpdf2 functionality.""" + + def test_unicode_font_manager_with_real_fonts(self): + """Test UnicodeFontManager with real system fonts.""" + manager = UnicodeFontManager() + + # This test will pass even if no fonts are found + # It just ensures the manager doesn't crash + fonts = manager.list_available_unicode_fonts() + assert isinstance(fonts, dict) + + # Test recommendations for different scripts + for script in [UnicodeScript.CYRILLIC, UnicodeScript.ARABIC, UnicodeScript.HAN]: + recommendations = manager.get_recommended_fonts_for_script(script) + assert isinstance(recommendations, list) + + def test_script_detection_accuracy(self): + """Test script detection accuracy with various texts.""" + manager = UnicodeFontManager() + + test_cases = [ + ("Привет, мир!", UnicodeScript.CYRILLIC), + ("مرحبا بالعالم", UnicodeScript.ARABIC), + ("你好世界", UnicodeScript.HAN), + ("안녕하세요", UnicodeScript.HANGUL), + ("こんにちは世界", UnicodeScript.HIRAGANA), + ("नमस्ते दुनिया", UnicodeScript.DEVANAGARI), + ("สวัสดีชาวโลก", UnicodeScript.THAI), + ("שלום עולם", UnicodeScript.HEBREW), + ] + + for text, expected_script in test_cases: + detected_script = manager.detect_script_in_text(text) + assert detected_script == expected_script, f"Failed for text: {text}" diff --git a/tutorial/unicode.py b/tutorial/unicode.py index fb4891f9b2..bb016faed2 100755 --- a/tutorial/unicode.py +++ b/tutorial/unicode.py @@ -2,6 +2,9 @@ from fpdf import FPDF +# Note: For automatic Unicode font detection and recommendations, +# see the new tutorial: unicode_font_detection.py + pdf = FPDF() pdf.add_page() diff --git a/tutorial/unicode_font_detection.py b/tutorial/unicode_font_detection.py new file mode 100644 index 0000000000..3206e80937 --- /dev/null +++ b/tutorial/unicode_font_detection.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +""" +Unicode Font Detection and Management Tutorial + +This tutorial demonstrates how to use fpdf2's new Unicode font detection +and management features to automatically handle different scripts like +Cyrillic, Arabic, Chinese, etc. +""" + +from fpdf import FPDF +from fpdf.unicode_font_utils import ( + UnicodeFontManager, + get_unicode_font_recommendation, + suggest_unicode_font_for_error, +) +from fpdf.unicode_script import UnicodeScript + +def demonstrate_unicode_font_detection(): + """Demonstrate automatic Unicode font detection and recommendations.""" + + print("=== Unicode Font Detection Demo ===\n") + + # Initialize the font manager + manager = UnicodeFontManager() + + # List available fonts + print("Available Unicode fonts on this system:") + available_fonts = manager.list_available_unicode_fonts() + for font_name, font_path in list(available_fonts.items())[:5]: # Show first 5 + print(f" - {font_name}: {font_path}") + if len(available_fonts) > 5: + print(f" ... and {len(available_fonts) - 5} more fonts") + print() + + # Test different scripts + test_texts = [ + ("English", "Hello World"), + ("Russian (Cyrillic)", "Привет, мир!"), + ("Arabic", "مرحبا بالعالم"), + ("Chinese (Han)", "你好世界"), + ("Korean (Hangul)", "안녕하세요"), + ("Japanese (Hiragana)", "こんにちは世界"), + ("Hindi (Devanagari)", "नमस्ते दुनिया"), + ("Thai", "สวัสดีชาวโลก"), + ("Hebrew", "שלום עולם"), + ] + + for script_name, text in test_texts: + print(f"=== {script_name} ===") + print(f"Text: {text}") + + # Detect script + detected_script = manager.detect_script_in_text(text) + if detected_script: + print(f"Detected script: {detected_script.name}") + + # Get font recommendations + recommendations = manager.get_recommended_fonts_for_script(detected_script) + if recommendations: + print("Recommended fonts:") + for i, (font_name, font_path) in enumerate(recommendations[:3], 1): + print(f" {i}. {font_name}") + else: + print("No specific font recommendations available") + else: + print("No specific script detected (likely Latin/Common)") + + print() + +def demonstrate_error_handling(): + """Demonstrate improved error handling with font suggestions.""" + + print("=== Error Handling Demo ===\n") + + # Simulate the error that would occur with Cyrillic text + error_text = "ul. Zapadna obikolna, sgradа 8" + font_name = "helvetica" + + print(f"Text causing error: {error_text}") + print(f"Font that failed: {font_name}") + print() + + # Get helpful error suggestion + suggestion = suggest_unicode_font_for_error(error_text, font_name) + print("Helpful error message:") + print(suggestion) + print() + +def create_multilingual_pdf(): + """Create a PDF with multiple scripts using automatic font detection.""" + + print("=== Creating Multilingual PDF ===\n") + + pdf = FPDF() + pdf.add_page() + + # Initialize font manager + manager = UnicodeFontManager() + + # Test texts in different scripts + test_texts = [ + ("English", "Hello World - This is English text"), + ("Russian (Cyrillic)", "Привет, мир! - Это русский текст"), + ("Arabic", "مرحبا بالعالم - هذا نص عربي"), + ("Chinese (Han)", "你好世界 - 这是中文文本"), + ("Korean (Hangul)", "안녕하세요 - 이것은 한국어 텍스트입니다"), + ("Japanese (Hiragana)", "こんにちは世界 - これは日本語のテキストです"), + ("Hindi (Devanagari)", "नमस्ते दुनिया - यह हिंदी पाठ है"), + ("Thai", "สวัสดีชาวโลก - นี่คือข้อความภาษาไทย"), + ("Hebrew", "שלום עולם - זה טקסט בעברית"), + ] + + y_position = 20 + + for script_name, text in test_texts: + # Get font recommendation for this text + recommendation = manager.get_font_recommendation_for_text(text) + + if recommendation: + font_name, font_path = recommendation + try: + # Add the recommended font + pdf.add_font(font_name, '', font_path) + pdf.set_font(font_name, size=12) + + # Add text + pdf.set_y(y_position) + pdf.cell(0, 8, f"{script_name}: {text}", new_x="LMARGIN", new_y="NEXT") + + print(f"✓ {script_name}: Using {font_name}") + y_position += 10 + + except Exception as e: + # Fallback to default font + pdf.set_font("helvetica", size=12) + pdf.set_y(y_position) + pdf.cell(0, 8, f"{script_name}: [Font not available] {text}", new_x="LMARGIN", new_y="NEXT") + print(f"✗ {script_name}: Font not available ({e})") + y_position += 10 + else: + # Use default font for Latin text + pdf.set_font("helvetica", size=12) + pdf.set_y(y_position) + pdf.cell(0, 8, f"{script_name}: {text}", new_x="LMARGIN", new_y="NEXT") + print(f"• {script_name}: Using default font") + y_position += 10 + + # Save the PDF + filename = "multilingual_unicode_demo.pdf" + pdf.output(filename) + print(f"\nPDF saved as: {filename}") + +def demonstrate_convenience_function(): + """Demonstrate the convenience function for quick font recommendations.""" + + print("=== Convenience Function Demo ===\n") + + # Test the convenience function + test_texts = [ + "Привет, мир!", + "مرحبا بالعالم", + "你好世界", + "Hello World", + ] + + for text in test_texts: + recommendation = get_unicode_font_recommendation(text) + if recommendation: + font_name, font_path = recommendation + print(f"Text: {text}") + print(f"Recommended font: {font_name}") + print(f"Font path: {font_path}") + else: + print(f"Text: {text}") + print("No specific recommendation (likely Latin text)") + print() + +if __name__ == "__main__": + print("fpdf2 Unicode Font Detection Tutorial") + print("=" * 50) + print() + + try: + # Run demonstrations + demonstrate_unicode_font_detection() + demonstrate_error_handling() + demonstrate_convenience_function() + create_multilingual_pdf() + + print("Tutorial completed successfully!") + + except Exception as e: + print(f"Error during tutorial: {e}") + print("This might be due to missing fonts on your system.") + print("The functionality will work when appropriate fonts are available.")