From 39b315d702c88ea2ed2e6dcbadd1c70dc0cd0918 Mon Sep 17 00:00:00 2001 From: VooDisss Date: Mon, 18 Aug 2025 11:49:02 +0300 Subject: [PATCH] fix(data): Handle missing papers.jsonl file This commit fixes a bug where the data processing pipeline would crash if the `data/papers.jsonl` file was missing or empty. The `PaperLoader` class in `peerqa/data_loader.py` would unconditionally try to read `papers.jsonl` at initialization, causing a `ValueError` if the file didn't exist. This would prevent the `extract_text_from_pdf.py` script from running and creating the file in the first place. This commit makes the `PaperLoader` more robust by: - Checking if `papers.jsonl` exists and is not empty before reading it. - Initializing an empty DataFrame if the file is missing, allowing the script to proceed. - Adding a safeguard to `has_paper_id` to handle an empty DataFrame. This ensures that the data processing pipeline can be run from a clean state without errors. --- peerqa/data_loader.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/peerqa/data_loader.py b/peerqa/data_loader.py index 8ac3457..03c688b 100644 --- a/peerqa/data_loader.py +++ b/peerqa/data_loader.py @@ -1,17 +1,22 @@ from typing import Iterator, Literal, Union +from pathlib import Path import pandas as pd from tqdm.auto import tqdm class PaperLoader: - def __init__(self, papers_file: str): - self.df = pd.read_json(papers_file, lines=True) + def __init__(self, papers_file: Union[str, Path]): + papers_file = Path(papers_file) + if papers_file.exists() and papers_file.stat().st_size > 0: + self.df = pd.read_json(papers_file, lines=True) + else: + self.df = pd.DataFrame({"paper_id": []}) def __call__( self, granularity: Literal["sentences", "paragraphs"], - template: str = None, + template: Union[str, None] = None, show_progress: bool = True, ) -> Iterator[tuple[str, list[str], list[str]]]: """Yields a the paper_id, sentence/paragraph index and sentence/paragraph.""" @@ -85,6 +90,8 @@ def get_title(self, paper_id: str) -> str: return title def has_paper_id(self, paper_id: str) -> bool: + if self.df.empty: + return False return paper_id in self.df.paper_id.unique()