diff --git a/article/models.py b/article/models.py index 85a4e434d..f53571ecd 100755 --- a/article/models.py +++ b/article/models.py @@ -772,12 +772,10 @@ def check_availability(self, user, force_update=False): if not force_update and self.is_available(): return True - event = None urls = [] for item in self.article_availability.all(): urls.append(item.url) - event = self.add_event(user, _("register urls")) for item in self.urls_data: if item["url"] in urls: urls.remove(item["url"]) @@ -794,17 +792,15 @@ def check_availability(self, user, force_update=False): return self.mark_as_available() except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() - if event: - event.finish(completed=False, exceptions=traceback.format_exc()) - else: - UnexpectedEvent.create( - item=str(self), - exception=e, - exc_traceback=exc_traceback, - detail=dict( - function="article.models.Article.check_availability", - ), - ) + UnexpectedEvent.create( + action="article.models.Article.check_availability", + item=str(self), + exception=e, + exc_traceback=exc_traceback, + detail={ + "traceback": traceback.format_exc() + }, + ) def mark_as_available(self): save = False diff --git a/article/sources/preprint.py b/article/sources/preprint.py index 318e3c2a9..9ae75908e 100644 --- a/article/sources/preprint.py +++ b/article/sources/preprint.py @@ -41,9 +41,7 @@ def harvest_preprints(URL, user): # Clear existing contrib_persons to avoid duplication on reharvest article.contrib_persons.all().delete() get_or_create_contrib_persons( - article=article, - user=user, - authors=article_info.get("authors") + article=article, user=user, authors=article_info.get("authors") ) article.keywords.set( get_or_create_keyword(keywords=article_info.get("subject"), user=user) @@ -262,10 +260,10 @@ def set_dates(article, date): def get_or_create_contrib_persons(article, user, authors): """ Create or update ContribPerson objects for preprint authors. - - Note: In preprint processing, only basic name fields (given_names, surname, - declared_name) are currently extracted from the OAI-DC format. Affiliation - data is not available in the preprint metadata structure, so affiliation + + Note: In preprint processing, only basic name fields (given_names, surname, + declared_name) are currently extracted from the OAI-DC format. Affiliation + data is not available in the preprint metadata structure, so affiliation is set to None. """ data = [] diff --git a/article/sources/xmlsps.py b/article/sources/xmlsps.py index 24624220e..be0530988 100755 --- a/article/sources/xmlsps.py +++ b/article/sources/xmlsps.py @@ -1,11 +1,8 @@ -import logging -import sys import traceback from datetime import datetime from itertools import product from django.utils.translation import gettext_lazy as _ -from lxml import etree from packtools.sps.models.article_abstract import ArticleAbstract from packtools.sps.models.article_and_subarticles import ArticleAndSubArticles from packtools.sps.models.article_contribs import ArticleContribs, XMLContribs @@ -40,12 +37,9 @@ from issue.models import Issue, TableOfContents, AMIssue from issue.articlemeta.loader import load_issue_sections from journal.models import Journal -from location.models import Location -from pid_provider.choices import PPXML_STATUS_UNMATCHED_JOURNAL_OR_ISSUE, PPXML_STATUS_INVALID -from pid_provider.models import PidProviderXML + # Researcher no longer used - replaced by ContribPerson # from researcher.models import Affiliation, Researcher -from tracker.models import UnexpectedEvent from vocabulary.models import Keyword @@ -90,61 +84,44 @@ def load_article(user, pp_xml): ou se o usuário não for informado Note: - - Erros durante o processamento são coletados em article.errors + - Erros durante o processamento são coletados em `errors` + - Mensagens de progresso são coletadas em `messages` e passadas + como `detail` para `event.finish(...)`, que as persiste no evento - O processamento continua mesmo com falhas parciais - O campo article.valid indica se o processamento foi completo """ - logging.info(f"load article {pp_xml}") - detail = {"pp_xml": str(pp_xml)} + article = None + messages = [f"load article {pp_xml}"] + errors = [] # Validações iniciais if not user: raise ValueError("User is required") if not pp_xml: - raise ValueError( - "load_article() requires params: pp_xml" + raise ValueError("load_article() requires params: pp_xml") + + xml_with_pre = pp_xml.xml_with_pre + if not xml_with_pre: + Article.objects.filter(pp_xml=pp_xml).exclude( + data_status=choices.DATA_STATUS_INVALID, + ).update( + data_status=choices.DATA_STATUS_INVALID, ) + raise ValueError(f"Unable to get XML to load article from {pp_xml}") try: - xml_with_pre = pp_xml.xml_with_pre - except Exception as e: - updated = ( - Article.objects.filter(pp_xml=pp_xml) - .exclude( - data_status=choices.DATA_STATUS_INVALID, - ) - .update( - data_status=choices.DATA_STATUS_INVALID, - ) - ) - errors = [ - { - "function": "load_article", - "error_type": e.__class__.__name__, - "error_message": str(e), - "timestamp": datetime.now().isoformat(), - } - ] - pp_xml.add_event(name="load_article", proc_status=PPXML_STATUS_INVALID, detail=detail, errors=errors, exceptions=e) - raise ValueError(f"Unable to get XML to load article from {pp_xml}: {e}") - - - try: - errors = [] - article = None - event = None - xmltree = xml_with_pre.xmltree - pid_v3 = xml_with_pre.v3 sps_pkg_name = xml_with_pre.sps_pkg_name - logging.info(f"Pid Provider XML: {pid_v3} {sps_pkg_name}") - + messages.append(f"Pid Provider XML: {pid_v3} {sps_pkg_name}") + journal = get_journal(xmltree=xmltree, errors=errors) if not journal: - raise ValueError(f"Not found journal for pid provider xml: {pid_v3} {sps_pkg_name}") + raise ValueError( + f"Not found journal for pid provider xml: {pid_v3} {sps_pkg_name}" + ) issue = get_issue( xmltree=xmltree, journal=journal, @@ -152,22 +129,30 @@ def load_article(user, pp_xml): errors=errors, ) if not issue: - raise ValueError(f"Not found issue for pid provider xml: {pid_v3} {sps_pkg_name}") + raise ValueError( + f"Not found issue for pid provider xml: {pid_v3} {sps_pkg_name}" + ) # CRIAÇÃO/OBTENÇÃO DO OBJETO PRINCIPAL - article = Article.create_or_update( - user=user, - pid_v3=pid_v3, - sps_pkg_name=sps_pkg_name, - ) - logging.info(f"...Article {pid_v3} {sps_pkg_name}") - - article.events.all().delete() - event = article.add_event(user, _("load article")) + try: + article = Article.objects.get( + pp_xml=pp_xml, + ) + except Article.MultipleObjectsReturned: + article = Article.objects.filter( + pp_xml=pp_xml, + ).order_by("-updated").first() + except Article.DoesNotExist: + article = Article.create_or_update( + user=user, + pid_v3=pid_v3, + sps_pkg_name=sps_pkg_name, + ) + messages.append(f"...Article {pid_v3} {sps_pkg_name}") # Configurar todos os campos antes de salvar (Sugestão 9) article.valid = False - article.data_status = choices.DATA_STATUS_PENDING + # article.data_status = choices.DATA_STATUS_PENDING article.pp_xml = pp_xml article.sps_pkg_name = sps_pkg_name @@ -181,17 +166,14 @@ def load_article(user, pp_xml): article.article_type = get_or_create_article_type( xmltree=xmltree, user=user, errors=errors ) - add_peer_review_dates( - xmltree=xmltree, article=article, errors=errors - ) + add_peer_review_dates(xmltree=xmltree, article=article, errors=errors) # FOREIGN KEYS SIMPLES article.journal = journal article.issue = issue article.save() - # Salvar uma vez após definir todos os campos simples - logging.info( + messages.append( f"Saving article {article.pid_v3} {sps_pkg_name} {xml_with_pre.main_doi}" ) @@ -207,7 +189,9 @@ def load_article(user, pp_xml): article.languages.add(main_lang) article.sections.set( - get_or_create_toc_sections(xmltree=xmltree, user=user, errors=errors, issue=article.issue) + get_or_create_toc_sections( + xmltree=xmltree, user=user, errors=errors, issue=article.issue + ) ) article.titles.set( create_or_update_titles( @@ -224,8 +208,6 @@ def load_article(user, pp_xml): xmltree=xmltree, user=user, item=pid_v3, errors=errors ) ) - # Create contrib_persons (replaces researchers) - # Clear existing contrib_persons to avoid duplication on reload article.contrib_persons.all().delete() create_or_update_contrib_persons( xmltree=xmltree, article=article, user=user, item=pid_v3, errors=errors @@ -240,31 +222,36 @@ def load_article(user, pp_xml): ) article.doi.set(get_or_create_doi(xmltree=xmltree, user=user, errors=errors)) - # Adicionar artigos relacionados add_related_articles(xmltree=xmltree, article=article, user=user, errors=errors) article.create_legacy_keys(user) if not article.pid_v2: raise ValueError(f"Article has no PID v2: {article.pid_v3}") - if not errors: - article.mark_as_completed() - event.finish(completed=not errors, errors=errors) - logging.info( - f"The article {pid_v3} has been processed with {len(errors)} errors" - ) - return article + return finish(article, errors, messages) except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() add_error(errors, "load_article", e) + finish(article, errors, messages) + raise - if event: - event.finish(errors=errors, exceptions=traceback.format_exc()) - raise - pp_xml.add_event(name="load_article", proc_status=PPXML_STATUS_UNMATCHED_JOURNAL_OR_ISSUE, detail=detail, errors=errors, exceptions=e) - - raise +def finish(article, errors, messages): + if not article: + return + + detail = {} + if errors: + detail["errors"] = errors + if messages: + detail["messages"] = messages + + # atualmente o nome do campo é errors, mas reusá-lo para outros detalhes + article.errors = detail + if errors: + article.save() + else: + article.mark_as_completed() + return article def add_peer_review_dates(xmltree, article, errors): @@ -291,18 +278,38 @@ def add_peer_review_dates(xmltree, article, errors): article.accepted_dateiso = peer_review_stats.get("accepted_date") # Extrair intervalos em dias - article.days_preprint_to_received = peer_review_stats.get("days_from_preprint_to_received") - article.days_received_to_accepted = peer_review_stats.get("days_from_received_to_accepted") - article.days_accepted_to_published = peer_review_stats.get("days_from_accepted_to_published") - article.days_preprint_to_published = peer_review_stats.get("days_from_preprint_to_published") - article.days_receive_to_published = peer_review_stats.get("days_from_received_to_published") + article.days_preprint_to_received = peer_review_stats.get( + "days_from_preprint_to_received" + ) + article.days_received_to_accepted = peer_review_stats.get( + "days_from_received_to_accepted" + ) + article.days_accepted_to_published = peer_review_stats.get( + "days_from_accepted_to_published" + ) + article.days_preprint_to_published = peer_review_stats.get( + "days_from_preprint_to_published" + ) + article.days_receive_to_published = peer_review_stats.get( + "days_from_received_to_published" + ) # Extrair flags de estimativa - article.days_preprint_to_received_estimated = peer_review_stats.get("estimated_days_from_preprint_to_received") - article.days_received_to_accepted_estimated = peer_review_stats.get("estimated_days_from_received_to_accepted") - article.days_accepted_to_published_estimated = peer_review_stats.get("estimated_days_from_accepted_to_published") - article.days_preprint_to_published_estimated = peer_review_stats.get("estimated_days_from_preprint_to_published") - article.days_receive_to_published_estimated = peer_review_stats.get("estimated_days_from_received_to_published") + article.days_preprint_to_received_estimated = peer_review_stats.get( + "estimated_days_from_preprint_to_received" + ) + article.days_received_to_accepted_estimated = peer_review_stats.get( + "estimated_days_from_received_to_accepted" + ) + article.days_accepted_to_published_estimated = peer_review_stats.get( + "estimated_days_from_accepted_to_published" + ) + article.days_preprint_to_published_estimated = peer_review_stats.get( + "estimated_days_from_preprint_to_published" + ) + article.days_receive_to_published_estimated = peer_review_stats.get( + "estimated_days_from_received_to_published" + ) except Exception as e: add_error(errors, "add_peer_review_dates", e) @@ -356,9 +363,7 @@ def add_data_availability_status(xmltree, errors, article, user): for item in items: DataAvailabilityStatement.create_or_update( - user=user, - article=article, - **item + user=user, article=article, **item ) except Exception as e: add_error(errors, "add_data_availability_status", e) @@ -510,9 +515,13 @@ def get_or_create_toc_sections(xmltree, user, errors, issue): if not section_title: continue try: - issue_sections = TableOfContents.get_items_by_title(issue=issue, title=section_title) + issue_sections = TableOfContents.get_items_by_title( + issue=issue, title=section_title + ) if not issue_sections.exists(): - raise TableOfContents.DoesNotExist(f"Unable to find TOC section {section_title} for issue {issue}") + raise TableOfContents.DoesNotExist( + f"Unable to find TOC section {section_title} for issue {issue}" + ) for obj in issue_sections: data.append(obj) except Exception as e: @@ -687,9 +696,9 @@ def create_or_update_contrib_persons(xmltree, article, user, item, errors): ) data.append(obj) else: - # When an author has multiple affiliations in XML, we create one - # ContribPerson record per affiliation. This is intentional as per - # SciELO's data model where each author-affiliation combination + # When an author has multiple affiliations in XML, we create one + # ContribPerson record per affiliation. This is intentional as per + # SciELO's data model where each author-affiliation combination # should be tracked separately. for aff in affs: raw_email = author.get("email") or aff.get("email") @@ -1042,7 +1051,7 @@ def add_related_articles(xmltree, article, user, errors): user=user, href=href, ext_link_type=ext_link_type, - related_type=related_type + related_type=related_type, ) except Exception as e: @@ -1050,7 +1059,7 @@ def add_related_articles(xmltree, article, user, errors): errors, "add_related_articles.process_item", e, - related_article_data=related_article_data + related_article_data=related_article_data, ) except Exception as e: diff --git a/article/tasks.py b/article/tasks.py index 4905ca045..c58a1e9fb 100644 --- a/article/tasks.py +++ b/article/tasks.py @@ -485,6 +485,7 @@ def task_export_article_to_articlemeta( - Requer que o artigo exista na base local antes da exportação """ try: + item = pid_v3 if not pid_v3: raise ValueError("task_export_article_to_articlemeta requires pid_v3") @@ -493,6 +494,7 @@ def task_export_article_to_articlemeta( valid=True, is_classic_public=True, ) + item = str(article) user = _get_user(self.request, username=username, user_id=user_id) @@ -507,10 +509,12 @@ def task_export_article_to_articlemeta( except Exception as exception: exc_type, exc_value, exc_traceback = sys.exc_info() UnexpectedEvent.create( + action="article.tasks.task_export_article_to_articlemeta", + item=item, exception=exception, exc_traceback=exc_traceback, detail={ - "task": "article.tasks.task_export_article_to_articlemeta", + "collection_acron_list": collection_acron_list, "pid_v3": pid_v3, "force_update": force_update, }, @@ -919,9 +923,10 @@ def task_process_article_pipeline( ) """ try: + unexpected_event_item = None user = _get_user(self.request, username=username, user_id=user_id) - if xml_url: + unexpected_event_item = xml_url if not collection_acron: raise ValueError("collection_acron is required when xml_url is provided") if not pid: @@ -946,6 +951,7 @@ def task_process_article_pipeline( if article_source_id: article_source = ArticleSource.objects.get(id=article_source_id) + unexpected_event_item = str(article_source) article_source.add_pid_provider( user=user, force_update=force_update, @@ -962,8 +968,11 @@ def task_process_article_pipeline( pp_xml = PidProviderXML.objects.select_related( "current_version" ).get(id=pp_xml_id) + unexpected_event_item = str(pp_xml) article = load_article(user, pp_xml=pp_xml) + unexpected_event_item = str(article) + pp_xml.collections.set(article.collections) article.check_availability(user, force_update=export_to_articlemeta or force_update) @@ -982,15 +991,20 @@ def task_process_article_pipeline( except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() UnexpectedEvent.create( + action="article.tasks.task_process_article_pipeline", + item=unexpected_event_item, exception=e, exc_traceback=exc_traceback, detail={ - "task": "article.tasks.task_process_article_pipeline", "xml_url": xml_url, "article_source_id": article_source_id, "pp_xml_id": pp_xml_id, "pid": pid, "collection_acron": collection_acron, + "source_date": source_date, + "collection_acron_list": collection_acron_list, + "auto_solve_pid_conflict": auto_solve_pid_conflict, + "version": version, "export_to_articlemeta": export_to_articlemeta, "force_update": force_update, },