diff --git a/pid_provider/migrations/0014_pidproviderxml_readable_data_and_more.py b/pid_provider/migrations/0014_pidproviderxml_readable_data_and_more.py new file mode 100644 index 000000000..8e67f8451 --- /dev/null +++ b/pid_provider/migrations/0014_pidproviderxml_readable_data_and_more.py @@ -0,0 +1,144 @@ +# Generated by Django 5.2.3 on 2026-06-27 16:16 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("pid_provider", "0013_alter_xmlurl_status"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="pidproviderxml", + name="readable_data", + field=models.JSONField(blank=True, null=True, verbose_name="Readable data"), + ), + migrations.CreateModel( + name="PidProviderXMLRegistration", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + auto_now_add=True, verbose_name="Creation date" + ), + ), + ( + "updated", + models.DateTimeField( + auto_now=True, verbose_name="Last update date" + ), + ), + ( + "pkg_name", + models.CharField( + blank=True, + max_length=100, + null=True, + verbose_name="Package name", + ), + ), + ( + "event_status", + models.CharField( + blank=True, + choices=[ + ("created", "created"), + ("updated", "updated"), + ("skipped", "skipped"), + ("forbidden", "forbidden"), + ("conflict", "conflict"), + ("unmatched", "unmatched"), + ("bad_request", "bad_request"), + ("error", "error"), + ], + max_length=15, + null=True, + verbose_name="Event status", + ), + ), + ( + "input_data", + models.JSONField( + blank=True, null=True, verbose_name="Input data (readable)" + ), + ), + ( + "best_matches", + models.JSONField( + blank=True, null=True, verbose_name="Best matches detail" + ), + ), + ( + "error_type", + models.CharField( + blank=True, max_length=255, null=True, verbose_name="Error type" + ), + ), + ( + "creator", + models.ForeignKey( + editable=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_creator", + to=settings.AUTH_USER_MODEL, + verbose_name="Creator", + ), + ), + ( + "pid_provider_xml", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="registration_events", + to="pid_provider.pidproviderxml", + ), + ), + ( + "updated_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_last_mod_user", + to=settings.AUTH_USER_MODEL, + verbose_name="Updater", + ), + ), + ], + options={ + "verbose_name": "PidProviderXML Registration", + "verbose_name_plural": "PidProviderXML Registrations", + "ordering": ["-created"], + "indexes": [ + models.Index( + fields=["pkg_name"], name="pid_provide_pkg_nam_2db0b2_idx" + ), + models.Index( + fields=["event_status"], name="pid_provide_event_s_3c9ae7_idx" + ), + models.Index( + fields=["-created"], name="pid_provide_created_94fb08_idx" + ), + models.Index( + fields=["pid_provider_xml"], + name="pid_provide_pid_pro_c9fb0e_idx", + ), + ], + }, + ), + ] diff --git a/pid_provider/models.py b/pid_provider/models.py index c46016e61..8689b0d72 100644 --- a/pid_provider/models.py +++ b/pid_provider/models.py @@ -9,6 +9,7 @@ from zlib import crc32 from django.core.files.base import ContentFile +from django.core.exceptions import FieldError from django.db import IntegrityError, models from django.db.models import Q, Count from django.utils.translation import gettext_lazy as _ @@ -29,14 +30,16 @@ profile_property, profile_staticmethod, ) -from core.utils.similarity import how_similar from pid_provider import choices, exceptions from pid_provider.query_params import ( - get_score, + compare, zero_to_none, QueryBuilderPidProviderXML, ) -from tracker.models import BaseEvent, EventSaveError, UnexpectedEvent +from tracker.models import UnexpectedEvent + + +PARTIAL_BODY_MAX = 300 try: from django_prometheus.models import ExportModelOperationsMixin @@ -100,26 +103,52 @@ def xml_directory_path(instance, filename): return f"pid_provider/{subdir_sps_pkg_name}/{filename}" +def get_collection_from_article_id_authority(article_id_authority): + article_id_authority = (article_id_authority or "").split("-")[-1] + if article_id_authority: + try: + return Collection.objects.get( + acronym__iexact=article_id_authority.lower() + ) + except Collection.DoesNotExist: + return None + return None + + class XMLVersion(CommonControlField): """ Tem função de guardar a versão do XML """ - pid_provider_xml = models.ForeignKey( "PidProviderXML", null=True, blank=True, on_delete=models.SET_NULL ) file = models.FileField(upload_to=xml_directory_path, null=True, blank=True, max_length=300) finger_print = models.CharField(max_length=64, null=True, blank=True) + body_fragment_fingerprint = models.CharField(_("body fragment fingerprint"), max_length=300, null=True, blank=True) + + pkg_name = models.CharField(_("Package name"), max_length=100, null=True, blank=True) + pid_v3 = models.CharField(_("pid_v3"), max_length=23, null=True, blank=True) + pid_v2 = models.CharField(_("pid_v2"), max_length=24, null=True, blank=True) + aop_pid = models.CharField(_("AOP PID"), max_length=64, null=True, blank=True) + collection = models.ForeignKey("Collection", null=True, blank=True, on_delete=models.SET_NULL) class Meta: ordering = ["-created"] indexes = [ models.Index(fields=["pid_provider_xml"]), models.Index(fields=["finger_print"]), + models.Index(fields=["pid_v2"]), + models.Index(fields=["collection"]), + models.Index(fields=["pid_v3"]), + models.Index(fields=["aop_pid"]), + models.Index(fields=["pkg_name"]), + models.Index(fields=["body_fragment_fingerprint"]), ] def __str__(self): - return f"{self.pid_provider_xml.pkg_name} {self.created}" + if self.collection and self.pkg_name and self.created: + return f"{self.collection} {self.pkg_name} {self.created}" + return f"{self.pid_provider_xml} {self.created}" @classmethod @profile_classmethod @@ -128,11 +157,21 @@ def create( user, pid_provider_xml, xml_with_pre, + collection, ): + finger_print = xml_with_pre.finger_print + body_fragment_fingerprint = xml_with_pre.body_fragment_fingerprint try: obj = cls() obj.pid_provider_xml = pid_provider_xml - obj.finger_print = xml_with_pre.finger_print + obj.collection = collection + obj.finger_print = finger_print + obj.body_fragment_fingerprint = body_fragment_fingerprint + obj.pid_v3 = xml_with_pre.v3 + obj.pid_v2 = xml_with_pre.v2 + obj.collection = get_collection_from_article_id_authority(xml_with_pre.v2_authority) + obj.aop_pid = xml_with_pre.aop_pid + obj.pkg_name = xml_with_pre.pkg_name obj.creator = user obj.save() obj.save_file( @@ -141,7 +180,7 @@ def create( obj.save() return obj except IntegrityError: - return cls.get(pid_provider_xml, xml_with_pre.finger_print) + return cls.get(pid_provider_xml, finger_print, body_fragment_fingerprint) def save_file(self, filename, content): try: @@ -178,26 +217,31 @@ def xml(self): @classmethod @profile_classmethod - def get(cls, pid_provider_xml, finger_print): + def get(cls, pid_provider_xml, finger_print=None, body_fragment_fingerprint=None): """ Retorna última versão se finger_print corresponde """ - if not pid_provider_xml or not finger_print: + if not pid_provider_xml or not (body_fragment_fingerprint or finger_print): raise XMLVersionGetError( "XMLVersion.get requires pid_provider_xml and xml_with_pre parameters" ) - found = cls.objects.filter( - pid_provider_xml=pid_provider_xml, finger_print=finger_print + # .latest() já levanta DoesNotExist se vazio + if body_fragment_fingerprint: + return cls.objects.filter( + pid_provider_xml=pid_provider_xml, body_fragment_fingerprint=body_fragment_fingerprint + ).latest("created") + + return cls.objects.filter( + pid_provider_xml=pid_provider_xml, finger_print=finger_print, ).latest("created") - if found: - return found - raise cls.DoesNotExist(f"{pid_provider_xml} {finger_print}") @classmethod @profile_classmethod def get_or_create(cls, user, pid_provider_xml, xml_with_pre): try: - latest = cls.get(pid_provider_xml, xml_with_pre.finger_print) + finger_print = xml_with_pre.finger_print + body_fragment_fingerprint = xml_with_pre.body_fragment_fingerprint + latest = cls.get(pid_provider_xml, finger_print, body_fragment_fingerprint) try: file_exist = os.path.isfile(latest.file.path) except (AttributeError, TypeError, ValueError) as e: @@ -215,6 +259,7 @@ def get_or_create(cls, user, pid_provider_xml, xml_with_pre): user=user, pid_provider_xml=pid_provider_xml, xml_with_pre=xml_with_pre, + collection=get_collection_from_article_id_authority(xml_with_pre.v2_authority) ) @@ -394,7 +439,7 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): z_links = models.CharField(_("links"), max_length=64, null=True, blank=True) z_partial_body = models.CharField( _("partial_body"), max_length=64, null=True, blank=True - ) + ) # data de atualização / criação do registro fonte origin_date = models.CharField( _("Origin date"), max_length=10, null=True, blank=True @@ -407,6 +452,11 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): other_pid_count = models.PositiveIntegerField(default=0) registered_in_core = models.BooleanField(default=False) collections = models.ManyToManyField(Collection, blank=True) + + # dados legíveis para facilitar a análise + readable_data = models.JSONField( + _("Readable data"), null=True, blank=True + ) base_form_class = CoreAdminModelForm @@ -430,10 +480,11 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): InlinePanel("other_pid", label=_("Other PID")), ] panel_c = [ - FieldPanel("z_surnames"), - FieldPanel("z_collab"), - FieldPanel("z_links"), - FieldPanel("z_partial_body"), + FieldPanel("z_surnames", read_only=True), + FieldPanel("z_collab", read_only=True), + FieldPanel("z_links", read_only=True), + FieldPanel("z_partial_body", read_only=True), + FieldPanel("readable_data", widget=ReadOnlyPrettyJSONWidget(), read_only=True), ] edit_handler = TabbedInterface( @@ -518,19 +569,6 @@ class Meta: def __str__(self): return f"{self.pkg_name} {self.v3}" - @property - def article_pid_suffix_source(self): - try: - return self.xml_with_pre.get_article_pid_suffix_source() - except AttributeError: - return self.elocation_id or self.fpage or self.xml_with_pre.order - - def get_article_pid_suffix(self): - data = self.article_pid_suffix_source - if not data: - data = self.pkg_name.split("-")[-1] - return string_to_5_digits(data) - @property def collection_list(self): return "|".join(c.acron3 for c in self.collections.all()) @@ -609,7 +647,7 @@ def get_xml_with_pre(cls, v3): except Exception: return None - @property + @cached_property @profile_property def xml_with_pre(self): try: @@ -630,6 +668,20 @@ def is_aop(self): return False return True + @property + def data_to_compare(self): + readable = self.readable_data or {} + titles = readable.get("article_titles") + body_fragment = readable.get("body_fragment") + return { + "article_titles": titles or self.xml_with_pre.article_titles_texts, + "z_surnames": self.z_surnames, + "z_collab": self.z_collab, + "z_links": self.z_links, + "z_partial_body": self.z_partial_body, + "body_fragment": body_fragment or self.xml_with_pre.get_body_fragment(300), + } + @classmethod @profile_classmethod def register( @@ -645,102 +697,78 @@ def register( registered_in_core=None, auto_solve_pid_conflict=True, ): - """ - Registra documento XML no sistema de PIDs, retornando PIDs v3, v2 e aop_pid. + response = {} + readable_input = None + pkg_name = filename + error_type = None + event_status = None + get_records_result = None + registered = None - Parameters - ---------- - xml_with_pre : XMLWithPre - Dados XML preprocessados - filename : str - Nome do arquivo XML - user : User - Usuário responsável pelo registro - origin_date : datetime, optional - Data de origem do documento - force_update : bool, optional - Força atualização mesmo sem alterações - is_published : bool, default False - Status de publicação - available_since : datetime, optional - Data de disponibilização - origin : str, optional - Origem do documento - registered_in_core : bool, optional - Se já registrado no sistema core - auto_solve_pid_conflict : bool, default False - Resolve conflitos de PID automaticamente - - Returns - ------- - dict - Sucesso: {"v3", "v2", "aop_pid", "xml_uri", "article", "created", - "updated", "xml_changed", "record_status"} - Erro: {"error_type", "error_message", "id", "filename"} - - Raises - ------ - QueryDocumentMultipleObjectsReturnedError - Múltiplos documentos encontrados - RequiredPublicationYearErrorToGetPidProviderXMLError - Ano de publicação obrigatório ausente - RequiredISSNErrorToGetPidProviderXMLError - ISSN obrigatório ausente - NotEnoughParametersToGetPidProviderXMLError - Parâmetros insuficientes para identificar documento - """ try: - input_data = None - xml_adapter_data = None - - response = {} response["input_data"] = xml_with_pre.data response["input_data"].update({"origin": origin}) - # adaptador do xml with pre xml_adapter = xml_sps_adapter.PidProviderXMLAdapter(xml_with_pre) response["xml_adapter_data"] = xml_adapter.data + # dados legíveis do XML entrando (mesmo formato do readable_data) + readable_input = xml_with_pre.get_article_data() + pkg_name = xml_adapter.sps_pkg_name + # consulta se documento já está registrado try: - records = cls.get_records(xml_adapter) - registered = cls.get_record(xml_adapter, records=records) - except cls.DoesNotExist as exc: - registered = None - except (cls.MultipleObjectsReturned, exceptions.UnmatchedPidProviderXMLError) as exc: - response["records"] = [item.data for item in records] - raise exceptions.QueryDocumentMultipleObjectsReturnedError(exc) + + get_records_result = cls.get_records(xml_adapter) + registered = get_records_result.get("registered") + if not registered and get_records_result.get("failed"): + raise exceptions.UnmatchedPidProviderXMLError + except cls.DoesNotExist: + pass except ( - exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError + cls.MultipleObjectsReturned, + exceptions.UnmatchedPidProviderXMLError, ) as exc: + event_status = "unmatched" raise exc - except exceptions.RequiredISSNErrorToGetPidProviderXMLError as exc: + except ( + exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError, + exceptions.RequiredISSNErrorToGetPidProviderXMLError, + exceptions.NotEnoughParametersToGetPidProviderXMLError, + ) as exc: + event_status = "bad_request" raise exc - except exceptions.NotEnoughParametersToGetPidProviderXMLError as exc: + + # valida/condiciona PIDs do XML (pode levantar conflito) + try: + response["xml_changed"] = cls.complete_missing_xml_pids( + xml_adapter, registered, auto_solve_pid_conflict + ) + except PidProviderXMLPidV3ConflictError as exc: + event_status = "conflict" raise exc - # valida os PIDs do XML - # - não podem ter conflito com outros registros - # - identifica mudança - response["xml_changed"] = cls.complete_missing_xml_pids( - xml_adapter, registered, auto_solve_pid_conflict - ) + # analisa se continua o registro (skip) + try: + updated_data = PidProviderXML.is_updated( + xml_with_pre, + registered, + force_update, + origin_date, + registered_in_core, + ) + except exceptions.ForbiddenPidProviderXMLRegistrationError as exc: + event_status = "forbidden" + raise exc - # analisa se continua o registro - updated_data = cls.is_updated( - xml_with_pre, - registered, - force_update, - origin_date, - registered_in_core, - ) if updated_data: response["skip_update"] = True response.update(updated_data) - return response + event_status = "skipped" + raise Exception(f"{pkg_name} is already up to date") # cria ou atualiza registro - registered = cls._save( + registered, event_status = cls._save( registered, xml_adapter, user, @@ -748,22 +776,29 @@ def register( available_since, registered_in_core, ) - - # data to return response.update(registered.data) - return response - except Exception as e: + except Exception as exc: exc_type, exc_value, exc_traceback = sys.exc_info() - UnexpectedEvent.create( - item=xml_with_pre.sps_pkg_name, - action="PidProviderXML.register", - exception=e, - exc_traceback=exc_traceback, - detail=response, + logging.exception(exc) + response["event_status"] = event_status + if event_status == "skipped": + response["message"] = str(exc) + else: + error_type = str(type(exc)) + response.update({"error_msg": str(exc), "error_type": error_type}) + + finally: + PidProviderXMLRegistration.record( + user=user, + pid_provider_xml=registered, + pkg_name=pkg_name, + event_status=event_status, + input_data=readable_input, + best_matches=get_records_result, + error_type=error_type, ) - response.update({"error_msg": str(e), "error_type": str(type(e))}) - return response + return response @classmethod @profile_classmethod @@ -835,55 +870,54 @@ def _save( registered_in_core=None, ): if registered: - # obtém os dados de substituição para registrar em other_pid + event_status = "updated" registered_changed = registered.check_registered_pids_changed( xml_adapter.xml_with_pre ) registered.updated_by = user else: + event_status = "created" registered = cls() registered.creator = user registered_changed = None - + registered.proc_status = choices.PPXML_STATUS_TODO registered._add_dates(xml_adapter, origin_date, available_since) registered._add_data(xml_adapter, registered_in_core) registered._add_journal(xml_adapter) registered._add_issue(xml_adapter) - + registered.save() - + if registered_changed: registered._add_other_pid(registered_changed, user) registered._add_current_version(xml_adapter.xml_with_pre, user) + + registered.add_collections(xml_adapter) + return registered, event_status + + def add_collections(self, xml_adapter): q = Q() - if COLLECTION_PREFIX == "scielojournal": - if xml_adapter.journal_issn_print: - q |= Q( - scielojournal__journal__official__issn_print=xml_adapter.journal_issn_print - ) - if xml_adapter.journal_issn_electronic: - q |= Q( - scielojournal__journal__official__issn_electronic=xml_adapter.journal_issn_electronic - ) - else: - if xml_adapter.journal_issn_print: - q |= Q( - journalproc__journal__official_journal__issn_print=xml_adapter.journal_issn_print - ) - if xml_adapter.journal_issn_electronic: - q |= Q( - journalproc__journal__official_journal__issn_electronic=xml_adapter.journal_issn_electronic - ) + issn_print = xml_adapter.journal_issn_print + issn_electronic = xml_adapter.journal_issn_electronic + + try: + Collection.objects.filter(scielojournal__isnull=True).exists() + issn_path = "scielojournal__journal__official" + except FieldError: + issn_path = "journalproc__journal__official_journal" + + if issn_print: + q |= Q(**{f"{issn_path}__issn_print": issn_print}) + if issn_electronic: + q |= Q(**{f"{issn_path}__issn_electronic": issn_electronic}) for collection in Collection.objects.filter(q): - registered.collections.add(collection) - return registered + self.collections.add(collection) - @classmethod - @profile_classmethod + @staticmethod def is_updated( - cls, xml_with_pre, registered, force_update, origin_date, registered_in_core + xml_with_pre, registered, force_update, origin_date, registered_in_core ): """ XML é versão AOP, mas @@ -939,21 +973,36 @@ def is_equal_to(self, xml_with_pre): @profile_classmethod def get_records(cls, xml_adapter): qbuilder = QueryBuilderPidProviderXML(xml_adapter) - q_ids = qbuilder.identifier_queries - q_journal = qbuilder.issn_query - q_issue = Q(**qbuilder.issue_params) - return cls.objects.filter(q_ids | (q_journal & q_issue)).distinct() - - @classmethod - @profile_classmethod - def get_record(cls, xml_adapter, records): - results = records - if not results.exists(): - raise cls.DoesNotExist - matched = cls.best_matches(results, xml_adapter) - if not matched: - raise cls.DoesNotExist - return cls.objects.get(id=sorted(matched)[-1][-1]) + + def _try(qs): + candidates = list(qs.select_related("current_version").distinct()) + if not candidates: + return None + result = cls.best_matches(candidates, xml_adapter) + return result if result.get("registered") else None + + # 1) correspondência direta por identificadores + result = _try(cls.objects.filter(qbuilder.identifier_queries)) + if result: + return result + + selected_journal = cls.objects.filter(qbuilder.issn_query) + + # 2) journal + issue + dados do artigo + result = _try( + selected_journal.filter( + Q(**qbuilder.issue_params) & qbuilder.article_data_query + ) + ) + if result: + return result + + # 3) journal + dados do artigo + result = _try(selected_journal.filter(qbuilder.article_data_query)) + if result: + return result + + raise cls.DoesNotExist @classmethod @profile_classmethod @@ -962,106 +1011,61 @@ def get_record_by_pid_v3(cls, xml_adapter): if not xml_adapter.v3: raise ValueError("get_record_by_pid_v3: XML has not pid v3") xml_pid_v3 = xml_adapter.v3 - results = ( - cls.objects.filter(Q(v3=xml_pid_v3) | Q(other_pid__pid_in_xml=xml_pid_v3)) + results = list( + cls.objects.filter( + Q(v3=xml_pid_v3) | Q(other_pid__pid_in_xml=xml_pid_v3) + ) + .select_related("current_version") + .distinct() ) - if not results.exists(): + if not results: raise cls.DoesNotExist - matched = cls.best_matches(results, xml_adapter) - if not matched: - UnexpectedEvent.create( - item=xml_adapter.sps_pkg_name, - action="PidProviderXML.get_record_by_pid_v3", - exception=PidProviderXMLPidV3ConflictError, - detail={"xml_adapter": xml_adapter.data, "results": [i.data for i in results]}, - ) - raise PidProviderXMLPidV3ConflictError( - _("No matching record found for the provided XML data.") - ) - return cls.objects.get(id=sorted(matched)[-1][-1]) - - @profile_method - def match(self, xml_adapter): - """ - """ - labels = [] - score = self.title_similarity(xml_adapter) * 100 - if score > 50: - labels.append("title") - if score_item := get_score(self.z_surnames, xml_adapter.z_surnames, 10, 100): - labels.append("z_surnames") - score += score_item - if score_item := get_score(self.z_collab, xml_adapter.z_collab, 10, 100): - labels.append("z_collab") - score += score_item - if score_item := get_score(self.z_links, xml_adapter.z_links, 10, 100): - labels.append("z_links") - score += score_item - if score_item := get_score(self.z_partial_body, xml_adapter.z_partial_body, 10, 100): - labels.append("z_partial_body") - score += score_item - return {"score": score, "labels": labels} - - def title_similarity(self, xml_adapter): + best_matches_results = cls.best_matches(results, xml_adapter) try: - registered = self.xml_with_pre.article_titles_texts - except Exception: - registered = [] - xml_adapter_titles = xml_adapter.xml_with_pre.article_titles_texts - if xml_adapter_titles == registered: - return 1 - if not xml_adapter_titles: - return 0 - if not registered: - return 0 - words1 = set() - for item in xml_adapter_titles: - words1.update(item.split()) - words2 = set() - for item in registered: - words2.update(item.split()) - return how_similar(" ".join(sorted(words1)), " ".join(sorted(words2))) + return best_matches_results["registered"] + except KeyError: + if best_matches_results["failed"]: + raise PidProviderXMLPidV3ConflictError( + _("No matching record found for the provided XML data.") + ) + raise cls.DoesNotExist @classmethod def best_matches(cls, results, xml_adapter): - data = [] - matched = [] - for item in results.iterator(): - response = item.match(xml_adapter) - score = response["score"] - - if xml_adapter.v2: - if item.v2 == xml_adapter.v2: - score += 100 - elif xml_adapter.order and item.v2 and item.v2.endswith(xml_adapter.order): - score += 100 - if item.v3 == xml_adapter.v3: - score += 100 - if item.pkg_name == xml_adapter.pkg_name: - score += 100 - if item.main_doi == xml_adapter.main_doi: - score += 100 - - _data = response - _data.update(item.data) - data.append(_data) - - if score > 50: - matched.append((score, item.updated.isoformat(), item.id)) - - if results.count() > 1 or not matched: - detail = { - "xml_adapter_data": xml_adapter.data, - "data": data, - "matched": matched, - } - UnexpectedEvent.create( - item=xml_adapter.sps_pkg_name, - action="PidProviderXML.best_matches", - exception=cls.MultipleObjectsReturned, - detail=detail, + # results agora é uma LISTA materializada (não queryset) + input_data = xml_adapter.get_data_to_compare() + detail = { + "total_results": len(results), + "input_data": input_data, + } + responses = {} + found = [] + items = {} + for item in results: + item_data = item.data_to_compare + response = compare(item_data, input_data) + response["id"] = item.id + responses[item.id] = response + items[item.id] = item + found.append( + (response["percentual_score"], item.updated.isoformat(), item.id) ) - return matched + + found = sorted(found, reverse=True) + + ok = [] + failed = [] + for percentual_score, updated, item_id in found: + if percentual_score > 0.6: + ok.append(responses[item_id]) + else: + failed.append(responses[item_id]) + + detail["ok"] = ok + detail["failed"] = failed + if ok: + detail["registered"] = items.get(ok[0]["id"]) + return detail @profile_method def _add_data(self, xml_adapter, registered_in_core): @@ -1085,6 +1089,10 @@ def _add_data(self, xml_adapter, registered_in_core): self.z_links = xml_adapter.z_links self.z_partial_body = xml_adapter.z_partial_body + # NOVO: dados legíveis (somente inspeção) + self.readable_data = xml_adapter.xml_with_pre.get_article_data() + + @profile_method def _add_dates(self, xml_adapter, origin_date, available_since): # evita que artigos WIP fique disponíveis antes de estarem públicos @@ -1158,20 +1166,15 @@ def check_registered_pids_changed(self, xml_with_pre): @profile_method def _add_other_pid(self, registered_changed, user): - # registrados passam a ser other pid - # os pids do XML passam a ser os vigentes if not registered_changed: return for change_args in registered_changed: - change_args["pid_in_xml"] = change_args.pop("registered") - change_args["user"] = user change_args["pid_provider_xml"] = self - OtherPid.get_or_create(**change_args) self.other_pid_count = self.other_pid.count() - self.save() + self.save(update_fields=["other_pid_count"]) @classmethod @profile_classmethod @@ -1232,8 +1235,10 @@ def is_registered( response["xml_adapter_data"] = xml_adapter_data try: - records = cls.get_records(xml_adapter) - registered = cls.get_record(xml_adapter, records=records) + get_records_result = cls.get_records(xml_adapter) + registered = get_records_result.get("registered") + if not registered and get_records_result.get("failed"): + raise exceptions.UnmatchedPidProviderXMLError except cls.DoesNotExist as exc: response.update( {"filename": xml_with_pre.filename, "registered": False} @@ -1241,13 +1246,12 @@ def is_registered( return response except (cls.MultipleObjectsReturned, exceptions.UnmatchedPidProviderXMLError) as exc: exc_type, exc_value, exc_traceback = sys.exc_info() - response["records"] = [item.data for item in records] UnexpectedEvent.create( item=xml_with_pre.sps_pkg_name, action="PidProviderXML.is_registered", exception=exc, exc_traceback=exc_traceback, - detail=response, + detail=get_records_result, ) response.update({"error_msg": str(exc), "error_type": str(type(exc))}) return response @@ -1327,13 +1331,19 @@ def mark_as_done(self): @classmethod @profile_classmethod def mark_items_as_invalid(cls, issns): - for item in cls.objects.filter( + items = cls.objects.filter( Q(issn_print__in=issns) | Q(issn_electronic__in=issns), - ).iterator(): + ) + items_to_update = [] + for item in items.iterator(): try: - invalid = bool(item.xml_with_pre) + valid = bool(item.xml_with_pre) except Exception as e: - invalid = True + valid = False + if not valid: + item.proc_status = choices.PPXML_STATUS_INVALID + items_to_update.append(item) + cls.objects.bulk_update(items_to_update, ["proc_status"], batch_size=100) @classmethod @profile_classmethod @@ -1785,3 +1795,119 @@ def record(cls, user, url, status, document_item, *, exception=None, response=No xmlurl_obj.save_file(xml_with_pre.tostring(), filename=filename) return xmlurl_obj + + +# ----------------------------------------------------------------------------- +# [models.py] MODELO NOVO — PidProviderXMLRegistration +# Auditoria por documento. Grava SEMPRE (created/updated/skipped/forbidden/ +# conflict/unmatched/error). FK nullable (unmatched/error podem não ter PPX). +# ----------------------------------------------------------------------------- +class PidProviderXMLRegistration(CommonControlField): + EVENT_CREATED = "created" + EVENT_UPDATED = "updated" + EVENT_SKIPPED = "skipped" + EVENT_FORBIDDEN = "forbidden" + EVENT_CONFLICT = "conflict" + EVENT_UNMATCHED = "unmatched" + EVENT_ERROR = "error" + EVENT_BAD_REQUEST = "bad_request" + + EVENT_STATUS_CHOICES = ( + (EVENT_CREATED, "created"), + (EVENT_UPDATED, "updated"), + (EVENT_SKIPPED, "skipped"), + (EVENT_FORBIDDEN, "forbidden"), + (EVENT_CONFLICT, "conflict"), + (EVENT_UNMATCHED, "unmatched"), + (EVENT_BAD_REQUEST, "bad_request"), + (EVENT_ERROR, "error"), + ) + + pid_provider_xml = models.ForeignKey( + PidProviderXML, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="registration_events", + ) + pkg_name = models.CharField( + _("Package name"), max_length=100, null=True, blank=True + ) + event_status = models.CharField( + _("Event status"), + max_length=15, + null=True, + blank=True, + choices=EVENT_STATUS_CHOICES, + ) + input_data = models.JSONField(_("Input data (readable)"), null=True, blank=True) + best_matches = models.JSONField(_("Best matches detail"), null=True, blank=True) + error_type = models.CharField( + _("Error type"), max_length=255, null=True, blank=True + ) + + base_form_class = CoreAdminModelForm + + panels = [ + FieldPanel("event_status", read_only=True), + FieldPanel("pkg_name", read_only=True), + AutocompletePanel("pid_provider_xml", read_only=True), + FieldPanel("error_type", read_only=True), + FieldPanel("input_data", widget=ReadOnlyPrettyJSONWidget(), read_only=True), + FieldPanel("best_matches", widget=ReadOnlyPrettyJSONWidget(), read_only=True), + ] + + class Meta: + ordering = ["-created"] + verbose_name = _("PidProviderXML Registration") + verbose_name_plural = _("PidProviderXML Registrations") + indexes = [ + models.Index(fields=["pkg_name"]), + models.Index(fields=["event_status"]), + models.Index(fields=["-created"]), + models.Index(fields=["pid_provider_xml"]), + ] + + def __str__(self): + return f"{self.pkg_name} {self.event_status} {self.created}" + + @staticmethod + def _serialize_best_matches(best_matches): + """ + O detail do best_matches contém o objeto PidProviderXML em + detail['registered']. Para gravar em JSON, troca pelo v3/id. + """ + if not best_matches: + return None + data = dict(best_matches) + registered = data.get("registered") + if registered is not None and hasattr(registered, "v3"): + data["registered"] = {"id": registered.id, "v3": registered.v3} + return data + + @classmethod + def record( + cls, + user, + event_status, + pid_provider_xml=None, + pkg_name=None, + input_data=None, + best_matches=None, + error_type=None, + ): + try: + obj = cls() + obj.creator = user + obj.pid_provider_xml = pid_provider_xml + obj.pkg_name = pkg_name or (pid_provider_xml and pid_provider_xml.pkg_name) + obj.event_status = event_status + obj.input_data = input_data + obj.best_matches = cls._serialize_best_matches(best_matches) + obj.error_type = error_type + obj.save() + return obj + except Exception as e: + # registro de auditoria nunca deve derrubar o register + logging.exception(f"Unable to record PidProviderXMLRegistration: {e}") + return None diff --git a/pid_provider/query_params.py b/pid_provider/query_params.py index bfb6901bd..d25297e20 100644 --- a/pid_provider/query_params.py +++ b/pid_provider/query_params.py @@ -3,10 +3,55 @@ from django.db.models import Q from django.utils.translation import gettext_lazy as _ -from core.utils.profiling_tools import profile_function +from core.utils.similarity import how_similar from pid_provider import exceptions +def compare(registered_items, input_data): + """ + """ + total_score = 0 + items = [] + for label, registered_item in registered_items.items(): + result = compare_items(label, registered_item, input_data.get(label)) + items.append(result) + total_score += result["score"] + return { + "items": items, + "total_score": total_score, + "percentual_score": total_score / len(items) + } + + +def compare_lists(registered, xml_adapter_titles): + if xml_adapter_titles == registered: + return 1 + if not xml_adapter_titles: + return 0 + if not registered: + return 0 + words1 = set() + for item in xml_adapter_titles: + words1.update(item.split()) + words2 = set() + for item in registered: + words2.update(item.split()) + return how_similar(" ".join(sorted(words1)), " ".join(sorted(words2))) + + +def compare_items(label, registered, input_data): + if isinstance(registered, list): + score = compare_lists(registered, input_data) + elif (input_data or None) == (registered or None): + score = 1 + else: + score = how_similar(input_data, registered) + response = {"label": label, "score": score} + if score != 1: + response["registered"] = registered + return response + + def get_score(registered, xml_data, min_value, max_value): if registered == xml_data: if registered: @@ -45,7 +90,7 @@ def __init__(self, xml_adapter): self.xml_adapter = xml_adapter # ========== Cached Properties para Atributos do XML Adapter ========== - + @cached_property def v3(self): """PID v3 do documento.""" @@ -173,17 +218,21 @@ def identifier_queries(self): """ q = Q() + other_pids = set() # PID v3 - máxima prioridade if self.v3: q |= Q(v3=self.v3) + other_pids.add(self.v3) # PID v2 if self.v2: q |= Q(v2=self.v2) + other_pids.add(self.v2) # AOP PID if self.aop_pid: q |= Q(v2=self.aop_pid) | Q(aop_pid=self.aop_pid) + other_pids.add(self.aop_pid) # Package name pkg_names = set() @@ -196,10 +245,11 @@ def identifier_queries(self): if pkg_names: q |= Q(pkg_name__in=pkg_names) - # # DOI principal - # if self.main_doi: - # q |= Q(main_doi=self.main_doi) + if self.main_doi: + q |= Q(main_doi=self.main_doi) + if other_pids: + q |= Q(other_pid__pid_in_xml__in=other_pids) return q @cached_property @@ -245,23 +295,36 @@ def issue_params(self): Returns ------- dict - Dicionário com elocation_id, fpage, fpage_seq, lpage, - pub_year, volume, number e suppl + Dicionário com pub_year, volume, number e suppl + """ + return { + "pub_year": self.pub_year, + "volume": self.volume, + "number": self.number, + "suppl": self.suppl, + } + + @cached_property + def article_location_params(self): + """ + Constrói dicionário com metadados do fascículo e paginação do artigo. + + Retorna todos os campos sem verificar presença, permitindo + que o ORM do Django filtre automaticamente valores None. + + Returns + ------- + dict + Dicionário com elocation_id, fpage, fpage_seq, lpage, v2__endswith """ data = { "elocation_id": self.elocation_id, "fpage": self.fpage, "fpage_seq": self.fpage_seq, "lpage": self.lpage, - "pub_year": self.pub_year, - "volume": self.volume, - "number": self.number, - "suppl": self.suppl, } if self.order: data["v2__endswith"] = self.order - elif not self.elocation_id and not self.fpage and self.main_doi: - data["main_doi__iexact"] = self.main_doi return data @cached_property @@ -279,33 +342,30 @@ def article_data_query(self): ou None se nenhum dado textual estiver disponível """ # Verifica se há algum dado textual disponível - if not any([ - self.z_surnames, - self.z_collab, - self.z_links, - self.z_partial_body, - ]): - return Q( - z_surnames=self.z_surnames, - z_collab=self.z_collab, - z_links=self.z_links, - z_partial_body=self.z_partial_body, - ) - - q = Q() - - # Adiciona query para sobrenomes se disponível - if self.z_surnames: - q |= Q(z_surnames=self.z_surnames) + if self.z_surnames or self.z_partial_body or self.z_collab or self.z_links: + q = Q() - # Adiciona queries para outros campos textuais - if self.z_collab: - q |= Q(z_collab=self.z_collab) - - if self.z_links: - q |= Q(z_links=self.z_links) + # Adiciona query para sobrenomes se disponível + if self.z_surnames: + q |= Q(z_surnames=self.z_surnames) + + # Adiciona queries para outros campos textuais + if self.z_collab: + q |= Q(z_collab=self.z_collab) + + if self.z_links: + q |= Q(z_links=self.z_links) + + if self.z_partial_body: + q |= Q(z_partial_body=self.z_partial_body) + + return q - if self.z_partial_body: - q |= Q(z_partial_body=self.z_partial_body) + return Q( + z_surnames=self.z_surnames, + z_collab=self.z_collab, + z_links=self.z_links, + z_partial_body=self.z_partial_body, + ) - return q \ No newline at end of file + \ No newline at end of file diff --git a/pid_provider/tests/__init__.py b/pid_provider/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pid_provider/tests/test_best_matches.py b/pid_provider/tests/test_best_matches.py new file mode 100644 index 000000000..b85861955 --- /dev/null +++ b/pid_provider/tests/test_best_matches.py @@ -0,0 +1,104 @@ +from django.test import TestCase +from unittest.mock import MagicMock, patch +from pid_provider.models import PidProviderXML + + +class PidProviderXMLBestMatchesTests(TestCase): + + def setUp(self): + # Mock do xml_adapter e das propriedades exigidas por best_matches + self.xml_adapter_mock = MagicMock() + self.xml_adapter_mock.xml_with_pre.article_titles_texts = "Titulo Original" + self.xml_adapter_mock.z_surnames = "Silva; Santos" + self.xml_adapter_mock.z_collab = "Grupo SpS" + self.xml_adapter_mock.z_links = "http://link1.com" + self.xml_adapter_mock.z_partial_body = "Texto do corpo do artigo..." + + @patch("pid_provider.models.compare") + def test_best_matches_success_with_valid_and_invalid_scores(self, mock_compare): + """Deve ordenar os resultados por score descrescente e separar entre 'ok' (>0.5) e 'failed' (<=0.5).""" + + # Cria mocks para simular instâncias de PidProviderXML do banco de dados + item_bom = MagicMock(spec=PidProviderXML) + item_bom.id = 101 + item_bom.updated.isoformat.return_value = "2026-06-27T12:00:00" + item_bom.data_to_compare = {"title": "Titulo Original", "z_surnames": "Silva; Santos"} + + item_ruim = MagicMock(spec=PidProviderXML) + item_ruim.id = 102 + item_ruim.updated.isoformat.return_value = "2026-06-27T13:00:00" + item_ruim.data_to_compare = {"title": "Outro Titulo Completamente Diferente", "z_surnames": "Alves"} + + # Configura o efeito colateral do mock 'compare' baseado no item recebido + def side_effect_compare(item_data, input_data): + if item_data["title"] == "Titulo Original": + return {"id": 101, "percentual_score": 0.95, "match": True} + return {"id": 102, "percentual_score": 0.20, "match": False} + + mock_compare.side_effect = side_effect_compare + + # Executa o método passando a lista materializada de candidatos + candidates = [item_ruim, item_bom] # Enviados fora de ordem propositalmente + result = PidProviderXML.best_matches(candidates, self.xml_adapter_mock) + + # Validações de estrutura e contagem + self.assertEqual(result["total_results"], 2) + self.assertEqual(len(result["ok"]), 1) + self.assertEqual(len(result["failed"]), 1) + + # Valida a ordenação e separação (o de score 0.95 deve ser o 'registered') + self.assertEqual(result["registered"], item_bom) + self.assertEqual(result["ok"][0]["id"], 101) + self.assertEqual(result["failed"][0]["id"], 102) + + @patch("pid_provider.models.compare") + def test_best_matches_no_candidates_approved(self, mock_compare): + """Quando nenhum candidato atinge score > 0.5, a chave 'registered' não deve existir no retorno.""" + + item_fraco = MagicMock(spec=PidProviderXML) + item_fraco.id = 201 + item_fraco.updated.isoformat.return_value = "2026-06-27T14:00:00" + item_fraco.data_to_compare = {"title": "Quase igual, mas nao o suficiente"} + + # Mock retorna score abaixo da linha de corte (0.5) + mock_compare.return_value = {"id": 201, "percentual_score": 0.48, "match": False} + + result = PidProviderXML.best_matches([item_fraco], self.xml_adapter_mock) + + # Validações + self.assertEqual(result["total_results"], 1) + self.assertEqual(len(result["ok"]), 0) + self.assertEqual(len(result["failed"]), 1) + self.assertNotIn("registered", result) # Não pode haver match oficial + + @patch("pid_provider.models.compare") + def test_best_matches_tie_breaking_by_updated_date(self, mock_compare): + """Em caso de empate no percentual_score, o critério de desempate do sorted() deve usar a data 'updated' descrescente.""" + + # Dois itens com o mesmo score, mas datas de atualização diferentes + item_antigo = MagicMock(spec=PidProviderXML) + item_antigo.id = 301 + item_antigo.updated.isoformat.return_value = "2026-01-01T00:00:00" + item_antigo.data_to_compare = {"title": "Clone"} + + item_recente = MagicMock(spec=PidProviderXML) + item_recente.id = 302 + item_recente.updated.isoformat.return_value = "2026-06-27T00:00:00" # Mais recente + item_recente.data_to_compare = {"title": "Clone"} + + # Mock retorna o mesmo score alto para ambos + def side_effect_compare(item_data, input_data): + items = [ + {"label": "title", "score": 0.9}, + {"label": "title", "score": 0.9}, + ] + return {"items": items, "percentual_score": 0.90, "total_score": 0.90} + + mock_compare.side_effect = side_effect_compare + + # Executa passando o antigo primeiro + result = PidProviderXML.best_matches([item_antigo, item_recente], self.xml_adapter_mock) + + # Como a ordenação usa reverse=True no par (score, data.isoformat(), id), + # o item_recente ("2026-06-27...") deve ficar em primeiro lugar na ordenação. + self.assertEqual(result["registered"], item_recente) \ No newline at end of file diff --git a/pid_provider/test_controller.py b/pid_provider/tests/test_controller.py similarity index 98% rename from pid_provider/test_controller.py rename to pid_provider/tests/test_controller.py index 3ff177825..f4235cec6 100644 --- a/pid_provider/test_controller.py +++ b/pid_provider/tests/test_controller.py @@ -3,7 +3,7 @@ from django.contrib.auth import get_user_model from django.test import TestCase -from pid_provider.controller import PidProvider +from pid_provider.provider import PidProvider from pid_provider.models import PidProviderXML User = get_user_model() diff --git a/pid_provider/tests/test_get_records.py b/pid_provider/tests/test_get_records.py new file mode 100644 index 000000000..7b0f9e981 --- /dev/null +++ b/pid_provider/tests/test_get_records.py @@ -0,0 +1,145 @@ +from unittest.mock import MagicMock, patch + +from django.contrib.auth import get_user_model +from django.db.models import Q +from django.test import TestCase + +from pid_provider.models import PidProviderXML +from pid_provider.exceptions import UnmatchedPidProviderXMLError + + +User = get_user_model() + + +class PidProviderXMLGetRecordsTests(TestCase): + + def setUp(self): + # Cria um usuário para satisfazer a restrição de 'creator_id' + self.user = User.objects.create_user(username="testuser", password="password") + + # Mocks existentes... + self.xml_adapter_mock = MagicMock() + self.xml_adapter_mock.xml_with_pre.article_titles_texts = "Titulo de Teste" + self.xml_adapter_mock.z_surnames = "Silva" + self.xml_adapter_mock.z_collab = None + self.xml_adapter_mock.z_links = None + self.xml_adapter_mock.z_partial_body = "Corpo parcial do artigo" + self.xml_adapter_mock.sps_pkg_name = "test_package" + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + @patch.object(PidProviderXML, "best_matches") + def test_get_records_by_identifiers_success(self, mock_best_matches, mock_qbuilder_cls): + """1) Deve retornar o registro quando encontrado por identificadores diretos.""" + # Configura o mock do QueryBuilder + mock_qbuilder = mock_qbuilder_cls.return_value + mock_qbuilder.identifier_queries = Q(v3="12345") + + # Cria um registro no banco que case com a query + record = PidProviderXML.objects.create(creator=self.user, v3="12345", registered_in_core=True) + + # Mock do retorno do best_matches + expected_result = {"registered": record, "total_results": 1} + mock_best_matches.return_value = expected_result + + # Executa o método + result = PidProviderXML.get_records(self.xml_adapter_mock) + + # Validações + self.assertEqual(result, expected_result) + mock_best_matches.assert_called_once_with([record], self.xml_adapter_mock) + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + @patch.object(PidProviderXML, "best_matches") + def test_get_records_by_journal_and_issue_and_article_success(self, mock_best_matches, mock_qbuilder_cls): + """2) Deve encontrar o registro por Journal + Issue + Dados do Artigo quando os IDs falharem.""" + mock_qbuilder = mock_qbuilder_cls.return_value + # Força falha no passo 1 + mock_qbuilder.identifier_queries = Q(v3="id_inexistente") + + # Configura os passos do fluxo de fallback + mock_qbuilder.issn_query = Q(issn_print="1234-5678") + mock_qbuilder.issue_params = {"pub_year": 2026} + mock_qbuilder.article_data_query = Q(z_surnames="Silva") + + # Cria o registro correspondente no banco + record = PidProviderXML.objects.create(creator=self.user, + issn_print="1234-5678", + z_surnames="Silva", + v3="outro_id" + ) + + expected_result = {"registered": record, "total_results": 1} + mock_best_matches.return_value = expected_result + + result = PidProviderXML.get_records(self.xml_adapter_mock) + + self.assertEqual(result, expected_result) + mock_best_matches.assert_called_once_with([record], self.xml_adapter_mock) + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + @patch.object(PidProviderXML, "best_matches") + def test_get_records_by_journal_and_article_only_success(self, mock_best_matches, mock_qbuilder_cls): + """3) Deve encontrar o registro por Journal + Dados do Artigo (ignorando Issue) se os passos anteriores falharem.""" + mock_qbuilder = mock_qbuilder_cls.return_value + # Força falha nos passos 1 e 2 + mock_qbuilder.identifier_queries = Q(v3="id_inexistente") + mock_qbuilder.issn_query = Q(issn_print="1234-5678") + mock_qbuilder.issue_params = {"pub_year": 9999} # Ano errado para falhar o passo 2 + mock_qbuilder.article_data_query = Q(z_surnames="Silva") + + # Registro no banco compartilha apenas o ISSN e o dado do artigo (o ano/issue seria diferente) + record = PidProviderXML.objects.create(creator=self.user, + issn_print="1234-5678", + z_surnames="Silva" + ) + + expected_result = {"registered": record, "total_results": 1} + mock_best_matches.return_value = expected_result + + result = PidProviderXML.get_records(self.xml_adapter_mock) + + self.assertEqual(result, expected_result) + mock_best_matches.assert_called_once_with([record], self.xml_adapter_mock) + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + @patch.object(PidProviderXML, "best_matches") + def test_get_records_raises_does_not_exist(self, mock_best_matches, mock_qbuilder_cls): + """4) Deve levantar PidProviderXML.DoesNotExist se nenhuma das estratégias encontrar candidatos.""" + mock_qbuilder = mock_qbuilder_cls.return_value + mock_qbuilder.identifier_queries = Q(v3="nao_existe") + mock_qbuilder.issn_query = Q(issn_print="0000-0000") + mock_qbuilder.issue_params = {} + mock_qbuilder.article_data_query = Q(z_surnames="Ninguém") + + with self.assertRaises(PidProviderXML.DoesNotExist): + PidProviderXML.get_records(self.xml_adapter_mock) + + # O best_matches nunca deve ter sido chamado porque nenhuma lista de candidatos foi gerada + mock_best_matches.assert_not_called() + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + @patch.object(PidProviderXML, "best_matches") + def test_get_records_fallback_when_best_matches_returns_no_registered(self, mock_best_matches, mock_qbuilder_cls): + """5) Se o passo 1 achar candidatos mas o best_matches não validar um 'registered', deve prosseguir para o passo seguinte.""" + mock_qbuilder = mock_qbuilder_cls.return_value + mock_qbuilder.identifier_queries = Q(v3="id_com_score_baixo") + mock_qbuilder.issn_query = Q(issn_print="1234-5678") + mock_qbuilder.issue_params = {"pub_year": 2026} + mock_qbuilder.article_data_query = Q(z_surnames="Silva") + + # Cria candidato para o passo 1 e o alvo real para o passo 2 + candidato_ruim = PidProviderXML.objects.create(creator=self.user, v3="id_com_score_baixo") + alvo_correto = PidProviderXML.objects.create(creator=self.user, issn_print="1234-5678", z_surnames="Silva") + + # Configura o mock do best_matches para simular comportamento diferente por chamada + # 1ª chamada (Passo 1): Encontra resultado mas sem a chave 'registered' validada (percentual baixo) + # 2ª chamada (Passo 2): Encontra e valida o 'registered' + mock_best_matches.side_effect = [ + {"registered": None, "failed": [candidato_ruim]}, + {"registered": alvo_correto, "total_results": 1} + ] + + result = PidProviderXML.get_records(self.xml_adapter_mock) + + self.assertEqual(result["registered"], alvo_correto) + self.assertEqual(mock_best_matches.call_count, 2) \ No newline at end of file diff --git a/pid_provider/test_models.py b/pid_provider/tests/test_models.py similarity index 98% rename from pid_provider/test_models.py rename to pid_provider/tests/test_models.py index 4249c7c2a..f874ff514 100644 --- a/pid_provider/test_models.py +++ b/pid_provider/tests/test_models.py @@ -6,10 +6,10 @@ from django.contrib.auth import get_user_model from django.test import TestCase from lxml import etree -from xmlsps.xml_sps_lib import XMLWithPre +from packtools.xmlsps.xml_sps_lib import XMLWithPre from pid_provider import exceptions, models -from pid_provider.xml_sps_adapter import PidProviderXMLAdapter +from packtools.pid_provider.xml_sps_adapter import PidProviderXMLAdapter User = get_user_model() @@ -100,7 +100,7 @@ def test_validate_query_params_all_present_plus_issue_params(self): def test_validate_query_params_issue_params_only(self): params = {} params.update(self.issue_params) - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): result = models.PidProviderXML.validate_query_params(params) def test_validate_query_params_journal_issns_absence(self): @@ -108,7 +108,7 @@ def test_validate_query_params_journal_issns_absence(self): params.update(self.issue_params) del params["journal__issn_print"] del params["journal__issn_electronic"] - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): result = models.PidProviderXML.validate_query_params(params) def test_validate_query_params_pub_year_absence(self): @@ -116,7 +116,7 @@ def test_validate_query_params_pub_year_absence(self): params.update(self.issue_params) del params["article_pub_year"] del params["issue__pub_year"] - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): result = models.PidProviderXML.validate_query_params(params) def test_validate_query_params_main_doi_absence(self): @@ -200,7 +200,7 @@ def test_validate_query_params_z_collab_id_absence(self): del params["z_collab"] del params["z_links"] - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): result = models.PidProviderXML.validate_query_params(params) @@ -314,10 +314,10 @@ def test_query_document_raises_error( ] mock_query_list.return_value = params_list mock_validate_params.side_effect = ( - exceptions.NotEnoughParametersToGetDocumentRecordError + exceptions.NotEnoughParametersToGetPidProviderXMLError ) - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): xml_adapter = _get_xml_adapter() result = models.PidProviderXML._query_document(xml_adapter) @@ -379,7 +379,7 @@ def test_get_registered_returns_error_not_enough_params( mock_query_document, ): mock_query_document.side_effect = ( - exceptions.NotEnoughParametersToGetDocumentRecordError + exceptions.NotEnoughParametersToGetPidProviderXMLError ) result = models.PidProviderXML.get_registered(self.xml_with_pre) @@ -731,7 +731,7 @@ def test_register_returns_error( mock_now, ): expected = { - "result_type": "", + "result_type": "", "result_message": "No attribute enough for disambiguations {'z_surnames': None, 'z_collab': None, 'main_doi': None, 'z_links': None, 'z_partial_body': None, 'pkg_name': None, 'elocation_id': None, 'journal__issn_print': None, 'journal__issn_electronic': None, 'article_pub_year': None}", "origin": "filename.xml", "xml": "
", @@ -778,7 +778,7 @@ def test_register_with_success( mock_now, ): expected = { - "result_type": "", + "result_type": "", "result_message": "No attribute enough for disambiguations {'z_surnames': None, 'z_collab': None, 'main_doi': None, 'z_links': None, 'z_partial_body': None, 'pkg_name': None, 'elocation_id': None, 'journal__issn_print': None, 'journal__issn_electronic': None, 'article_pub_year': None}", "origin": "filename.xml", "xml": "
", diff --git a/pid_provider/tests/test_register.py b/pid_provider/tests/test_register.py new file mode 100644 index 000000000..a8bdccc32 --- /dev/null +++ b/pid_provider/tests/test_register.py @@ -0,0 +1,313 @@ +""" +Testes unitários para PidProviderXML.register. + +Estratégia +---------- +register() é orquestrador: delega a get_records, complete_missing_xml_pids, +is_updated e _save. Os testes isolam register desses colaboradores via mock e +verificam, para cada caminho, DOIS contratos: + + 1. o event_status efetivamente gravado em PidProviderXMLRegistration.record + 2. a forma do response retornado + +Como register grava SEMPRE (no finally), o ponto de observação central é o +mock de PidProviderXMLRegistration.record: inspecionamos seus kwargs. + +Caminhos cobertos (event_status): + - created : sem registro existente -> _save cria + - updated : registro existente -> _save atualiza + - skipped : is_updated retorna data (já atualizado / igual) + - conflict : complete_missing_xml_pids levanta PidProviderXMLPidV3ConflictError + - forbidden : is_updated levanta ForbiddenPidProviderXMLRegistrationError + - unmatched : get_records levanta UnmatchedPidProviderXMLError / MultipleObjectsReturned + - bad_request : get_records levanta RequiredISSNError etc. + - error : exceção inesperada -> UnexpectedEvent + event_status=error + +Ajuste os caminhos de import (PATCH_BASE) conforme a estrutura do seu projeto. +""" + +from unittest.mock import patch, MagicMock + +from django.test import TestCase + +from pid_provider import exceptions +from pid_provider.models import ( + PidProviderXML, + PidProviderXMLPidV3ConflictError, +) + +# Caminho do módulo onde register está definido (para os patches "where used"). +PATCH_BASE = "pid_provider.models" + + +def make_xml_with_pre(**overrides): + """ + XMLWithPre falso, com os atributos que register/build_readable_data tocam. + """ + m = MagicMock(name="xml_with_pre") + m.data = {"pid_v3": overrides.get("v3"), "sps_pkg_name": "pkg-fake"} + m.sps_pkg_name = overrides.get("sps_pkg_name", "pkg-fake") + # build_readable_data: + m.authors = {"person": [{"surname": "SILVA"}]} + m.collab = None + m.links = [] + m.article_titles_texts = ["Some title"] + m.partial_body = "corpo parcial" + return m + + +class RegisterTestBase(TestCase): + """ + Mocka todos os colaboradores de register e o gravador de auditoria. + Cada teste configura os side_effects/returns conforme o caminho. + """ + + def setUp(self): + self.user = MagicMock(name="user") + self.xml = make_xml_with_pre(v3="ABCDEFGHIJKLMNOPQRSTUVW") + + # patch do adapter para não depender de packtools real + self.p_adapter = patch( + "packtools.sps.pid_provider.xml_sps_adapter.PidProviderXMLAdapter" + ) + self.m_adapter_cls = self.p_adapter.start() + self.m_adapter = self.m_adapter_cls.return_value + self.m_adapter.data = {"pkg_name": "pkg-fake"} + self.m_adapter.sps_pkg_name = "pkg-fake" + self.m_adapter.xml_with_pre = self.xml + self.addCleanup(self.p_adapter.stop) + + # patch do gravador de auditoria — ponto central de verificação + self.p_record = patch(f"{PATCH_BASE}.PidProviderXMLRegistration.record") + self.m_record = self.p_record.start() + self.addCleanup(self.p_record.stop) + + # build_readable_data é staticmethod; deixamos rodar (usa o xml fake), + # mas se preferir isolar, dá para mockar também. + + # -- helper de asserção ------------------------------------------------- + def assert_recorded_status(self, expected_status): + self.assertTrue( + self.m_record.called, "PidProviderXMLRegistration.record não foi chamado" + ) + kwargs = self.m_record.call_args.kwargs + self.assertEqual(kwargs.get("event_status"), expected_status) + return kwargs + + +class CreatedPathTest(RegisterTestBase): + def test_created_when_no_existing_record(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.side_effect = PidProviderXML.DoesNotExist + m_cmp.return_value = {} + m_upd.return_value = None + saved = MagicMock(name="saved_ppx") + saved.data = {"v3": "ABC", "record_status": "created"} + m_save.return_value = (saved, "created") + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("created") + self.assertEqual(response.get("v3"), "ABC") + self.assertNotIn("error_msg", response) + + +class UpdatedPathTest(RegisterTestBase): + def test_updated_when_existing_record(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.return_value = {"registered": existing, "failed": []} + m_cmp.return_value = {"pid_v3": "NEW"} + m_upd.return_value = None + saved = MagicMock(name="saved_ppx") + saved.data = {"v3": "ABC", "record_status": "updated"} + m_save.return_value = (saved, "updated") + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + kwargs = self.assert_recorded_status("updated") + # o evento referencia o objeto salvo + self.assertIs(kwargs.get("pid_provider_xml"), saved) + self.assertIn("xml_changed", response) + + +class SkippedPathTest(RegisterTestBase): + def test_skipped_returns_data_and_does_not_flag_error(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.return_value = {"registered": existing, "failed": []} + m_cmp.return_value = {} + m_upd.return_value = {"v3": "ABC", "record_status": "updated"} # já atualizado + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + # _save NÃO deve ser chamado no skip + m_save.assert_not_called() + + self.assert_recorded_status("skipped") + self.assertTrue(response.get("skip_update")) + # skip é sucesso de negócio: não deve marcar erro + self.assertNotIn("error_msg", response) + + +class ConflictPathTest(RegisterTestBase): + def test_conflict_when_pid_v3_conflict(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.return_value = {"registered": existing, "failed": []} + m_cmp.side_effect = PidProviderXMLPidV3ConflictError("conflict!") + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + m_save.assert_not_called() + + self.assert_recorded_status("conflict") + self.assertIn("error_msg", response) + self.assertIn("error_type", response) + + +class ForbiddenPathTest(RegisterTestBase): + def test_forbidden_when_aop_over_vor(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.return_value = {"registered": existing, "failed": []} + m_cmp.return_value = {} + m_upd.side_effect = ( + exceptions.ForbiddenPidProviderXMLRegistrationError("forbidden") + ) + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + m_save.assert_not_called() + + self.assert_recorded_status("forbidden") + self.assertIn("error_msg", response) + + +class UnmatchedPathTest(RegisterTestBase): + def test_unmatched_when_get_records_raises_unmatched(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.side_effect = exceptions.UnmatchedPidProviderXMLError("unmatched") + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("unmatched") + self.assertIn("error_msg", response) + + def test_unmatched_when_failed_without_registered(self): + # get_records retorna dict com failed e sem registered -> register levanta + # UnmatchedPidProviderXMLError internamente + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.return_value = {"registered": None, "failed": [{"id": 1}]} + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("unmatched") + + def test_multiple_objects_returned_is_unmatched(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.side_effect = PidProviderXML.MultipleObjectsReturned() + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("unmatched") + + +class BadRequestPathTest(RegisterTestBase): + """ + ATENÇÃO — MUDANÇA DE CONTRATO: + Na versão atual, as exceções de bad_request NÃO propagam mais ao chamador; + viram response com event_status='bad_request'. Estes testes DOCUMENTAM o + comportamento atual. Se a decisão for propagar (opção B), troque por + assertRaises e remova o return no finally. + """ + + def test_required_issn_becomes_response_not_raise(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.side_effect = ( + exceptions.RequiredISSNErrorToGetPidProviderXMLError("no issn") + ) + # NÃO levanta — retorna response + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("bad_request") + self.assertIn("error_msg", response) + + def test_required_pub_year_becomes_response(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.side_effect = ( + exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError("no year") + ) + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("bad_request") + + def test_not_enough_parameters_becomes_response(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.side_effect = ( + exceptions.NotEnoughParametersToGetPidProviderXMLError("not enough") + ) + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("bad_request") + + +class ErrorPathTest(RegisterTestBase): + def test_unexpected_exception_records_error_and_logs_unexpected_event(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.UnexpectedEvent.create") as m_unexpected: + + m_get.side_effect = ValueError("algo totalmente inesperado") + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("error") + self.assertIn("error_msg", response) + # erro inesperado deve registrar UnexpectedEvent + m_unexpected.assert_called_once() + + +class RecordAlwaysCalledTest(RegisterTestBase): + """Garante a premissa 'grava SEMPRE': record é chamado exatamente 1 vez.""" + + def test_record_called_exactly_once_on_success(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.side_effect = PidProviderXML.DoesNotExist + m_cmp.return_value = {} + m_upd.return_value = None + saved = MagicMock() + saved.data = {"v3": "ABC"} + m_save.return_value = (saved, "created") + + PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assertEqual(self.m_record.call_count, 1) + + def test_record_called_exactly_once_on_conflict(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp: + m_get.return_value = {"registered": MagicMock(), "failed": []} + m_cmp.side_effect = PidProviderXMLPidV3ConflictError("x") + PidProviderXML.register(self.xml, "file.xml", self.user) + + # antes havia risco de gravar 2x (except + finally); deve ser 1 + self.assertEqual(self.m_record.call_count, 1) \ No newline at end of file diff --git a/pid_provider/wagtail_hooks.py b/pid_provider/wagtail_hooks.py index 550d167eb..08b1b8329 100644 --- a/pid_provider/wagtail_hooks.py +++ b/pid_provider/wagtail_hooks.py @@ -6,8 +6,7 @@ from config.menu import get_menu_order from core.views import CommonControlFieldViewSet -from pid_provider import choices -from pid_provider.models import XMLURL, XMLVersion, FixPidV2, OtherPid, PidProviderConfig, PidProviderXML +from pid_provider.models import XMLURL, XMLVersion, FixPidV2, OtherPid, PidProviderConfig, PidProviderXML, PidProviderXMLRegistration class PidProviderXMLViewSet(CommonControlFieldViewSet): @@ -204,6 +203,38 @@ class XMLURLViewSet(CommonControlFieldViewSet): ) +class PidProviderXMLRegistrationViewSet(CommonControlFieldViewSet): + model = PidProviderXMLRegistration + icon = "doc-empty-inverse" + menu_label = _("PID Registration Events") + menu_name = "pid_provider_xml_registration" + add_to_admin_menu = True + + # ordenação na listagem + ordering = ["-created"] + + # colunas da listagem + list_display = ( + "pkg_name", + "event_status", + "pid_provider_xml", + "error_type", + "created", + ) + + # filtros laterais + list_filter = ("event_status", "error_type", "created") + + # busca + search_fields = ("pkg_name", "error_type", "pid_provider_xml__v3") + + # paginação (tabela cresce em volume) + list_per_page = 50 + + # ---- READ ONLY: auditoria não é editada pela UI ---- + # def get_queryset(self): + # return super().get_queryset().select_related("pid_provider_xml") + # Grupo de ViewSets class PidProviderViewSetGroup(SnippetViewSetGroup): menu_label = _("Pid Provider") @@ -216,6 +247,7 @@ class PidProviderViewSetGroup(SnippetViewSetGroup): PidProviderConfigViewSet, XMLVersionViewSet, XMLURLViewSet, + PidProviderXMLRegistrationViewSet, ) diff --git a/requirements/base.txt b/requirements/base.txt index ff4a4eea1..0bb791a54 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -90,7 +90,7 @@ iso639-lang==2.6.3 # Mantendo versão maior # SciELO Specific Packages # ======================================== # Using specific versions for stability --e git+https://github.com/scieloorg/packtools.git@4.16.4#egg=packtools +-e git+https://github.com/scieloorg/packtools.git@4.16.6#egg=packtools -e git+https://github.com/scieloorg/scielo_scholarly_data#egg=scielo_scholarly_data -e git+https://github.com/scieloorg/opac_schema.git@v2.66#egg=opac_schema -e git+https://github.com/scieloorg/scielo_migration.git@1.10.8#egg=scielo_classic_website