diff --git a/article/choices.py b/article/choices.py index ecaf8afed..e3e67cbd5 100644 --- a/article/choices.py +++ b/article/choices.py @@ -131,3 +131,17 @@ (AS_SCHEDULED_TO_PUBLISH, _("Scheduled to publish")), (AS_PUBLISHED, _("Published")), ) + + +ARTICLE_WEBPAGE_STATUS_UNAVAILABLE = "unavailable" +ARTICLE_WEBPAGE_STATUS_AVAILABLE = "available" +ARTICLE_WEBPAGE_STATUS_VALID_CONTENT = "valid_content" +ARTICLE_WEBPAGE_STATUS_CONTENT_MISMATCH = "content_mismatch" +ARTICLE_WEBPAGE_STATUS_NOT_CHECKED = "not-checked" +ARTICLE_WEBPAGE_STATUS = ( + (ARTICLE_WEBPAGE_STATUS_AVAILABLE, _("Available")), + (ARTICLE_WEBPAGE_STATUS_UNAVAILABLE, _("Unavailable")), + (ARTICLE_WEBPAGE_STATUS_VALID_CONTENT, _("Valid content")), + (ARTICLE_WEBPAGE_STATUS_CONTENT_MISMATCH, _("Content mismatch")), + (ARTICLE_WEBPAGE_STATUS_NOT_CHECKED, _("Not checked")), +) diff --git a/article/migrations/0009_articlewebpage.py b/article/migrations/0009_articlewebpage.py new file mode 100644 index 000000000..c6738dfa8 --- /dev/null +++ b/article/migrations/0009_articlewebpage.py @@ -0,0 +1,138 @@ +# Generated by Django 5.2.3 on 2026-05-12 23:57 + +import django.db.models.deletion +import modelcluster.fields +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("article", "0008_add_unique_pid_v3"), + ("collection", "0005_collection_url"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="ArticleWebPage", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + auto_now_add=True, verbose_name="Creation date" + ), + ), + ( + "updated", + models.DateTimeField( + auto_now=True, verbose_name="Last update date" + ), + ), + ( + "fmt", + models.CharField( + blank=True, max_length=4, null=True, verbose_name="Format" + ), + ), + ( + "url", + models.CharField( + blank=True, max_length=265, null=True, verbose_name="URL" + ), + ), + ( + "status", + models.CharField( + blank=True, + choices=[ + ("available", "Available"), + ("unavailable", "Unavailable"), + ("valid_content", "Valid content"), + ("content_mismatch", "Content mismatch"), + ("not-checked", "Not checked"), + ], + default="not-checked", + max_length=16, + null=True, + verbose_name="Status", + ), + ), + ( + "detail", + models.JSONField(blank=True, null=True, verbose_name="Detail"), + ), + ( + "article", + modelcluster.fields.ParentalKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="article_webpages", + to="article.article", + ), + ), + ( + "collection", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="collection.collection", + verbose_name="Collection", + ), + ), + ( + "creator", + models.ForeignKey( + editable=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_creator", + to=settings.AUTH_USER_MODEL, + verbose_name="Creator", + ), + ), + ( + "lang", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="collection.language", + verbose_name="Language", + ), + ), + ( + "updated_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_last_mod_user", + to=settings.AUTH_USER_MODEL, + verbose_name="Updater", + ), + ), + ], + options={ + "indexes": [ + models.Index(fields=["url"], name="article_art_url_152ec8_idx"), + models.Index( + fields=["article", "collection", "fmt", "lang"], + name="article_art_article_b0d342_idx", + ), + ], + "unique_together": {("article", "collection", "url", "fmt", "lang")}, + }, + ), + ] diff --git a/article/models.py b/article/models.py index 9fd135804..378fc41c2 100644 --- a/article/models.py +++ b/article/models.py @@ -16,8 +16,10 @@ from wagtailautocomplete.edit_handlers import AutocompletePanel from migration.models import MigratedArticle +from article.page_checker import check_url, check_content, format_url, format_classic_url from article.forms import ArticleForm, RelatedItemForm, RequestArticleChangeForm -from collection.models import Language +from collection.choices import PUBLIC +from collection.models import Language, Collection from core.models import CommonControlField, HTMLTextModel from doi.models import DOIWithLang from issue.models import TOC, Issue, TocSection @@ -25,7 +27,6 @@ from package.models import SPSPkg from pid_provider.models import PidProviderXML from pid_provider.choices import PPXML_STATUS_INVALID -from tracker import choices as tracker_choices from . import choices from .permission_helper import MAKE_ARTICLE_CHANGE, REQUEST_ARTICLE_CHANGE @@ -34,9 +35,35 @@ class Article(ClusterableModel, CommonControlField): """ + Modelo que representa um artigo no contexto de Upload. + No contexto de Upload, Article deve conter o mínimo de campos, - suficiente para o processo de ingresso / validações, - pois os dados devem ser obtidos do XML + suficiente para o processo de ingresso/validações, pois os dados + devem ser obtidos do XML. + + Attributes + ---------- + pid_v3 : CharField + Identificador persistente versão 3 (único). + pid_v2 : CharField + Identificador persistente versão 2. + sps_pkg : ForeignKey + Referência ao pacote SPS que contém o artigo. + pp_xml : ForeignKey + Referência ao registro no PidProvider XML. + article_type : CharField + Tipo do artigo (ex: research-article, editorial, etc). + status : CharField + Status do artigo no fluxo de publicação. + issue : ForeignKey + Referência ao fascículo que contém o artigo. + journal : ForeignKey + Referência ao periódico. + + Methods + ------- + create_or_update(user, sps_pkg, issue=None, journal=None, position=None) + Cria ou atualiza um artigo a partir de um pacote SPS. """ pp_xml = models.ForeignKey( @@ -80,6 +107,7 @@ class Article(ClusterableModel, CommonControlField): _("Elocation ID"), max_length=64, blank=True, null=True ) fpage = models.CharField(_("First page"), max_length=16, blank=True, null=True) + fpage_seq = models.CharField(_("First page seq"), max_length=16, blank=True, null=True) lpage = models.CharField(_("Last page"), max_length=16, blank=True, null=True) # External models @@ -114,11 +142,18 @@ class Article(ClusterableModel, CommonControlField): FieldPanel("fpage", read_only=True), FieldPanel("lpage", read_only=True), ] + panel_webpages = MultiFieldPanel( + heading=_("Article webpages"), classname="collapsible" + ) + panel_webpages.children = [ + InlinePanel("article_webpages", label="Webpages"), + ] panels = [ panel_article_ids, panel_article_details, FieldPanel("issue", classname="collapsible", read_only=True), + panel_webpages, ] base_form_class = ArticleForm @@ -145,7 +180,10 @@ def autocomplete_label(self): return str(self) def __str__(self): - return self.sps_pkg.sps_pkg_name + try: + return self.sps_pkg.sps_pkg_name + except AttributeError: + return self.pid_v3 or self.pid_v2 or f"Article {self.pk}" @property def pdfs(self): @@ -204,6 +242,29 @@ def article_langs(self): @classmethod def get(cls, pid_v3): + """ + Obtém um artigo pelo PID v3, removendo duplicatas se necessário. + + Se múltiplos artigos existem com o mesmo pid_v3, mantém o mais + recentemente atualizado e deleta os demais. + + Parameters + ---------- + pid_v3 : str + Identificador persistente versão 3. + + Returns + ------- + Article + Artigo correspondente ao pid_v3. + + Raises + ------ + ValueError + Se pid_v3 não foi informado. + Article.DoesNotExist + Se nenhum artigo é encontrado com o pid_v3. + """ if pid_v3: try: return cls.objects.get(pid_v3=pid_v3) @@ -242,6 +303,36 @@ def delete_items_duplicated_by_sps_pkg_name(cls, sps_pkg_name): @classmethod def create_or_update(cls, user, sps_pkg, issue=None, journal=None, position=None): + """ + Cria ou atualiza um artigo a partir de um pacote SPS. + + Processa o pacote SPS para extrair informações do XML, cria ou + atualiza o artigo e seus relacionamentos (títulos, seções, DOIs, + etc). + + Parameters + ---------- + user : User + Usuário que está criando/atualizando o artigo. + sps_pkg : SPSPkg + Pacote SPS que contém o artigo. + issue : Issue, optional + Fascículo do artigo. Se não informado, será obtido do XML. + journal : Journal, optional + Periódico do artigo. Se não informado, será obtido do XML. + position : int, optional + Posição do artigo no fascículo. + + Returns + ------- + Article + Artigo criado ou atualizado. + + Raises + ------ + ValueError + Se sps_pkg não é informado ou está inválido. + """ if not sps_pkg: raise ValueError("create_article requires sps_pkg with pid_v2") @@ -318,6 +409,12 @@ def add_related_item(self, target_doi, target_article_type): # self.related_items.add(item) def add_pages(self): + """ + Extrai informações de paginação do XML do pacote SPS. + + Popula os campos de primeira página, última página e elocation_id + a partir dos dados extraídos do XML do pacote. + """ xml_with_pre = self.sps_pkg.xml_with_pre self.fpage = xml_with_pre.fpage self.fpage_seq = xml_with_pre.fpage_seq @@ -325,6 +422,17 @@ def add_pages(self): self.elocation_id = xml_with_pre.elocation_id def add_issue(self, user): + """ + Obtém e associa o fascículo do artigo baseado no XML. + + Extrai as informações de volume, suplemento e número do XML + do pacote SPS e localiza ou cria o fascículo correspondente. + + Parameters + ---------- + user : User + Usuário que está realizando a operação. + """ xml_with_pre = self.sps_pkg.xml_with_pre self.issue = Issue.get( journal=self.journal, @@ -334,6 +442,17 @@ def add_issue(self, user): ) def add_journal(self, user): + """ + Obtém e associa o periódico do artigo baseado no XML. + + Extrai as informações ISSN do XML do pacote SPS e localiza + ou cria o periódico correspondente. + + Parameters + ---------- + user : User + Usuário que está realizando a operação. + """ xml_with_pre = self.sps_pkg.xml_with_pre self.journal = Journal.get( official_journal=OfficialJournal.get( @@ -343,6 +462,18 @@ def add_journal(self, user): ) def add_article_titles(self, user): + """ + Extrai e armazena títulos do artigo em múltiplos idiomas. + + Processa os títulos do XML do pacote SPS, cria ou obtém as + linguagens correspondentes, e associa os títulos ao artigo. + Remove títulos anteriores antes de adicionar novos. + + Parameters + ---------- + user : User + Usuário que está realizando a operação. + """ titles = ArticleTitles( xmltree=self.sps_pkg.xml_with_pre.xmltree, ).article_title_list @@ -371,6 +502,19 @@ def add_article_titles(self, user): logging.exception(e) def add_sections(self, user): + """ + Extrai e associa seções do artigo a partir do XML. + + Processa as seções do XML do pacote SPS, cria ou obtém as + linguagens correspondentes, associa as seções ao artigo e + atualiza a tabela de conteúdo (TOC) do fascículo. + Remove seções anteriores antes de adicionar novas. + + Parameters + ---------- + user : User + Usuário que está realizando a operação. + """ self.sections.all().delete() xml_sections = ArticleTocSections( @@ -517,25 +661,48 @@ def update_status(self, new_status=None, rollback=False): def get_zip_filename_and_content(self): return self.sps_pkg.get_zip_filename_and_content() - def get_urls(self, website_url): - journal_acron = self.journal.journal_acron + def get_html_urls(self, website_url, classic=True, new=True, only_first=None): pid_v2 = self.pid_v2 pid_v3 = self.pid_v3 + journal_acron = None + if new: + journal_acron = self.journal.journal_acron for item in self.htmls: lang = item.get("lang") if not lang: continue - yield f"{website_url}/j/{journal_acron}/a/{pid_v3}/?lang={lang}" - yield f"{website_url}/scielo.php?script=sci_arttext&pid={pid_v2}&tlng={lang}" + if classic: + url = format_classic_url(website_url, pid_v2=pid_v2, format="html", lang_code=lang) + yield {"url": url, "format": "html", "lang": lang, "website": "classic"} + if new: + url = format_url(website_url, pid_v3, journal_acron, format="html", lang_code=lang) + yield {"url": url, "format": "html", "lang": lang, "website": "new"} + if only_first: + return + def get_pdf_urls(self, website_url, classic=True, new=True, only_first=None): + pid_v2 = self.pid_v2 + pid_v3 = self.pid_v3 + for item in self.pdfs: lang = item.get("lang") if not lang: continue - yield f"{website_url}/j/{journal_acron}/a/{pid_v3}/?lang={lang}&format=pdf" - yield f"{website_url}/scielo.php?script=sci_pdf&pid={pid_v2}&tlng={lang}" - + if classic: + url = format_classic_url(website_url, pid_v2=pid_v2, format="pdf", lang_code=lang) + yield {"url": url, "format": "pdf", "lang": lang, "website": "classic"} + if new: + journal_acron = self.journal.journal_acron + url = format_url(website_url, pid_v3, journal_acron, format="pdf", lang_code=lang) + yield {"url": url, "format": "pdf", "lang": lang, "website": "new"} + if only_first: + return + + def get_webpage_items(self, website_url): + yield from self.get_html_urls(website_url) + yield from self.get_pdf_urls(website_url) + @classmethod def get_repeated_items(cls, field_name, issue=None): if issue: @@ -552,10 +719,23 @@ def get_repeated_items(cls, field_name, issue=None): @classmethod def exclude_articles_with_invalid_pid_v2(cls, issue=None): """ - Find and delete migrated articles whose pid_v2 last 5 digits - don't match the order (v121) from MigratedArticle.document.order. - Uses ArticleProc.migrated_data to access the migration data. - Only applies to migrated articles. + Remove artigos migrados com pid_v2 inválido. + + Localiza e deleta artigos migrados cujos últimos 5 dígitos do + pid_v2 não correspondem à ordem (v121) do documento em + MigratedArticle.document.order. Usa ArticleProc.migrated_data + para acessar os dados de migração. Aplicável apenas a artigos + migrados. + + Parameters + ---------- + issue : Issue, optional + Se fornecido, filtra apenas artigos deste fascículo. + + Returns + ------- + list[str] + Lista de eventos/mensagens descrevendo as ações realizadas. """ from proc.models import ArticleProc @@ -625,43 +805,59 @@ def exclude_articles_with_invalid_pid_v2(cls, issue=None): @classmethod def exclude_inconvenient_articles(cls, issue, user, timeout=None): """ - Remove all inconvenient article records in a unified operation: - 1. Migrated articles with invalid pid_v2 (suffix doesn't match order from v121) - 2. Duplicate articles (repeated pid_v2 or sps_pkg_name) + Orquestra a limpeza de artigos problemáticos em um fascículo. + + Executa as seguintes operações em fases: + 1. Remove artigos migrados com pid_v2 inválido + 2. Remove duplicatas por pid_v2 e sps_pkg_name + + Parameters + ---------- + issue : Issue + Fascículo a ser processado. + user : User + Usuário que autoriza a limpeza. + timeout : int, optional + Timeout em segundos para verificações HTTP. + + Returns + ------- + dict + Dicionário contendo: + - events: lista de eventos/mensagens de ação + - numbers: contagem de itens repetidos por campo + - exceptions: lista de exceções capturadas durante o processo """ - results = { - "events": [], - "numbers": {}, - "exceptions": [], - } + results = {"events": [], "numbers": {}, "exceptions": []} + # Fase 1: pid_v2 inválidos (artigos migrados com erro) try: events = cls.exclude_articles_with_invalid_pid_v2(issue) results["events"].extend(events) except Exception as e: - results["exceptions"].append( - { - "exclude_articles_with_invalid_pid_v2": str(e), - "traceback": traceback.format_exc(), - } - ) + results["exceptions"].append({ + "step": "exclude_articles_with_invalid_pid_v2", + "error": str(e), + "traceback": traceback.format_exc(), + }) + # Fase 2: duplicatas for field_name in ("pid_v2", "sps_pkg__sps_pkg_name"): - repeated_items = cls.get_repeated_items(field_name, issue) - results["numbers"][f"repeated_by_{field_name}"] = repeated_items.count() - for repeated_value in repeated_items: + repeated_values = list(cls.get_repeated_items(field_name, issue)) + results["numbers"][f"repeated_by_{field_name}"] = len(repeated_values) + + for value in repeated_values: try: events = cls.exclude_repetitions( - user, field_name, repeated_value, timeout=timeout + user, field_name, value, timeout=timeout ) results["events"].extend(events) except Exception as e: - results["exceptions"].append( - { - f"repeated_by_{field_name}": repeated_value, - "traceback": traceback.format_exc(), - } - ) + results["exceptions"].append({ + "step": f"repeated_by_{field_name}", + "value": value, + "traceback": traceback.format_exc(), + }) return results @@ -681,9 +877,10 @@ def is_valid_record(self): self.pp_xml = PidProviderXML.get_by_pid_v3(self.pid_v3) except PidProviderXML.DoesNotExist: return False - sps_pkg__pkg_name = self.sps_pkg.xml_with_pre.sps_pkg_name - pp_xml__pkg_name = self.pp_xml.xml_with_pre.sps_pkg_name - return self.pkg_name == pp_xml__pkg_name == sps_pkg__pkg_name + sps_pkg__pkg_name = self.sps_pkg.sps_pkg_name + sps_pkg__xml_with_pre__pkg_name = self.sps_pkg.xml_with_pre.sps_pkg_name + pp_xml__xml_with_pre__pkg_name = self.pp_xml.xml_with_pre.sps_pkg_name + return sps_pkg__pkg_name == pp_xml__xml_with_pre__pkg_name == sps_pkg__xml_with_pre__pkg_name except Exception as e: if self.pp_xml is not None: if self.pp_xml.proc_status != PPXML_STATUS_INVALID: @@ -734,106 +931,424 @@ def fix_sps_pkg_name(self): @classmethod def exclude_repetitions(cls, user, field_name, field_value, timeout=None): """ - Remove artigos duplicados baseado em um campo específico, - mantendo o artigo mais relevante (publicado e válido tem prioridade). + Remove artigos duplicados para um dado campo. + + Remove artigos duplicados para um campo específico (pid_v2 ou + sps_pkg_name). Mantém exatamente 1 artigo, descartando os demais + mesmo que vários estejam publicados no site público. + + Processo em 4 fases: + 1. Ranking SQL com status persistido (annote para ranking) + 2. Check HTTP apenas dos 2 melhores candidatos + 3. Decisão final com status atualizado + 4. Deleção em bulk dos demais + + Parameters + ---------- + user : User + Usuário responsável pela ação. + field_name : str + Nome do campo a verificar (ex: "pid_v2" ou "sps_pkg__sps_pkg_name"). + field_value : str + Valor do campo que identifica duplicatas. + timeout : int, optional + Timeout em segundos para verificações HTTP. + + Returns + ------- + list[str] + Lista de eventos/mensagens descrevendo o processo. """ repeated_items = cls.objects.filter(**{field_name: field_value}) total_initial = repeated_items.count() - + if total_initial <= 1: - return [f"{field_name}='{field_value}': {total_initial} artigo(s), nenhuma ação necessária"] - + return [ + f"{field_name}='{field_value}': {total_initial} artigo(s), " + "nenhuma ação necessária" + ] + events = [f"{field_name}='{field_value}': {total_initial} artigos encontrados"] - - # Atualiza status de disponibilidade antes de decidir qual manter - cls.update_availability_status(user, timeout, repeated_items) - - # Recarrega queryset após atualização de status + + # ── Fase 1: pré-ranking via SQL ── + candidate_ids = list( + repeated_items + .annotate( + has_valid_public_webpage=models.Exists( + ArticleWebPage.objects.filter( + article_id=models.OuterRef("pk"), + status=choices.ARTICLE_WEBPAGE_STATUS_VALID_CONTENT, + website__purpose=PUBLIC, + ) + ), + has_pp_xml=models.Case( + models.When(pp_xml__isnull=False, then=True), + default=False, + output_field=models.BooleanField(), + ), + ) + .order_by("-has_valid_public_webpage", "-has_pp_xml", "-updated") + .values_list("id", flat=True)[:2] + ) + + # ── Fase 2: check HTTP só dos finalistas ── + # Confirma o estado real antes de decidir. + # Se o 1º candidato cai, o 2º assume. + for article in cls.objects.filter(id__in=candidate_ids): + for check_fn, params in article.get_check_url_and_params( + user, timeout=timeout, force_update=True + ): + try: + check_fn(**params) + except Exception as e: + logging.exception(e) + + events.append( + f"Disponibilidade verificada para {len(candidate_ids)} finalista(s)" + ) + + # ── Fase 3: decisão com status atualizado ── repeated_items = cls.objects.filter(**{field_name: field_value}) - item_to_keep_id = cls.choose_item_to_keep(repeated_items) + if not item_to_keep_id: - # Fallback: mantém o mais recentemente atualizado - item_to_keep_id = repeated_items.order_by('-updated').values_list('id', flat=True).first() - + item_to_keep_id = ( + repeated_items.order_by("-updated") + .values_list("id", flat=True) + .first() + ) + if not item_to_keep_id: events.append("Erro: nenhum item encontrado para manter") return events - + events.append(f"Artigo mantido: ID {item_to_keep_id}") - - # Coleta IDs relacionados em uma única passagem - items_to_delete = repeated_items.exclude(id=item_to_keep_id).select_related('sps_pkg', 'pp_xml') - - sps_pkg_ids = set() - pp_xml_ids = set() - article_ids = [] - - for item in items_to_delete: - article_ids.append(item.id) - if item.sps_pkg_id: - sps_pkg_ids.add(item.sps_pkg_id) - if item.pp_xml_id: - pp_xml_ids.add(item.pp_xml_id) - + + # ── Fase 4: deleção em bulk ── + items_to_delete = repeated_items.exclude(id=item_to_keep_id) + + sps_pkg_ids = set( + items_to_delete + .exclude(sps_pkg__isnull=True) + .values_list("sps_pkg_id", flat=True) + ) + pp_xml_ids = set( + items_to_delete + .exclude(pp_xml__isnull=True) + .values_list("pp_xml_id", flat=True) + ) + article_ids = list(items_to_delete.values_list("id", flat=True)) + total_to_delete = len(article_ids) events.append(f"Artigos a deletar: {total_to_delete}") - + if not total_to_delete: return events - - # Executa deleções em transação atômica + with transaction.atomic(): deleted_articles, _ = cls.objects.filter(id__in=article_ids).delete() events.append(f"Articles deletados: {deleted_articles}") - + if sps_pkg_ids: deleted_sps, _ = SPSPkg.objects.filter(id__in=sps_pkg_ids).delete() events.append(f"SPSPkg deletados: {deleted_sps}") - + if pp_xml_ids: - deleted_pp, _ = PidProviderXML.objects.filter(id__in=pp_xml_ids).delete() + deleted_pp, _ = PidProviderXML.objects.filter( + id__in=pp_xml_ids + ).delete() events.append(f"PidProviderXML deletados: {deleted_pp}") - - return events + return events + @classmethod def choose_item_to_keep(cls, queryset): - result = {} - for item in queryset.order_by("-updated"): - valid = item.is_valid_record() - try: - published = item.availability_status.first().completed - except AttributeError: - published = False - result.setdefault((published, valid), []).append(item) - status = ( - (True, True), - (True, False), - (False, True), - (False, False), + """ + Escolhe qual artigo duplicado deve ser mantido. + + Dado um queryset de artigos duplicados, retorna o ID do único + que deve ser mantido. Entre múltiplos artigos publicados no + site público, escolhe o mais confiável. Os demais serão deletados. + + Critérios de seleção (em ordem de prioridade): + 1. Webpage com conteúdo válido no site público + 2. Registro no PidProvider (pp_xml existe) + 3. Mais recentemente atualizado + + Parameters + ---------- + queryset : QuerySet + QuerySet de artigos duplicados. + + Returns + ------- + int or None + ID do artigo que deve ser mantido, ou None se nenhum encontrado. + """ + return ( + queryset + .annotate( + has_valid_public_webpage=models.Exists( + ArticleWebPage.objects.filter( + article_id=models.OuterRef("pk"), + status=choices.ARTICLE_WEBPAGE_STATUS_VALID_CONTENT, + website__purpose=PUBLIC, + ) + ), + has_pp_xml=models.Case( + models.When(pp_xml__isnull=False, then=True), + default=False, + output_field=models.BooleanField(), + ), + ) + .order_by( + "-has_valid_public_webpage", + "-has_pp_xml", + "-updated", + ) + .values_list("id", flat=True) + .first() ) - for key in status: - items = result.get(key) or [] - if len(items) >= 1: - return items[0].id - @classmethod - def update_availability_status(cls, user, timeout=None, queryset=None, filters=None): - if not queryset: - queryset = cls.objects - if filters: - queryset = queryset.filter(**filters) - else: - queryset = queryset.all() - for item in queryset: - try: - item.availability_status.first().retry(user, timeout, force_update=True) - except Exception: - pass + def create_or_update_urls( + self, + user, + website, + ): + qs = self.article_webpages.filter( + article=self, + website=website, + ) + webpage_items = list(self.get_webpage_items(website.url)) + ids = set() + for item in webpage_items: + lang = item.get("lang") + ids.add( + ArticleWebPage.create_or_update( + user, + self, + website, + item.get("url"), + item.get("format"), + lang, + ).id + ) + qs.exclude(id__in=ids).delete() + + def get_main_article_url(self, website_url): + return format_url(website_url, self.pid_v3, self.journal.journal_acron) + + def get_webpage_id_and_lang_items(self, collection=None, website=None): + params = {} + if collection: + params["website__collection"] = collection + if website: + params["website"] = website + ids = set() + for webpage in self.article_webpages.filter(**params): + ids.add((webpage.id, webpage.lang.code2)) + return ids + + def get_check_url_and_params( + self, + user, + force_update=False, + article_metadata=None, + timeout=None, + website=None, + ): + article_metadata = article_metadata or self.get_metadata_by_lang() + excluded_items = {} + if not force_update: + excluded_items["status"] = choices.ARTICLE_WEBPAGE_STATUS_VALID_CONTENT + + webpages = self.article_webpages.exclude(**excluded_items) + if website: + webpages = webpages.filter(website=website) + for webpage in webpages: + yield webpage.check_availability, { + "user": user, + "timeout": timeout, + "article_metadata": article_metadata.get(webpage.lang.code2), + "force_update": force_update, + } + + def get_metadata_by_lang(self): + langs = self.article_langs + metadata = {} + for lang in langs: + metadata[lang] = self.get_metadata_items(lang) + return metadata + + def get_metadata_items(self, lang=None): + """ + Retorna lista de tuplas (label, valor) com os metadados do artigo. + + Extrai os metadados do artigo (títulos, DOIs, seções, PIDs, + paginação, autores) pronta para ser usada com check_metadata ou + compute_rate. + + Parameters + ---------- + lang : str, optional + Código de idioma (ex: "pt", "en", "es"). + Quando informado, filtra títulos, DOIs e seções pelo idioma. + Campos sem idioma (PIDs, paginação, autores) são sempre incluídos. + + Returns + ------- + list[tuple] + Lista de tuplas (label, valor) com os metadados. + Exemplo: + [ + ("title.1", "Acesso aberto..."), + ("doi.1", "10.1590/..."), + ("author.1", "Maria da Silva"), + ("section.1", "Artigos Originais"), + ("pid_v2", "S0001-37652021000100101"), + ... + ] + """ + items = [] + lang_filter = {"language__code2": lang} if lang else {} + + # Títulos + for i, title in enumerate(self.title_with_lang.filter(**lang_filter), 1): + if title.text: + items.append((f"title.{i}", title.text)) + + # DOIs + doi_lang_filter = {"lang__code2": lang} if lang else {} + for i, doi in enumerate(self.doi_with_lang.filter(**doi_lang_filter), 1): + if doi.doi: + items.append((f"doi.{i}", doi.doi)) + + # Seções + for i, section in enumerate(self.sections.filter(**lang_filter), 1): + if section.text: + items.append((f"section.{i}", section.text)) + + # PIDs + if self.pid_v2: + items.append(("pid_v2", self.pid_v2)) + if self.pid_v3: + items.append(("pid_v3", self.pid_v3)) + + # Paginação / elocation + if self.elocation_id: + items.append(("elocation_id", self.elocation_id)) + if self.fpage: + items.append(("fpage", self.fpage)) + if self.lpage: + items.append(("lpage", self.lpage)) + + # Autores (do XML via sps_pkg — independente de idioma) + try: + xmltree = self.sps_pkg.xml_with_pre.xmltree + contribs = xmltree.findall( + ".//front/article-meta/contrib-group/contrib[@contrib-type='author']" + ) + for i, contrib in enumerate(contribs, 1): + # usa somente surname pois não é possível garantir a ordem de given-names e surname + surname = contrib.findtext("name/surname") or "" + if surname: + items.append((f"author.{i}", surname)) + except (AttributeError, TypeError): + pass + + return items + + def check_webpages_availability(self, user, website, timeout=None, force_update=None): + article_metadata_by_lang = self.get_metadata_by_lang() + for webpage in self.article_webpages.filter(website=website): + article_metadata = article_metadata_by_lang.get(webpage.lang.code2) + webpage.check_availability( + user, timeout, article_metadata, force_update) + + def all_webpages_available(self, collection=None, website=None): + # collection (PUBLIC e QA) + # website (PUBLIC ou QA) + params = {} + if website: + params["website"] = website + if collection: + params["website__collection"] = collection + if self.article_webpages.exists(): + return not self.article_webpages.exclude( + status=choices.ARTICLE_WEBPAGE_STATUS_VALID_CONTENT + ).filter(**params).exists() + return False + + def any_webpage_available(self, website=None, website_id=None, collection=None): + # está presente em algum website + params = {} + if website_id: + params["website_id"] = website_id + if website: + params["website"] = website + if collection: + params["website__collection"] = collection + return self.article_webpages.filter( + status=choices.ARTICLE_WEBPAGE_STATUS_VALID_CONTENT, + **params, + ).exists() + + def public_webpages(self, collection=None): + # está presente em no PUBLIC website + params = {} + if collection: + params["website__collection"] = collection + params["website__purpose"] = PUBLIC + return self.article_webpages.filter( + status=choices.ARTICLE_WEBPAGE_STATUS_VALID_CONTENT, + **params + ).exists() + + def get_availability_stats(self, collection=None, website=None): + params = {} + if website: + params["website"] = website + if collection: + params["website__collection"] = collection + qs = self.article_webpages.filter(**params) + total = len(qs) + if not total: + return {} + stats = ( + qs.values("status") + .annotate(count=Count("id")) + ) + return { + item["status"]: round(item["count"] * 100 / total, 1) + for item in stats + } class ArticleDOIWithLang(Orderable, DOIWithLang): + """ + Modelo que associa DOIs com idiomas específicos a um artigo. + + Permite que um artigo tenha múltiplos DOIs, cada um vinculado a um + idioma específico. + + Attributes + ---------- + article : ParentalKey + Referência ao artigo relacionado. + doi : CharField + Identificador de objeto digital (herança de DOIWithLang). + lang : ForeignKey + Idioma associado ao DOI (herança de DOIWithLang). + + Methods + ------- + get(article=None, doi=None, lang=None) + Obtém DOI(s) com base nos parâmetros. + create(user, article=None, doi=None, lang=None) + Cria um novo DOI para o artigo. + get_or_create(user, article=None, doi=None, lang=None) + Obtém ou cria um DOI para o artigo. + """ article = ParentalKey( "Article", on_delete=models.CASCADE, related_name="doi_with_lang" ) @@ -906,6 +1421,20 @@ def get_or_create(cls, user, article=None, doi=None, lang=None): class ArticleTitle(HTMLTextModel, CommonControlField): + """ + Modelo que armazena títulos de artigos em diferentes idiomas. + + Associa um título HTML a um artigo e seu idioma correspondente. + + Attributes + ---------- + parent : ParentalKey + Referência ao artigo relacionado. + text : TextField + Texto do título em HTML (herança de HTMLTextModel). + language : ForeignKey + Idioma do título (herança de HTMLTextModel). + """ parent = ParentalKey( "Article", on_delete=models.CASCADE, related_name="title_with_lang" ) @@ -917,6 +1446,27 @@ class ArticleTitle(HTMLTextModel, CommonControlField): class RelatedItem(CommonControlField): + """ + Modelo que representa relacionamentos entre artigos. + + Define relacionamentos entre um artigo de origem e um artigo de + destino, com um tipo de relacionamento específico (ex: erratum, + correction, etc). + + Attributes + ---------- + item_type : CharField + Tipo do relacionamento (ex: erratum, correction, original-article). + source_article : ForeignKey + Artigo de origem do relacionamento. + target_article : ForeignKey + Artigo de destino do relacionamento. + + Methods + ------- + __str__() + Retorna representação textual do relacionamento. + """ item_type = models.CharField( _("Related item type"), max_length=32, @@ -952,6 +1502,26 @@ def __str__(self): class RequestArticleChange(CommonControlField): + """ + Modelo que representa solicitações de alteração em artigos. + + Permite solicitar alterações/atualizações em um artigo, como + errata ou correções, com comentários detalhados sobre a mudança. + + Attributes + ---------- + change_type : CharField + Tipo de alteração solicitada (ex: erratum, update, correction). + comment : TextField + Comentários descrevendo a alteração solicitada. + article : ForeignKey + Artigo que será alterado. + + Methods + ------- + __str__() + Retorna representação textual da solicitação. + """ change_type = models.CharField( _("Change type"), @@ -975,3 +1545,212 @@ def __str__(self) -> str: return f"{self.article}" base_form_class = RequestArticleChangeForm + + +class ArticleWebPage(CommonControlField): + """ + Modelo que representa páginas web de artigos publicados. + + Armazena URLs de apresentação de artigos em websites específicos, + com informações de formato, idioma e status de disponibilidade. + + Attributes + ---------- + article : ParentalKey + Referência ao artigo. + website : ForeignKey + Configuração do website onde o artigo é publicado. + fmt : CharField + Formato do conteúdo (ex: "html", "pdf"). + lang : ForeignKey + Idioma da página. + url : CharField + URL da página. + status : CharField + Status da página (disponível, indisponível, conteúdo inválido, etc). + detail : JSONField + Detalhes do último check de disponibilidade. + + Methods + ------- + get(article, website, url, fmt, lang=None) + Obtém uma página específica. + create(user, article, website, url, fmt, lang) + Cria uma nova página. + create_or_update(user, article, website, url, fmt, lang) + Cria ou atualiza uma página. + check_availability(user, timeout, article_metadata=None, force_update=None) + Verifica disponibilidade da URL e valida conteúdo. + """ + article = ParentalKey( + Article, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="article_webpages", + ) + website = models.ForeignKey( + 'collection.WebSiteConfiguration', + verbose_name=_("Website"), + null=True, + blank=True, + on_delete=models.SET_NULL, + help_text=_("Website configuration where this article webpage is published"), + ) + fmt = models.CharField(_("Format"), max_length=4, null=True, blank=True) + lang = models.ForeignKey( + Language, + verbose_name=_("Language"), + null=True, + blank=True, + on_delete=models.SET_NULL, + ) + url = models.CharField(_("URL"), max_length=265, null=True, blank=True) + status = models.CharField( + _("Status"), max_length=16, null=True, blank=True, choices=choices.ARTICLE_WEBPAGE_STATUS, + default=choices.ARTICLE_WEBPAGE_STATUS_NOT_CHECKED, + ) + detail = models.JSONField(_("Detail"), null=True, blank=True) + + panels = [ + FieldPanel("article", read_only=True), + FieldPanel("website", read_only=True), + FieldPanel("url", read_only=True), + FieldPanel("fmt", read_only=True), + FieldPanel("lang", read_only=True), + FieldPanel("status", read_only=True), + FieldPanel("detail", read_only=True), + ] + + class Meta: + unique_together = ("article", "website", "url", "fmt", "lang") + indexes = [ + models.Index(fields=["url"]), + models.Index(fields=["article", "website", "fmt", "lang"]), + ] + + @classmethod + def get(cls, article, website, url, fmt, lang=None): + return cls.objects.get( + article=article, + website=website, + url=url, + fmt=fmt, + lang=lang, + ) + + @classmethod + def create( + cls, + user, + article, + website, + url, + fmt, + lang, + ): + try: + obj = cls( + article=article, + website=website, + url=url, + fmt=fmt, + lang=lang, + creator=user, + ) + obj.save() + return obj + except IntegrityError: + return cls.get(article, website, url, fmt, lang) + + @classmethod + def create_or_update( + cls, + user, + article, + website, + url, + fmt, + lang, + ): + try: + if lang: + lang = Language.objects.filter(code2=lang).first() + return cls.get(article, website, url, fmt, lang) + except cls.DoesNotExist: + return cls.create( + user=user, + article=article, + website=website, + url=url, + fmt=fmt, + lang=lang, + ) + + def check_availability(self, user, timeout, article_metadata=None, force_update=None): + """ + Verifica disponibilidade e validade do conteúdo da página. + + Acessa a URL da página, verifica se está disponível e valida + se o conteúdo contém os metadados esperados do artigo. Atualiza + o status da página com o resultado da verificação. + + Parameters + ---------- + user : User + Usuário que está realizando a verificação. + timeout : int + Timeout em segundos para a requisição HTTP. + article_metadata : list[tuple], optional + Metadados do artigo para validação de conteúdo. + Se não fornecido, será extraído do artigo. + force_update : bool, optional + Se True, ignora status anterior e refaz a verificação. + + Returns + ------- + dict + Dicionário com detalhes da verificação (URL, status, erros, taxa de acurácia). + """ + detail = { + "url": self.url, + "format": self.fmt, + "lang": self.lang.code2 if self.lang else None, + "status": self.status, + "force_update": force_update, + } + + if self.status == choices.ARTICLE_WEBPAGE_STATUS_VALID_CONTENT: + if not force_update: + return detail + + try: + response = check_url(self.url, timeout) + content = response.get("content") + if not content: + raise ValueError("No content retrieved from URL") + + self.status = choices.ARTICLE_WEBPAGE_STATUS_AVAILABLE + + if not article_metadata: + article_metadata = self.article.get_metadata_items(self.lang.code2 if self.lang else None) + if not article_metadata: + raise ValueError("No article metadata available for content check") + + response = check_content(article_metadata, content) + if response.get("error"): + raise ValueError(response["error"]) + rate = response.get("rate", 0) + if rate > 0.8: + self.status = choices.ARTICLE_WEBPAGE_STATUS_VALID_CONTENT + else: + self.status = choices.ARTICLE_WEBPAGE_STATUS_CONTENT_MISMATCH + detail.update(response) + except Exception as e: + detail = {"error": str(e)} + self.status = choices.ARTICLE_WEBPAGE_STATUS_UNAVAILABLE + self.detail = detail + self.updated_by = user + self.save() + detail["status"] = self.status + return detail \ No newline at end of file diff --git a/article/page_checker.py b/article/page_checker.py new file mode 100644 index 000000000..b6b63d9fc --- /dev/null +++ b/article/page_checker.py @@ -0,0 +1,130 @@ +""" +Verifica a presença exata de metadados de artigo em um texto. +""" + +import re +import unicodedata +from core.utils.requester import fetch_data + + +def format_url(public_website_url, pid_v3, journal_acron, format, lang_code=None): + url = f"{public_website_url}/j/{journal_acron}/a/{pid_v3}/" + if format or lang_code: + url += "?" + if format: + url += f"format={format}" + if lang_code: + if format: + url += "&" + url += f"lang={lang_code}" + return url + + +def format_classic_url(website_url, pid_v2, format, lang_code=None): + if format == "pdf": + return f"{website_url}/scielo.php?script=sci_pdf&pid={pid_v2}&tlng={lang_code}" + return f"{website_url}/scielo.php?script=sci_arttext&pid={pid_v2}&tlng={lang_code}" + + +def check_url(url, timeout): + try: + if not url: + raise ValueError("check_page_url_and_content: URL is required for availability check.") + content = fetch_data(url, timeout=timeout or 30) + if not content: + raise ValueError("check_page_url_and_content: No content fetched from URL.") + return {"content": content} + except Exception as e: + return {"error": str(e)} + + +def check_content(article_metadata, content): + try: + if not article_metadata: + raise ValueError("check_page_url_and_content: Article metadata is required for availability check.") + if not content: + raise ValueError("check_page_url_and_content: Content is required for availability check.") + try: + content = content.split(" 0.8: + response["pid_status"] = migration_choices.PID_STATUS_CLASSIC_MATCHED + else: + response["pid_status"] = migration_choices.PID_STATUS_CLASSIC_MISMATCHED + return response + + def set_pid_status(self, user, pid_status): + if pid_status != self.pid_status: + self.pid_status = pid_status + self.updated_by = user + self.save() + + def available_status_delete(self, website_url): + """ + Remove registros legados de ArticleAvailability/ScieloURLStatus. + Chamado após migração bem-sucedida para ArticleWebPage. + """ + try: + article = self.article + if not article: + return + # availability_status é related_name (ForeignKey unique) + aa = getattr(article, 'availability_status', None) + if aa: + if website_url: + aa.scielo_url.filter(url__startswith=website_url).delete() + if not aa.scielo_url.exists(): + aa.delete() + except Exception: + pass # legado pode já ter sido removido + + def all_webpage_available(self, website=None, website_id=None): + article = self.article + if not article: + return False + params = {} + if website: + params["website"] = website + if website_id: + params["website_id"] = website_id + + qs = self.article.article_webpages.filter( + **params, + ) + if not qs.exists(): + return False + return not qs.exclude( + status=article_choices.ARTICLE_WEBPAGE_STATUS_VALID_CONTENT + ).exists() + + def any_webpage_available(self, website): + collection = self.collection + article = self.article + qs = self.article.article_webpages.filter( + website=website, + ) + return qs.filter(status=article_choices.ARTICLE_WEBPAGE_STATUS_AVAILABLE).exists() + diff --git a/proc/source_classic_website.py b/proc/source_classic_website.py index 2a29c0ae8..ae65c5e0f 100644 --- a/proc/source_classic_website.py +++ b/proc/source_classic_website.py @@ -113,84 +113,6 @@ def create_or_update_migrated_issue( ) -def create_collection_procs_from_pid_list( - user, - collection, - pid_list_path, - force_update, -): - """ - Cria procs de collection baseado numa lista de PIDs. - Processa PIDs de artigos, issues e journals de forma hierárquica. - """ - has_changes = controller.id_file_has_changes( - user, - collection, - pid_list_path, - force_update, - ) - if not has_changes: - logging.info(f"skip reading {pid_list_path}") - return - - try: - pid = None - journal_pids = set() - issue_pids = set() - with open(pid_list_path, "r") as fp: - pids = fp.readlines() - - for pid in pids: - pid = pid.strip() or "" - if not len(pid) == 23: - continue - - # Registra PID do artigo - ArticleProc.register_pid( - user, - collection, - pid, - force_update=False, - ) - - # Extrai e registra PID do issue - issue_pid = pid[1:-5] - if issue_pid not in issue_pids: - issue_pids.add(issue_pid) - IssueProc.register_pid( - user, - collection, - issue_pid, - force_update=False, - ) - - # Extrai e registra PID do journal - journal_pid = pid[1:10] - if journal_pid not in journal_pids: - journal_pids.add(journal_pid) - JournalProc.register_pid( - user, - collection, - journal_pid, - force_update=False, - ) - - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - UnexpectedEvent.create( - e=e, - exc_traceback=exc_traceback, - detail={ - "task": "proc.sources.classic_website.create_collection_procs_from_pid_list", - "user_id": user.id, - "username": user.username, - "collection": collection.acron, - "pid_list_path": pid_list_path, - "force_update": force_update, - }, - ) - - def migrate_journal( user, journal_proc, diff --git a/proc/tasks.py b/proc/tasks.py index 3dec57af9..85481b826 100644 --- a/proc/tasks.py +++ b/proc/tasks.py @@ -1,3 +1,56 @@ +""" +Tasks Celery do módulo proc. + +Organização hierárquica das tasks de migração e publicação: + + Journals: + task_migrate_and_publish_journals + └─ task_migrate_and_publish_journals_by_collection (por coleção) + task_publish_journals + └─ task_publish_journal (por periódico) + + Issues: + task_migrate_and_publish_issues + └─ task_migrate_and_publish_issues_by_collection (por coleção) + task_publish_issues + └─ task_publish_issue (por fascículo) + + Articles: + task_migrate_and_publish_articles + └─ task_migrate_and_publish_articles_by_journal (por periódico) + └─ task_migrate_and_publish_articles_by_issue (por fascículo) + └─ task_publish_issue_articles (publica artigos + sincroniza issue) + ├─ task_publish_article (por artigo, síncrono) + │ └─ task_check_article_webpages (verifica disponibilidade) + │ ├─ task_check_article_page_availability (por webpage, síncrono) + │ └─ task_update_article_proc_availability (callback) + └─ task_sync_issue (sincroniza fascículo no site) + + Publicação avulsa (somente publicação, sem migração): + task_publish_articles + └─ task_publish_issue_articles (por fascículo) + + Verificação de disponibilidade (em lote): + task_check_articles_availability + └─ task_check_article_webpages (por artigo × website) + ├─ task_check_article_page_availability (por webpage) + └─ task_update_article_proc_availability (callback) + + Rastreamento de PIDs do site clássico: + task_track_classic_website_article_pids + └─ task_track_classic_website_article_pids_for_collection (por coleção) + └─ task_track_article_page_url_and_content (por artigo) + + Verificação no site clássico (migração): + task_check_classic_website_article + + Utilitários: + task_fetch_and_create_journal + task_exclude_invalid_issue_articles + task_remove_duplicate_issues + task_check_main_article_page_availability +""" + import logging import sys import traceback @@ -7,38 +60,63 @@ from django.db.models import Q from django.utils.translation import gettext_lazy as _ -from article.models import Article +from article.models import Article, ArticleWebPage +from article import choices as article_choices from journal.models import Journal from issue.models import Issue from collection.choices import PUBLIC, QA from collection.models import Collection, WebSiteConfiguration from config import celery_app from migration import controller - +from migration import choices as migration_choices from proc.controller import ( - create_collection_procs_from_pid_list, create_or_update_migrated_issue, create_or_update_migrated_journal, fetch_and_create_journal, migrate_issue, ) -from proc.article_controller import track_classic_website_article_pids +from proc.article_controller import ClassicWebsiteArticlePidTracker from proc.models import ArticleProc, IssueProc, JournalProc from publication.api.document import publish_article from publication.api.issue import publish_issue, sync_issue from publication.api.journal import publish_journal -from publication.api.publication import get_api, get_api_data -from publication.models import ArticleAvailability +from publication.api.publication import get_api_data from tracker import choices as tracker_choices from tracker.models import TaskTracker, UnexpectedEvent User = get_user_model() class NothingToProcess(Exception): + """Sinaliza que não há itens pendentes para processamento.""" ... class TaskExecution: + """ + Wrapper para TaskTracker que acumula eventos, estatísticas e exceções + durante a execução de uma task, e persiste tudo ao finalizar. + + Uso típico:: + + task_exec = TaskExecution(name="minha.task", item="col-jrn", params={...}) + try: + # lógica da task + task_exec.total_to_process = n + for item in items: + process(item) + task_exec.total_processed += 1 + task_exec.finish() + except Exception as e: + task_exec.finish(exception=e, exc_traceback=...) + + Attributes: + params: Dicionário de parâmetros da task (persistido no detail). + task_tracker: Instância de TaskTracker subjacente. + events: Lista de strings descritivas acumuladas durante execução. + stats: Dicionário nome→número com métricas coletadas. + exceptions: Lista de dicionários {"type": ..., "message": ...}. + """ + def __init__(self, name, item, params): self.params = params self.task_tracker = TaskTracker.create( @@ -74,18 +152,32 @@ def total_processed(self, value): self.task_tracker.total_processed = value def add_exception(self, exception): + """Registra uma exceção capturada (sem interromper a task).""" self.exceptions.append({"type": str(type(exception)), "message": str(exception)}) def add_event(self, event): + """Registra um ou mais eventos descritivos (string ou lista de strings).""" if isinstance(event, list): self.events.extend(event) else: self.events.append(event) def add_number(self, name, number): + """Registra uma métrica numérica no dicionário ``stats``.""" self.stats[name] = number def finish(self, exception=None, exc_traceback=None): + """ + Persiste o resultado da execução no TaskTracker. + + Monta o ``detail`` com params, stats, events e exceptions. + Caso o ``detail`` não seja serializável como JSON (ex: objetos + lazy translation), faz fallback convertendo cada valor para string. + + Args: + exception: Exceção capturada (se houver). + exc_traceback: Traceback associado à exceção. + """ if exception or exc_traceback or self.exceptions: completed = False else: @@ -123,6 +215,12 @@ def finish(self, exception=None, exc_traceback=None): def _get_user(user_id, username): + """ + Obtém usuário por ID ou username. + + Retorna None se ambos forem None ou se o usuário não for encontrado. + Em caso de erro, registra UnexpectedEvent e retorna None. + """ try: if user_id: return User.objects.get(pk=user_id) @@ -144,6 +242,11 @@ def _get_user(user_id, username): def _get_collections(collection_acron): + """ + Retorna iterator de coleções filtradas por acrônimo, ou todas se None. + + Em caso de erro, registra UnexpectedEvent e retorna lista vazia. + """ try: if collection_acron: return Collection.objects.filter(acron=collection_acron).iterator() @@ -176,6 +279,10 @@ def task_migrate_and_publish( force_import_acron_id_file=False, force_migrate_document_records=False, ): + """ + Descontinuada. Usar task_migrate_and_publish_journals, + task_migrate_and_publish_issues e task_migrate_and_publish_articles. + """ logging.info("task_migrate_and_publish is discontinued") logging.info("Use task_migrate_and_publish_journals") logging.info("Use task_migrate_and_publish_issues") @@ -195,6 +302,13 @@ def task_migrate_and_publish_journals( valid_status=None, force_import_acron_id_file=False, ): + """ + Ponto de entrada para migração e publicação de periódicos. + + Itera sobre as coleções selecionadas e agenda + ``task_migrate_and_publish_journals_by_collection`` (assíncrono) + para cada uma. + """ try: task_params = { "task": "proc.tasks.task_migrate_and_publish_journals", @@ -238,6 +352,17 @@ def task_migrate_and_publish_journals_by_collection( status=None, force_import_acron_id_file=False, ): + """ + Migra e publica periódicos de uma coleção. + + Etapas: + 1. Importa dados do site clássico (create_or_update_migrated_journal). + 2. Filtra JournalProcs com status pendente (migration, qa_ws, public_ws). + 3. Para cada JournalProc: + a. Cria/atualiza o Journal via controller. + b. Se necessário, sincroniza com a Core API (fetch_and_create_journal). + c. Agenda task_publish_journal para QA e PUBLIC. + """ task_params = { "task": "proc.tasks.task_migrate_and_publish_journals", "user_id": user_id, @@ -374,6 +499,15 @@ def task_publish_journals( force_update=False, verify=False, ): + """ + Agenda publicação de periódicos pendentes nos sites QA e PUBLIC. + + Para cada coleção e website_kind, seleciona JournalProcs com status + pendente (via ``JournalProc.items_to_publish``) e agenda + ``task_publish_journal`` individualmente. + + Não executa migração — apenas publicação. + """ task_params = { "task": "proc.tasks.task_publish_journals", "user_id": user_id, @@ -455,6 +589,11 @@ def task_publish_journal( api_data=None, force_update=None, ): + """ + Publica um periódico individual no site QA ou PUBLIC via API. + + Delega para ``journal_proc.publish(publish_journal, ...)``. + """ try: user = _get_user(user_id, username) journal_proc = JournalProc.objects.get(pk=journal_proc_id) @@ -505,6 +644,13 @@ def task_migrate_and_publish_issues( force_update=False, force_migrate_document_records=False, ): + """ + Ponto de entrada para migração e publicação de fascículos. + + Itera sobre as coleções selecionadas e agenda + ``task_migrate_and_publish_issues_by_collection`` (assíncrono) + para cada uma. + """ task_params = { "user_id": user_id, "username": username, @@ -546,6 +692,18 @@ def task_migrate_and_publish_issues_by_collection( force_update=False, force_migrate_document_records=False, ): + """ + Migra e publica fascículos de uma coleção. + + Etapas: + 1. Importa dados de fascículos do site clássico + (create_or_update_migrated_issue). + 2. Filtra IssueProcs com status pendente (migration, docs, files, + qa_ws, public_ws). + 3. Para cada IssueProc: + a. Executa migrate_issue (cria/atualiza Issue). + b. Agenda task_publish_issue para QA e PUBLIC. + """ task_params = { "user_id": user_id, "username": username, @@ -671,6 +829,15 @@ def task_publish_issues( force_update=False, verify=False, ): + """ + Agenda publicação de fascículos pendentes nos sites QA e PUBLIC. + + Para cada coleção e website_kind, seleciona IssueProcs com status + pendente (via ``IssueProc.items_to_publish``) e agenda + ``task_publish_issue`` individualmente. + + Não executa migração — apenas publicação. + """ task_params = { "collection_acron": collection_acron, "journal_acron": journal_acron, @@ -747,6 +914,11 @@ def task_publish_issue( api_data=None, force_update=None, ): + """ + Publica um fascículo individual no site QA ou PUBLIC via API. + + Delega para ``issue_proc.publish(publish_issue, ...)``. + """ try: user = _get_user(user_id, username) issue_proc = IssueProc.objects.get(pk=issue_proc_id) @@ -759,6 +931,7 @@ def task_publish_issue( api_data=api_data, force_update=force_update, ) + event.finish() except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() try: @@ -804,6 +977,17 @@ def task_migrate_and_publish_articles( force_migrate_document_files=False, skip_migrate_pending_document_records=False, ): + """ + Ponto de entrada para migração e publicação de artigos. + + Estratégia de seleção: + - Se ``publication_year`` ou ``issue_folder`` fornecidos: seleciona + IssueProcs específicos e agrupa por journal_proc_id. + - Caso contrário: seleciona todos os JournalProcs das coleções. + + Agenda ``task_migrate_and_publish_articles_by_journal`` para cada + periódico identificado. + """ task_params = { "user_id": user_id, "username": username, @@ -915,6 +1099,19 @@ def task_migrate_and_publish_articles_by_journal( force_migrate_document_records=False, force_migrate_document_files=False, ): + """ + Migra e publica artigos de um periódico. + + Etapas: + 1. Importa registros de acron_id do site clássico + (controller.import_journal_acron_id_records). + 2. Identifica fascículos a processar: + - Se ``issue_proc_id_list`` fornecida: usa diretamente. + - Senão: seleciona IssueProcs com status pendente e complementa + com ArticleProcs pendentes de issues já processados. + 3. Agenda ``task_migrate_and_publish_articles_by_issue`` para cada + fascículo. + """ task_params = { "user_id": user_id, @@ -1026,6 +1223,19 @@ def task_migrate_and_publish_articles_by_issue( qa_api_data=None, public_api_data=None, ): + """ + Migra e publica artigos de um fascículo. + + Etapas: + 1. Remove artigos duplicados/inconsistentes via + ``task_exclude_invalid_issue_articles`` (síncrono). + 2. Se ``article_proc_id_list`` fornecida: usa diretamente (pressupõe + que registros e arquivos já foram migrados). + Senão: migra registros e arquivos do site clássico, depois seleciona + ArticleProcs pendentes. + 3. Migra cada artigo (``article_proc.migrate_article``). + 4. Agenda ``task_publish_issue_articles`` para publicação e sincronização. + """ task_params = { "user_id": user_id, "username": username, @@ -1051,21 +1261,23 @@ def task_migrate_and_publish_articles_by_issue( task_exec.item = str(issue_proc) # corrige defeito de repetição de artigos, executando de forma síncrona - task_exclude_article_repetition_by_issue( + task_exclude_invalid_issue_articles( issue_proc_id=issue_proc_id, username=username, user_id=user_id, + public_api_data=public_api_data ) + total_articles_to_process = 0 if article_proc_id_list: # supõe-se que os registros e arquivos já foram migrados # (issue_proc.docs_status e issue_proc.files_status estão como DONE) - total_articles_to_process = len(article_proc_id_list) article_procs = ArticleProc.objects.select_related( "issue_proc", ).filter( id__in=article_proc_id_list ) + total_articles_to_process = article_procs.count() else: task_exec.add_event("Migrate document records") total_migrated_records = issue_proc.migrate_document_records(user, force_migrate_document_records) @@ -1099,29 +1311,16 @@ def task_migrate_and_publish_articles_by_issue( task_exec.add_exception(exceptions[article_proc.pid]) task_exec.total_processed = total_processed - - article_ids_to_publish = ArticleProc.objects.select_related( - "issue_proc", "sps_pkg", - ).filter( - Q(qa_ws_status__in=status) | Q(public_ws_status__in=status), - issue_proc=issue_proc, - sps_pkg__pid_v3__isnull=False, - ).values_list("id", flat=True) - total_articles_to_publish = article_ids_to_publish.count() - task_exec.add_number("total_articles_to_publish", total_articles_to_publish) - - for website_label in (QA, PUBLIC): - task_exec.add_event(f"Schedule Publish articles / sync issue tasks for {website_label}") - task_sync_issue.apply_async( - kwargs=dict( - user_id=user_id, - username=username, - issue_proc_id=issue_proc.id, - website_kind=website_label, - status=status, - force_update=force_update, - ) - ) + task_exec.add_number("total_processed", total_processed) + + task_exec.add_event(f"Schedule article publication {issue_proc} ({total_processed})") + task_publish_issue_articles.delay( + user_id=user_id, + username=username, + issue_proc_id=issue_proc_id, + status=status, + force_update=force_update, + ) task_exec.finish() except Exception as e: @@ -1133,25 +1332,34 @@ def task_migrate_and_publish_articles_by_issue( @celery_app.task(bind=True) -def task_sync_issue( +def task_publish_issue_articles( self, user_id=None, username=None, issue_proc_id=None, - website_kind=None, status=None, force_update=False, ): + """ + Publica artigos de um fascículo e sincroniza o fascículo no site. + + Para cada WebSiteConfiguration habilitado da coleção: + 1. Filtra ArticleProcs com pid_v3 e status pendente no website_kind. + 2. Publica cada artigo via ``task_publish_article`` — chamada direta + (síncrona), não ``.delay()``, para evitar sobrecarga no ambiente + de origem (ex: site clássico). + 3. Agenda ``task_sync_issue`` (assíncrono) para sincronizar o + fascículo no site. + """ task_params = { "user_id": user_id, "username": username, "issue_proc_id": issue_proc_id, - "website_kind": website_kind, "status": status, "force_update": force_update, } task_exec = TaskExecution( - name="proc.tasks.task_sync_issue", + name="proc.tasks.task_publish_issue_articles", item=f"{issue_proc_id}", params=task_params, ) @@ -1161,61 +1369,113 @@ def task_sync_issue( "collection", "journal_proc", "issue" ).get(id=issue_proc_id) - task_exec.item = f"{issue_proc} {website_kind}" + task_exec.item = f"{issue_proc}" status = tracker_choices.get_valid_status(status, force_update) - task_exec.add_event(f"Publishing articles for {website_kind} with status {status}") + task_exec.add_event(f"Publishing {issue_proc} articles which status is {status}") - query_by_status = Q() - if website_kind == QA: - query_by_status = Q(qa_ws_status__in=status) - elif website_kind == PUBLIC: - query_by_status = Q(public_ws_status__in=status) - - article_ids_to_publish = ArticleProc.objects.select_related( + articles = ArticleProc.objects.select_related( "issue_proc", "sps_pkg", ).filter( - query_by_status, issue_proc=issue_proc, sps_pkg__pid_v3__isnull=False, ).values_list("id", flat=True) - task_exec.total_to_process = article_ids_to_publish.count() + collection = issue_proc.collection total_processed = 0 + total_to_process = 0 + for website in WebSiteConfiguration.objects.filter( + collection=collection, + enabled=True, + ): + api_data = website.get_data(content_type="article") + website_kind = website.purpose + + query_by_status = Q() + if website_kind == QA: + query_by_status = Q(qa_ws_status__in=status) + elif website_kind == PUBLIC: + query_by_status = Q(public_ws_status__in=status) + + article_ids_to_publish = articles.filter( + query_by_status + ) + total_to_process += article_ids_to_publish.count() - api_data = get_api_data(issue_proc.collection, "article", website_kind) - if not api_data or api_data.get("error"): - task_exec.add_event(f"API data not available for {website_kind} {api_data}") - task_exec.finish() - return - - for article_proc_id in article_ids_to_publish: - try: + for article_proc_id in article_ids_to_publish: # executa de forma síncrona para evitar muitos processos em paralelo, o que pode causar lentidão e instabilidade no ambiente de origem (ex: site clássico) - task_publish_article( - user_id=user_id, - username=username, - website_kind=website_kind, - article_proc_id=article_proc_id, - api_data=api_data, - force_update=force_update, - ) - total_processed += 1 - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - task_exec.add_exception(traceback.format_exc()) + try: + # publica (síncrono dentro de task_publish_article) + task_publish_article( + user_id=user_id, + username=username, + website_id=website.id, + website_kind=website_kind, + article_proc_id=article_proc_id, + api_data=api_data, + force_update=force_update, + ) + total_processed += 1 + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + task_exec.add_exception(traceback.format_exc()) + + task_exec.add_event(f"Schedule sync_issue {issue_proc} {website_kind}") + task_sync_issue.delay( + user_id=user_id, + username=username, + website_kind=website_kind, + issue_proc_id=issue_proc_id, + api_data=api_data, + ) + task_exec.add_event(f"Scheduled sync_issue {issue_proc} {website_kind}") + task_exec.total_to_process = total_to_process task_exec.total_processed = total_processed + task_exec.finish() + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + task_exec.finish( + exception=e, + exc_traceback=exc_traceback, + ) - api_data = get_api_data(issue_proc.collection, "issue", website_kind) - if not api_data or api_data.get("error"): - task_exec.add_event(f"API data not available for {website_kind} {api_data}") - task_exec.finish() - return - task_exec.add_event(f"Syncing issue in {website_kind} website") +@celery_app.task(bind=True) +def task_sync_issue( + self, + user_id=None, + username=None, + issue_proc_id=None, + website_kind=None, + api_data=None, +): + """ + Sincroniza um fascículo no site (QA ou PUBLIC). + + Chama ``sync_issue(issue_proc, api_data)`` para atualizar o + fascículo no website após a publicação dos artigos. + + Nota: a docstring original repetia incorretamente a descrição de + task_publish_issue_articles. Esta task apenas sincroniza o fascículo. + """ + task_params = { + "user_id": user_id, + "username": username, + "issue_proc_id": issue_proc_id, + "website_kind": website_kind, + } + task_exec = TaskExecution( + name="proc.tasks.task_sync_issue", + item=f"{issue_proc_id}", + params=task_params, + ) + try: + user = _get_user(user_id, username) + issue_proc = IssueProc.objects.get(id=issue_proc_id) + task_exec.item = f"{issue_proc}" + task_exec.add_event(f"Syncing {issue_proc} {website_kind} website") sync_issue(issue_proc, api_data) - task_exec.add_event(f"Issue synced in {website_kind} website") - + task_exec.add_event(f"Issue synced {website_kind} website") task_exec.finish() except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() @@ -1234,9 +1494,20 @@ def task_publish_articles( journal_acron=None, issue_folder=None, publication_year=None, + issue_proc_id=None, force_update=False, + status=None, verify=False, + timeout=None, ): + """ + Agenda publicação de artigos pendentes nos sites QA e PUBLIC. + + Seleciona IssueProcs pelos filtros e agenda + ``task_publish_issue_articles`` para cada um. + + Não executa migração — apenas publicação. + """ task_params = { "user_id": user_id, "username": username, @@ -1247,51 +1518,34 @@ def task_publish_articles( "force_update": force_update, } title = f"{collection_acron}-{journal_acron}-{issue_folder}-{publication_year}" + task_exec = TaskExecution( + name="proc.tasks.task_publish_articles", + item=title, + params=task_params, + ) try: - params = {} - total_scheduled = 0 - - if journal_acron: - params["issue_proc__journal_proc__acron"] = journal_acron - if issue_folder: - params["issue_proc__issue_folder"] = issue_folder - if publication_year: - params["issue_proc__issue__publication_year"] = publication_year - - for collection in _get_collections(collection_acron): - for website_kind in (QA, PUBLIC): - api_data = get_api_data(collection, "article", website_kind) - if not api_data or api_data.get("error"): - continue - api_data["verify"] = verify - - task_exec = TaskExecution( - name="proc.tasks.task_publish_articles", - item=f"{title} {website_kind}", - params=task_params, - ) - items_to_publish = ArticleProc.items_to_publish( - website_kind=website_kind, - content_type="article", - collection=collection, - force_update=force_update, - params=params, - ) - total_scheduled = 0 - task_exec.total_to_process = items_to_publish.count() - for article_proc in items_to_publish: - task_publish_article.delay( - user_id=user_id, - username=username, - website_kind=website_kind, - article_proc_id=article_proc.id, - api_data=api_data, - force_update=force_update, - ) - total_scheduled += 1 - task_exec.total_processed = total_scheduled - task_exec.finish() - + + issue_procs = IssueProc.select_items( + collection_acron=collection_acron, + journal_acron=journal_acron, + issue_folder=issue_folder, + publication_year=publication_year, + issue_proc_id=issue_proc_id, + force_update=force_update, + status_list=status, + ) + total = issue_procs.count() + task_exec.add_event(f"Publishing articles of {total} issues") + + for issue_proc in issue_procs: + task_publish_issue_articles.delay( + user_id=user_id, + username=username, + issue_proc_id=issue_proc.id, + status=status, + force_update=force_update, + ) + task_exec.finish() except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() UnexpectedEvent.create( @@ -1309,10 +1563,21 @@ def task_publish_article( user_id=None, username=None, website_kind=None, + website_id=None, article_proc_id=None, api_data=None, force_update=None, + timeout=None, ): + """ + Publica um artigo individual no site QA ou PUBLIC. + + Etapas: + 1. Publica o artigo via ``article_proc.publish(publish_article, ...)``. + 2. Se publicação bem-sucedida (``response["completed"]``), agenda + ``task_check_article_webpages`` (assíncrono) para verificar + disponibilidade das URLs geradas. + """ user = None detail = {"published": False, "available": False} article_proc = None @@ -1335,19 +1600,15 @@ def task_publish_article( detail["available"] = False if response.get("completed"): - obj = ArticleAvailability.create_or_update( - user, - article_proc.article, - published_by="MIGRATION", - publication_rule="MIGRATION", + task_check_article_webpages.delay( + user_id=user_id, + username=username, + article_proc_id=article_proc_id, + article_id=article_proc.article.id, + website_id=website_id, + timeout=timeout, + force_update=force_update, ) - for website in WebSiteConfiguration.objects.filter( - collection=article_proc.collection, - purpose=website_kind, - ): - obj.create_or_update_urls(user, website.url) - - detail["available"] = obj.completed event.finish(user, detail=detail, completed=True) @@ -1373,65 +1634,6 @@ def task_publish_article( ) -@celery_app.task(bind=True) -def task_create_procs_from_pid_list( - self, username, user_id=None, collection_acron=None, force_update=None -): - user = _get_user(user_id=user_id, username=username) - try: - for collection in _get_collections(collection_acron): - task_create_collection_procs_from_pid_list.apply_async( - kwargs=dict( - username=user.username, - collection_acron=collection.acron, - force_update=force_update, - ) - ) - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - UnexpectedEvent.create( - e=e, - exc_traceback=exc_traceback, - detail={ - "function": "proc.tasks.task_create_procs_from_pid_list", - "collection_acron": collection_acron, - }, - ) - - -@celery_app.task(bind=True) -def task_create_collection_procs_from_pid_list( - self, username, collection_acron, force_update -): - task_params = { - "username": username, - "collection_acron": collection_acron, - "force_update": force_update, - } - task_exec = TaskExecution( - name="proc.tasks.task_create_collection_procs_from_pid_list", - item=f"{collection_acron}", - params=task_params, - ) - try: - user = _get_user(user_id=None, username=username) - classic_website_config = controller.get_classic_website_config(collection_acron) - collection = classic_website_config.collection - create_collection_procs_from_pid_list( - user, - classic_website_config.collection, - classic_website_config.pid_list_path, - force_update, - ) - task_exec.finish() - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - task_exec.finish( - exception=e, - exc_traceback=exc_traceback, - ) - - @celery_app.task(bind=True) def task_fetch_and_create_journal( self, @@ -1442,6 +1644,11 @@ def task_fetch_and_create_journal( issn_print=None, force_update=None, ): + """ + Busca dados de periódico na Core API e cria/atualiza o registro local. + + Delega para ``fetch_and_create_journal()``. + """ task_params = { "task": "proc.tasks.task_fetch_and_create_journal", "user_id": user_id, @@ -1478,28 +1685,32 @@ def task_fetch_and_create_journal( @celery_app.task(bind=True) -def task_exclude_article_repetition_by_issue(self, issue_proc_id, username=None, user_id=None, timeout=None): +def task_exclude_invalid_issue_articles(self, issue_proc_id, username=None, user_id=None, timeout=None, public_api_data=None): """ Remove artigos duplicados e inconsistentes de um fascículo. - Para o IssueProc indicado: - 1. Corrige nomes de sps_pkg de artigos de suplemento do fascículo que - estejam sem o sufixo "-s" (fix_sps_pkg_names). - 2. Exclui artigos "inconvenientes" — duplicatas ou registros que não - devem estar associados ao fascículo (exclude_inconvenient_articles). + Etapas: + 1. Corrige nomes de sps_pkg de artigos de suplemento que estejam + sem o sufixo "-s" (``Article.fix_sps_pkg_names``). + 2. Exclui artigos duplicados ou que não devem estar associados ao + fascículo (``Article.exclude_inconvenient_articles``). Args: issue_proc_id: ID do IssueProc a processar. username: Nome do usuário responsável pela operação. user_id: ID do usuário responsável pela operação. - timeout: Tempo máximo (segundos) para a etapa de exclusão; None = sem limite. + timeout: Tempo máximo (segundos) para a etapa de exclusão; + None = sem limite. + public_api_data: Dados da API pública (não utilizado diretamente + nesta task, mas presente na assinatura por consistência com + o caller). """ task_params = { "issue_proc_id": issue_proc_id, } issue_proc_str = str(issue_proc_id) task_exec = TaskExecution( - name="task_exclude_article_repetition_by_issue", + name="task_exclude_invalid_issue_articles", item=issue_proc_str, params=task_params, ) @@ -1536,7 +1747,7 @@ def task_exclude_article_repetition_by_issue(self, issue_proc_id, username=None, except Exception: UnexpectedEvent.create( item=issue_proc_str, - action="proc.tasks.task_exclude_article_repetition_by_issue", + action="proc.tasks.task_exclude_invalid_issue_articles", e=e, exc_traceback=exc_traceback, detail=task_params, @@ -1551,10 +1762,14 @@ def task_remove_duplicate_issues( journal_id=None, ): """ - Remove Issue duplicados. - - Args: - dry_run: Se True, apenas identifica duplicatas sem remover. + Remove Issues duplicados de um periódico (ou de todos). + + Identifica Issues com mesmos campos-chave via ``Issue.get_duplicates``. + Para cada grupo de duplicatas, mantém o mais recente (por ``updated``) + e para os demais: + - Migra Articles para o Issue mantido. + - Atualiza IssueProc para apontar ao Issue mantido. + - Exclui o Issue duplicado. """ task_params = { "user_id": user_id, @@ -1617,11 +1832,22 @@ def task_remove_duplicate_issues( @celery_app.task(bind=True) def task_track_classic_website_article_pids( - self, username, user_id=None, collection_acron=None, + self, + username, + user_id=None, + collection_acron=None, + timeout=None, ): + """ + Ponto de entrada para rastreamento de PIDs de artigos do site clássico. + + Agenda ``task_track_classic_website_article_pids_for_collection`` + para cada coleção (ou para a coleção especificada). + """ task_params = { "username": username, "collection_acron": collection_acron, + "timeout": timeout, } task_exec = TaskExecution( name="proc.tasks.task_track_classic_website_article_pids", @@ -1630,16 +1856,401 @@ def task_track_classic_website_article_pids( ) try: user = _get_user(user_id=user_id, username=username) + for collection in _get_collections(collection_acron): - classic_website_config = controller.get_classic_website_config( - collection.acron + task_track_classic_website_article_pids_for_collection.delay( + username=username, + user_id=user_id, + collection_acron=collection.acron, + timeout=timeout, ) - result = track_classic_website_article_pids( - user, collection, classic_website_config, + + task_exec.finish() + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + task_exec.finish(exception=e, exc_traceback=exc_traceback) + + +@celery_app.task(bind=True) +def task_track_classic_website_article_pids_for_collection( + self, + username, + user_id=None, + collection_acron=None, + timeout=None, + force_check=None, +): + """ + Rastreia PIDs e verifica URLs/conteúdo dos artigos de uma coleção. + + Etapas: + 1. Reconcilia PIDs do site clássico com ArticleProcs via + ``ClassicWebsiteArticlePidTracker.update_pid_status``. + 2. Para cada artigo com verificação pendente, agenda + ``task_track_article_page_url_and_content`` (assíncrono). + """ + task_params = { + "username": username, + "collection_acron": collection_acron, + "timeout": timeout, + } + task_exec = TaskExecution( + name="proc.tasks.task_track_classic_website_article_pids_for_collection", + item=collection_acron, + params=task_params, + ) + try: + user = _get_user(user_id=user_id, username=username) + + collection = Collection.objects.get(acron=collection_acron) + tracker = ClassicWebsiteArticlePidTracker(user, collection) + result = tracker.update_pid_status() + task_exec.add_event(result) + + for item in ArticleProc.items_to_check_url_and_content(collection, force_check): + task_track_article_page_url_and_content.delay( + user_id=user_id, + username=username, + item_id=item.id, + timeout=timeout, ) - if result: - task_exec.add_event(result) + task_exec.finish() except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() task_exec.finish(exception=e, exc_traceback=exc_traceback) + + +@celery_app.task(bind=True) +def task_check_article_webpages( + self, + user_id=None, + username=None, + article_proc_id=None, + article_id=None, + website_id=None, + timeout=None, + force_update=None, +): + """ + Garante existência de ArticleWebPages e verifica disponibilidade. + + Etapas: + 1. Cria/atualiza webpages para o artigo no website + (``article.create_or_update_urls``). + 2. Calcula metadata por idioma uma vez + (``article.get_metadata_by_lang``). + 3. Para cada webpage pendente, executa + ``task_check_article_page_availability`` (síncrono). + 4. Se ``article_proc_id`` presente (artigos migrados), agenda + ``task_update_article_proc_availability`` (assíncrono) como + callback para atualizar pid_status. + """ + try: + user = _get_user(user_id, username) + article = Article.objects.select_related("journal").get(id=article_id) + + website = WebSiteConfiguration.objects.select_related("collection").get( + id=website_id, + ) + # cria/atualiza webpages (idempotente) + article.create_or_update_urls(user, website) + + # calcula metadata uma vez + article_metadata = article.get_metadata_by_lang() + + # seleciona webpages a verificar + wp_filter = {"website": website} + excluded_items = {} + if not force_update: + excluded_items["status"] = article_choices.ARTICLE_WEBPAGE_STATUS_AVAILABLE + + for webpage in article.article_webpages.filter(**wp_filter).exclude(**excluded_items): + lang_code = webpage.lang.code2 if webpage.lang else None + # executar sincronamente + task_check_article_page_availability( + user_id=user_id, + username=username, + webpage_id=webpage.id, + article_metadata=article_metadata.get(lang_code), + timeout=timeout, + force_update=force_update, + ) + + # callback pós-migração + if article_proc_id: + task_update_article_proc_availability.delay( + user_id=user_id, + username=username, + article_proc_id=article_proc_id, + website_id=website.id, + ) + + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "publication.tasks.task_check_article_webpages", + "article_id": article_id, + "article_proc_id": article_proc_id, + "website_id": website_id, + }, + ) + + +# ============================================================ +# VERIFICAÇÃO ATÔMICA POR WEBPAGE +# ============================================================ + +@celery_app.task(bind=True) +def task_check_article_page_availability( + self, + user_id=None, + username=None, + webpage_id=None, + article_metadata=None, + timeout=None, + force_update=None, +): + """ + Verifica disponibilidade e conteúdo de uma única ArticleWebPage. + + Delega para ``webpage.check_availability(user, timeout, + article_metadata, force_update)``. + + Raises: + ValueError: Se ``webpage_id`` não fornecido. + """ + try: + if not webpage_id: + raise ValueError("webpage_id must be provided") + user = _get_user(user_id, username) + webpage = ArticleWebPage.objects.get(id=webpage_id) + webpage.check_availability(user, timeout, article_metadata, force_update) + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "publication.tasks.task_check_article_page_availability", + "webpage_id": webpage_id, + }, + ) + + +# ============================================================ +# CALLBACK PÓS-MIGRAÇÃO +# ============================================================ + +@celery_app.task(bind=True) +def task_update_article_proc_availability( + self, + user_id=None, + username=None, + article_proc_id=None, + website_id=None, +): + """ + Callback pós-verificação: atualiza pid_status no ArticleProc. + + Se todas as webpages do artigo estão disponíveis no website, + atualiza ``article_proc.pid_status`` para ``PID_STATUS_PUBLIC_VALID``. + """ + try: + user = _get_user(user_id, username) + article_proc = ArticleProc.objects.select_related( + "collection", "sps_pkg", + ).get(pk=article_proc_id) + + if article_proc.all_webpage_available(website_id=website_id): + from migration.choices import PID_STATUS_PUBLIC_VALID + article_proc.set_pid_status(user, PID_STATUS_PUBLIC_VALID) + + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "publication.tasks.task_update_article_proc_availability", + "article_proc_id": article_proc_id, + }, + ) + + +# ============================================================ +# VERIFICAÇÃO EM LOTE (busca por filtros) +# ============================================================ + +@celery_app.task(bind=True) +def task_check_articles_availability( + self, + username, + user_id=None, + issn_print=None, + issn_electronic=None, + issue_folder=None, + publication_year=None, + article_pid_v3=None, + article_id=None, + article_proc_id=None, + collection_acron=None, + website_id=None, + timeout=None, + force_update=None, +): + """ + Verificação em lote: busca artigos por filtros e agenda verificação. + + Monta query dinâmica com os filtros fornecidos (ISSN, issue_folder, + publication_year, pid_v3, article_id, collection_acron) e para cada + par (artigo, website habilitado) agenda + ``task_check_article_webpages`` (assíncrono). + """ + try: + article_params = {} + j_query = Q() + + if article_id: + article_params["id"] = article_id + if article_pid_v3: + article_params["pid_v3"] = article_pid_v3 + if publication_year: + article_params["issue__publication_year"] = publication_year + if issue_folder: + article_params["issue__issue_folder"] = issue_folder + + if collection_acron or issn_electronic or issn_print: + j_params = {} + if collection_acron: + j_params["collection__acron"] = collection_acron + if issn_print: + j_query |= Q(journal__official_journal__issn_print=issn_print) + if issn_electronic: + j_query |= Q(journal__official_journal__issn_electronic=issn_electronic) + + article_params["journal__id__in"] = JournalProc.objects.filter( + j_query, **j_params + ).values_list("journal__id", flat=True).distinct() + + ws_filter = {"enabled": True} + if collection_acron: + ws_filter["collection__acron"] = collection_acron + if website_id: + ws_filter["id"] = website_id + + for website in WebSiteConfiguration.objects.filter(**ws_filter).select_related("collection"): + for article_id in Article.objects.filter( + journal__isnull=False, **article_params + ).values_list("id", flat=True): + task_check_article_webpages.apply_async( + kwargs=dict( + user_id=user_id, + username=username, + article_id=article_id, + article_proc_id=article_proc_id, + website_id=website.id, + timeout=timeout, + force_update=force_update, + ) + ) + + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "publication.tasks.task_check_articles_availability", + }, + ) + + +# ============================================================ +# VERIFICAÇÃO NO SITE CLÁSSICO (somente migração) +# ============================================================ + +@celery_app.task(bind=True) +def task_check_classic_website_article( + self, + user_id=None, + username=None, + article_proc_id=None, + timeout=None, + force_update=None, +): + """ + Confronta metadados do artigo com a página do site clássico. + + Passo extra de migração que verifica se o conteúdo da página HTML + do site clássico confere com os metadados do artigo migrado. + + Atualiza ``article_proc.pid_status`` conforme resultado: + - ``CLASSIC_MATCHED``: conteúdo confere. + - ``CLASSIC_MISMATCHED``: conteúdo diverge. + - ``CLASSIC_NOT_FOUND``: página não encontrada. + + Returns: + Resultado da verificação (dicionário retornado por + ``article_proc.check_classic_website_content``), ou None em + caso de erro. + """ + try: + user = _get_user(user_id, username) + article_proc = ArticleProc.objects.select_related( + "collection", "sps_pkg", "issue_proc__journal_proc", + ).get(pk=article_proc_id) + + article = article_proc.article + if not article: + raise ValueError(f"ArticleProc {article_proc_id} has no article") + + article_metadata_by_lang = article.get_metadata_by_lang() + response = article_proc.check_classic_website_content( + user, timeout, article_metadata_by_lang, force_update, + ) + + return response + + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "publication.tasks.task_check_classic_website_article", + "article_proc_id": article_proc_id, + }, + ) + + +@celery_app.task(bind=True) +def task_check_main_article_page_availability( + self, + article_id, + website_id, +): + """ + Verifica se alguma webpage do artigo está disponível no website. + + Returns: + True se ao menos uma webpage está disponível, False caso + contrário, ou None em caso de erro. + """ + try: + article = Article.objects.get(id=article_id) + return article.any_webpage_available(website=website_id) + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "publication.tasks.task_check_main_article_page_availability", + "article_id": article_id, + "website_id": website_id, + }, + ) \ No newline at end of file diff --git a/publication/api/document.py b/publication/api/document.py index e19793370..3f5242270 100644 --- a/publication/api/document.py +++ b/publication/api/document.py @@ -1,7 +1,3 @@ -import json -import logging -from datetime import datetime - from django.utils.translation import gettext_lazy as _ from publication.api.publication import PublicationAPI diff --git a/publication/models.py b/publication/models.py index 40489c8d2..96d3eb284 100644 --- a/publication/models.py +++ b/publication/models.py @@ -86,6 +86,7 @@ def create_or_update( publication_rule=None, website_url=None, timeout=None, + retry=False, ): try: obj = cls.get(article=article) @@ -95,41 +96,61 @@ def create_or_update( if published_by or publication_rule: obj.save() if website_url: - obj.create_or_update_urls(user, website_url, timeout) + obj.create_or_update_urls(user, website_url, timeout, retry=retry) return obj except cls.DoesNotExist: return cls.create( user, article, published_by, publication_rule, website_url, timeout ) - def create_or_update_urls(self, user, website_url, timeout=None): - for url in self.article.get_urls(website_url): + def create_or_update_urls(self, user, website_url, timeout=None, retry=False): + for item in self.article.get_webpage_items(website_url): + url = item.get("url") + if not url: + continue ScieloURLStatus.create_or_update( user=user, article=self.article, url=url, timeout=timeout, + retry=retry, + ) + self.update_completed() + + def update_urls_status(self, user, timeout=None, force_update=None): + filters = {} + if not force_update: + filters["available__in"] = (False, None) + for scielo_url_status in self.scielo_url.filter(**filters).all(): + scielo_url_status.update( + user=user, + timeout=timeout, + retry=force_update ) - self.check_is_completed() - - def retry(self, user, timeout=None, force_update=None): - for scielo_url_status in self.scielo_url.all(): - if not scielo_url_status.available or force_update: - scielo_url_status.update( - user=user, - timeout=timeout, - ) - self.check_is_completed() - - def check_is_completed(self): - if self.scielo_url.count(): - completed = not self.scielo_url.filter(available=False).exists() - else: - completed = False + self.update_completed() + + def update_completed(self): + completed = self.scielo_url.exclude(available=False).exists() if self.completed != completed: self.completed = completed self.save() + @property + def data(self): + return { + "article": self.article.pid_v3 if self.article else None, + "completed": self.completed, + "published_by": self.published_by, + "publication_rule": self.publication_rule, + "urls": [ + { + "url": url_status.url, + "available": url_status.available, + } + for url_status in self.scielo_url.all() + ], + } + class ScieloURLStatus(CommonControlField, Orderable): article_availability = ParentalKey( @@ -176,16 +197,18 @@ def create_or_update( article, url, timeout=None, + retry=False, ): try: obj = cls.get(url=url) - obj.update(user, timeout) + obj.update(user, timeout, retry) return obj except cls.DoesNotExist: return cls.create(article=article, url=url, user=user, timeout=timeout or 2) - def update(self, user, timeout=None): - self.available = check_url(self.url, timeout) + def update(self, user, timeout=None, retry=False): + if not self.available or retry: + self.available = check_url(self.url, timeout) self.updated_by = user self.save() diff --git a/publication/tasks.py b/publication/tasks.py index a5c053c22..9843ca2c1 100644 --- a/publication/tasks.py +++ b/publication/tasks.py @@ -226,9 +226,9 @@ def process_article_availability( try: user = _get_user(user_id=user_id, username=username) article = Article.objects.get(pid_v3=pid_v3) + logging.info(f"{domain} {pid_v3}") - obj = ArticleAvailability.create_or_update(user, article) - obj.create_or_update_urls(user, website_url=domain, timeout=timeout) + article.check_availability(user, domain, ArticleAvailability, timeout=timeout) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() UnexpectedEvent.create( diff --git a/publication/tests.py b/publication/tests.py deleted file mode 100644 index 2d4439bcc..000000000 --- a/publication/tests.py +++ /dev/null @@ -1,288 +0,0 @@ -from unittest.mock import PropertyMock, call, patch - -from django.core.files.uploadedfile import SimpleUploadedFile -from django.test import TestCase - -from article.models import Article, ArticleDOIWithLang -from collection.models import Collection, WebSiteConfiguration -from core.users.models import User -from core.utils.requester import NonRetryableError, RetryableError -from issue.models import Issue -from journal.models import Journal, JournalCollection, OfficialJournal -from proc.models import JournalProc - -from .models import ArticleAvailability, ScieloURLStatus -from .tasks import ( - fetch_data_and_register_result, - initiate_article_availability_check, - process_article_availability, - process_file_to_check_migrated_articles, - retry_failed_scielo_urls, -) - - -class ArticleAvailabilityTest(TestCase): - def setUp(self): - self.user = User.objects.create(username="user_test") - self.collection_scl = Collection.objects.create(acron="scl", creator=self.user) - self.collection_mex = Collection.objects.create(acron="mex", creator=self.user) - self.web_site_configuration_mex = WebSiteConfiguration.objects.create( - creator=self.user, - collection=self.collection_mex, - url="https://mocked-domain2.com", - enabled=True, - purpose="PUBLIC", - ) - self.web_site_configuration_scl = WebSiteConfiguration.objects.create( - creator=self.user, - collection=self.collection_scl, - url="https://mocked-domain.com", - enabled=True, - purpose="PUBLIC", - ) - self.web_site_configuration_scl = WebSiteConfiguration.objects.create( - creator=self.user, - collection=self.collection_scl, - url="https://qa-mocked-domain.com", - enabled=True, - purpose="QA", - ) - self.official_journal = OfficialJournal.objects.create( - issn_print="0000-0000", - issn_electronic="XXXX-XXXX", - creator=self.user, - ) - self.journal = Journal.objects.create( - official_journal=self.official_journal, - journal_acron="abdc", - creator=self.user, - ) - self.journal_collection_scl = JournalCollection.objects.create( - journal=self.journal, - collection=self.collection_scl, - creator=self.user, - ) - self.journal_collection_mex = JournalCollection.objects.create( - journal=self.journal, - collection=self.collection_mex, - creator=self.user, - ) - self.journal_proc = JournalProc.objects.create( - journal=self.journal, - collection=self.collection_scl, - acron="abdc", - creator=self.user, - ) - self.issue = Issue.objects.create(publication_year=2023, creator=self.user) - self.article = Article.objects.create( - journal=self.journal, - issue=self.issue, - pid_v3="test_pid_v3", - pid_v2="test_pid_v2", - creator=self.user, - ) - self.doi_en = ArticleDOIWithLang.objects.create( - doi_with_lang=self.article, - doi="10.1016/j.iheduc.2015.08.004", - lang="en", - creator=self.user, - ) - self.doi_pt = ArticleDOIWithLang.objects.create( - doi_with_lang=self.article, - doi="10.1016/j.iheduc.2015.08.004", - lang="pt", - creator=self.user, - ) - - def get_url(self, domain, journal_acron, pid_v2, pid_v3, lang): - return [ - f"{domain}/scielo.php?script=sci_arttext&pid={pid_v2}&lang={lang}&nrm=iso", - f"{domain}/j/{journal_acron}/a/{pid_v3}/?lang={lang}", - f"{domain}/scielo.php?script=sci_arttext&pid={pid_v2}&format=pdf&lng={lang}&nrm=iso", - f"{domain}/j/{journal_acron}/a/{pid_v3}/?format=pdf&lang={lang}", - ] - - @patch("publication.tasks.Article.article_langs", new_callable=PropertyMock) - @patch("publication.tasks.process_article_availability.apply_async") - def test_initiate_article_availability_check( - self, - mock_process_apply_async, - mock_property_article_langs, - ): - mock_property_article_langs.side_effect = ["pt", "es"] - initiate_article_availability_check( - user_id=1, username="user_test", collection_acron="scl", purpose="PUBLIC" - ) - - self.assertEqual(mock_process_apply_async.call_count, 2) - - @patch("publication.tasks.Article.article_langs", new_callable=PropertyMock) - @patch("publication.tasks.process_article_availability.apply_async") - def test_initiate_article_availability_check_with_params( - self, - mock_process_apply_async, - mock_property_article_langs, - ): - mock_property_article_langs.side_effect = ["pt", "es"] - initiate_article_availability_check( - user_id=1, - username="user_test", - issn_print="0000-0000", - issn_electronic="XXXX-XXXX", - article_pid_v3="test_pid_v3", - purpose="PUBLIC", - collection_acron="scl", - ) - - self.assertEqual(mock_process_apply_async.call_count, 2) - - @patch("publication.tasks.Article.article_langs", new_callable=PropertyMock) - @patch("publication.tasks.process_article_availability.apply_async") - def test_initiate_article_availability_check_all_collections( - self, - mock_process_apply_async, - mock_property_article_langs, - ): - mock_property_article_langs.side_effect = ["pt", "es"] - initiate_article_availability_check( - user_id=1, username="user_test", purpose="PUBLIC" - ) - - self.assertEqual(mock_process_apply_async.call_count, 4) - - @patch("publication.tasks.fetch_data_and_register_result.apply_async") - def test_process_article_availability_call_times(self, mock_apply_async): - process_article_availability( - user_id=None, - username="user_test", - pid_v3=self.article.pid_v3, - pid_v2=self.article.pid_v2, - journal_acron=self.article.journal.journal_acron, - lang="en", - domain=self.web_site_configuration_scl.url, - ) - process_article_availability( - user_id=None, - username="user_test", - pid_v3=self.article.pid_v3, - pid_v2=self.article.pid_v2, - journal_acron=self.article.journal.journal_acron, - lang="pt", - domain=self.web_site_configuration_scl.url, - ) - self.assertEqual(mock_apply_async.call_count, 8) - - @patch("publication.tasks.fetch_data") - def test_fetch_data_and_register_result_some_fail(self, mock_fetch_data): - mock_fetch_data.side_effect = [ - RetryableError, - "mock content", - "mock content", - NonRetryableError, - ] - urls = self.get_url( - domain=self.web_site_configuration_scl.url, - journal_acron=self.journal.journal_acron, - pid_v2=self.article.pid_v2, - pid_v3=self.article.pid_v3, - lang="en", - ) - for url in urls: - fetch_data_and_register_result( - user_id=None, - username="user_test", - pid_v3=self.article.pid_v3, - url=url, - ) - self.assertEqual(mock_fetch_data.call_count, 4) - - scielo_url_status_first = ScieloURLStatus.objects.filter( - available=False - ).first() - scielo_url_status_last = ScieloURLStatus.objects.filter(available=False).last() - - self.assertEqual(ScieloURLStatus.objects.filter(available=False).count(), 2) - self.assertEqual(ScieloURLStatus.objects.filter(available=True).count(), 2) - - self.assertEqual(scielo_url_status_first.available, False) - self.assertEqual(scielo_url_status_last.available, False) - self.assertEqual( - scielo_url_status_first.url, - f"{self.web_site_configuration_scl.url}/scielo.php?script=sci_arttext&pid={self.article.pid_v2}&lang=en&nrm=iso", - ) - self.assertEqual( - scielo_url_status_last.url, - f"{self.web_site_configuration_scl.url}/j/{self.article.journal.journal_acron}/a/{self.article.pid_v3}/?format=pdf&lang=en", - ) - - @patch("publication.tasks.fetch_data") - def test_process_article_avaibility_fail_and_success(self, mock_fetch_data): - mock_fetch_data.side_effect = [ - RetryableError, - "mock content", - "mock content", - "mock content", - ] - urls = self.get_url( - domain=self.web_site_configuration_scl.url, - journal_acron=self.journal.journal_acron, - pid_v2=self.article.pid_v2, - pid_v3=self.article.pid_v3, - lang="en", - ) - for url in urls: - fetch_data_and_register_result( - user_id=None, - username="user_test", - pid_v3=self.article.pid_v3, - url=url, - ) - self.assertEqual(mock_fetch_data.call_count, 4) - - self.assertEqual(ScieloURLStatus.objects.filter(available=False).count(), 1) - self.assertEqual(ScieloURLStatus.objects.filter(available=True).count(), 3) - self.assertEqual( - ScieloURLStatus.objects.get(available=False).url, - f"{self.web_site_configuration_scl.url}/scielo.php?script=sci_arttext&pid={self.article.pid_v2}&lang=en&nrm=iso", - ) - - mock_fetch_data.side_effect = [ - "mock content", - "mock content", - "mock content", - "mock content", - ] - for url in urls: - fetch_data_and_register_result( - user_id=None, - username="user_test", - pid_v3=self.article.pid_v3, - url=url, - ) - - self.assertEqual(ScieloURLStatus.objects.filter(available=False).count(), 0) - - @patch("publication.tasks.fetch_data_and_register_result.apply_async") - def test_retry_failed_scielo_urls(self, mock_apply_async): - article_availability = ArticleAvailability.objects.create( - article=self.article, creator=self.user - ) - ScieloURLStatus.objects.create( - article_availability=article_availability, - url="https://www.example.com", - available=False, - creator=self.user, - ) - expected_calls = [ - call( - kwargs={ - "pid_v3": "test_pid_v3", - "url": "https://www.example.com", - "username": "user_test", - "user_id": None, - } - ) - ] - retry_failed_scielo_urls(username="user_test") - self.assertEqual(mock_apply_async.call_count, 1) - mock_apply_async.assert_has_calls(expected_calls, any_order=False) diff --git a/team/migrations/0004_rename_team_collec_collect_idx_team_collec_collect_0ee96e_idx_and_more.py b/team/migrations/0004_rename_team_collec_collect_idx_team_collec_collect_0ee96e_idx_and_more.py new file mode 100644 index 000000000..9dd02fa92 --- /dev/null +++ b/team/migrations/0004_rename_team_collec_collect_idx_team_collec_collect_0ee96e_idx_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 5.2.3 on 2026-05-12 23:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("team", "0003_add_role_to_collectionteammember"), + ] + + operations = [ + migrations.RenameIndex( + model_name="collectionteammember", + new_name="team_collec_collect_0ee96e_idx", + old_name="team_collec_collect_idx", + ), + migrations.RenameIndex( + model_name="collectionteammember", + new_name="team_collec_user_id_6382ef_idx", + old_name="team_collec_user_id_idx", + ), + migrations.AddField( + model_name="company", + name="certified_since", + field=models.DateField( + blank=True, null=True, verbose_name="Certified Since" + ), + ), + migrations.AddField( + model_name="company", + name="logo", + field=models.ImageField( + blank=True, null=True, upload_to="logos/", verbose_name="Logo" + ), + ), + migrations.AddField( + model_name="company", + name="personal_contact", + field=models.CharField( + blank=True, max_length=30, null=True, verbose_name="Personal Contact" + ), + ), + migrations.AddField( + model_name="company", + name="url", + field=models.URLField(blank=True, null=True, verbose_name="URL"), + ), + ]