feat: adiciona status NOT_PUBLIC para evitar requisições a documentos despublicados

article/controller.py

@@ -12,7 +12,7 @@
 from core.mongodb import write_item
 from core.utils.harvesters import AMHarvester, OPACHarvester
 from institution.models import Sponsor
-from journal.models import Journal
+from journal.models import Journal, SciELOJournal
 from pid_provider.choices import (
     PPXML_STATUS_TODO,
     PPXML_STATUS_INVALID,
@@ -446,6 +446,7 @@ def __init__(
         timeout=None,
         opac_url=None,
         force_update=None,
+        stop=None,
     ):
         self.user = user
         self.collection_acron_list = collection_acron_list
@@ -461,6 +462,7 @@ def __init__(
         self.timeout = timeout
         self.opac_url = opac_url
         self.force_update = force_update
+        self.stop = stop
 
         self._iter_from_harvest_count = 0
         self._iter_from_article_source_count = 0
@@ -553,17 +555,30 @@ def _iter_from_harvest(self):
             Collection.load(self.user)
 
         count = 0
-        for collection_acron in self.collection_acron_list or list(Collection.get_acronyms()):
-            logging.info(collection_acron)
-            harvester = self._build_harvester(collection_acron)
-            logging.info(harvester)
+        params = {}
+        if self.collection_acron_list:
+            params["collection__acron3__in"] = self.collection_acron_list
+        if self.journal_acron_list:
+            params["journal_acron__in"] = self.journal_acron_list
+
+        collection_and_journal_items = SciELOJournal.objects.select_related(
+            "collection"
+        ).filter(
+            **params
+        ).values_list(
+            "collection__acron3", "journal_acron", "issn_scielo"
+        ).distinct()
+
+        for collection_acron, journal_acron, issn_scielo in collection_and_journal_items:
+            harvester = self._build_harvester(collection_acron, journal_acron, issn_scielo)
             for document in harvester.harvest_documents():
                 count += 1
                 yield {
                     "xml_url": document["url"],
                     "collection_acron": collection_acron,
                     "pid": document["pid_v2"],
                     "source_date": document.get("processing_date") or document.get("origin_date"),
+                    "is_public": document.get("is_public")
                 }
 
         self._iter_from_harvest_count = count
@@ -587,7 +602,7 @@ def _iter_from_article_source(self):
     # Helpers privados
     # ------------------------------------------------------------------
 
-    def _build_harvester(self, collection_acron):
+    def _build_harvester(self, collection_acron, journal_acron=None, journal_id=None):
         """Instancia o harvester adequado para a coleção."""
         kwargs = dict(
             from_date=self.from_date,
@@ -596,6 +611,12 @@ def _build_harvester(self, collection_acron):
             timeout=self.timeout,
         )
         if collection_acron == "scl":
-            return OPACHarvester(self.opac_url or "www.scielo.br", collection_acron, **kwargs)
+            if journal_acron:
+                kwargs["journal"] = journal_acron
+            if self.stop:
+                kwargs["stop"] = self.stop
+            return OPACHarvester(self.opac_url or "https://www.scielo.br", collection_acron, **kwargs)
+        if journal_id:
+            kwargs["journal"] = journal_id
         return AMHarvester("article", collection_acron, **kwargs)
 

article/migrations/0049_alter_articlesource_status.py

@@ -0,0 +1,33 @@
+# Generated by Django 5.2.7 on 2026-07-21 14:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("article", "0048_alter_articlesource_status"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="articlesource",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("pending", "Pending"),
+                    ("processing", "Processing"),
+                    ("completed", "Completed"),
+                    ("error", "Error"),
+                    ("reprocess", "Reprocess"),
+                    ("url_error", "URL Error"),
+                    ("xml_error", "XML Error"),
+                    ("not_public", "Not public"),
+                ],
+                default="pending",
+                help_text="Processing status of the article source",
+                max_length=20,
+                verbose_name="Status",
+            ),
+        ),
+    ]

article/models.py

@@ -1694,6 +1694,7 @@ class StatusChoices(models.TextChoices):
         REPROCESS = "reprocess", _("Reprocess")
         URL_ERROR = "url_error", _("URL Error")
         XML_ERROR = "xml_error", _("XML Error")
+        NOT_PUBLIC = "not_public", _("Not public")
 
     url = models.URLField(
         verbose_name=_("Article URL"),
@@ -1801,7 +1802,7 @@ def get(cls, url):
         raise ValueError("ArticleSource.get requires url")
 
     @classmethod
-    def create(cls, user, url=None, source_date=None, am_article=None, force_update=None, auto_solve_pid_conflict=False):
+    def create(cls, user, url=None, source_date=None, am_article=None, force_update=None, auto_solve_pid_conflict=False, is_public=None):
         if not url:
             raise ValueError("ArticleSource.create requires url")
 
@@ -1811,21 +1812,31 @@ def create(cls, user, url=None, source_date=None, am_article=None, force_update=
             obj.url = url
             obj.source_date = source_date
             obj.am_article = am_article
-            obj.status = cls.StatusChoices.PENDING
+            if is_public is False:
+                obj.status = cls.StatusChoices.NOT_PUBLIC
+            else:
+                obj.status = cls.StatusChoices.PENDING
             obj.add_pid_provider(user, force_update, auto_solve_pid_conflict=auto_solve_pid_conflict)
             return obj
         except IntegrityError:
             return cls.get(url=url)
 
     @classmethod
     def create_or_update(
-        cls, user, url=None, source_date=None, am_article=None, force_update=None, auto_solve_pid_conflict=False
+        cls, user, url=None, source_date=None, am_article=None, force_update=None, auto_solve_pid_conflict=False, is_public=None
     ):
         try:
             logging.info(
                 f"ArticleSource.create_or_update {url} {source_date} {am_article} {force_update}"
             )
+            changed = False
             obj = cls.get(url=url)
+            if is_public is False:
+                obj.status = cls.StatusChoices.NOT_PUBLIC
+                changed = True
+            elif is_public is True and obj.status == cls.StatusChoices.NOT_PUBLIC:
+                obj.status = cls.StatusChoices.PENDING
+                changed = True
             if (
                 force_update
                 or (source_date and source_date != obj.source_date)
@@ -1835,6 +1846,9 @@ def create_or_update(
                 obj.source_date = source_date
                 obj.am_article = am_article
                 obj.add_pid_provider(user, force_update, auto_solve_pid_conflict=auto_solve_pid_conflict)
+                changed = True
+            if changed:
+                obj.save()
             return obj
         except cls.DoesNotExist:
             return cls.create(
@@ -1843,7 +1857,8 @@ def create_or_update(
                 source_date=source_date,
                 am_article=am_article,
                 force_update=force_update,
-                auto_solve_pid_conflict=auto_solve_pid_conflict
+                auto_solve_pid_conflict=auto_solve_pid_conflict,
+                is_public=is_public,
             )
 
     @cached_property
@@ -2008,7 +2023,6 @@ def is_completed(self):
             return False
         if self.status != ArticleSource.StatusChoices.COMPLETED:
             self.status = ArticleSource.StatusChoices.COMPLETED
-            self.save()
         logging.info(f"Completed: ArticleSource {self.url} is completed")
         return True
 
@@ -2029,6 +2043,11 @@ def add_pid_provider(self, user, force_update=False, auto_solve_pid_conflict=Fal
         """
         try:
             detail = []
+
+            if self.status == ArticleSource.StatusChoices.NOT_PUBLIC:
+                if not force_update:
+                    return
+
             self.status = ArticleSource.StatusChoices.PENDING
 
             # --- Etapa 1: request_xml ---
@@ -3249,7 +3268,7 @@ def add_normalized_affiliation(self, user, organization=None, location=None,
             self.save()
 
         # Add normalized affiliation to the ArticleAffiliation
-        self.affiliation.add_normalized_affiliation(
+        self.affiliation.set_normalized(
             user=user,
             organization=organization,
             location=location,

article/tasks.py

@@ -726,6 +726,8 @@ def task_dispatch_articles(
     opac_url=None,
     # --- ativa article_source ---
     article_source_status_list=None,
+    verify=None,
+    stop=None,
 ):
     """
     Tarefa orquestradora que dispara processamento em lote de artigos.
@@ -801,6 +803,7 @@ def task_dispatch_articles(
             timeout=timeout,
             opac_url=opac_url,
             force_update=force_update,
+            stop=stop
         ):
             if item_kwargs is None:
                 skipped += 1
@@ -857,6 +860,7 @@ def task_process_article_pipeline(
     version=None,
     user_id=None,
     username=None,
+    is_public=None,
 ):
     """
     Pipeline principal de processamento de artigos com múltiplos pontos de entrada.
@@ -941,6 +945,7 @@ def task_process_article_pipeline(
                 force_update=force_update,
                 am_article=am_article,
                 auto_solve_pid_conflict=auto_solve_pid_conflict,
+                is_public=is_public,
             )
             pp_xml_id = article_source.pid_provider_xml.id
 

article/tests/test_mixins.py

@@ -0,0 +1,14 @@
+from unittest.mock import patch
+from article.models import Article
+
+
+class ArticleTestMixin:
+    """Mixin com helpers e mocks para o app Article."""
+
+    def make_article(self, user=None, pid_v3=None):
+        kwargs = {}
+        if user:
+            kwargs["creator"] = user
+        if pid_v3:
+            kwargs["pid_v3"] = pid_v3
+        return Article.objects.create(**kwargs)