feat(publisher): add VOAParser V1_1 with enhanced selectors

docs/how_to_add_a_publisher.md

@@ -255,6 +255,23 @@ sitemap_filter=inverse(regex_filter("sitemap-content-"))
 ````
 will exclude all sitemap URLs not containing the substring `sitemap-content-`.
 
+#### Ordering sitemaps
+
+Some indices list their sitemaps in an order that is neither ascending nor descending by date, e.g. numbered sitemaps ordered as text, where `sitemap_10` follows directly after `sitemap_1`.
+Use the `sort_key` parameter to reorder them.
+It is handed to `list.sort`, so ordering is ascending, and `numeric_sort_key` builds one from a regular expression by reading its capture groups as integers:
+
+````python
+Sitemap(
+    "https://www.voanews.com/sitemap.xml",
+    sitemap_filter=inverse(regex_filter(r"sitemap_[\d_]*\.xml\.gz")),
+    sort_key=numeric_sort_key(r"sitemap_\d+_(\d+)\.xml"),
+)
+````
+
+Pass `reverse=True` to `numeric_sort_key` if the number grows with recency instead.
+Unlike `Sitemap`'s `reverse`, which is applied afterwards and also flips the URLs within each sitemap, this only reorders the sitemaps.
+
 ### Finishing the Publisher Specification
 
 1. If your publisher requires to use custom request headers to work properly you can alter it by using the `request_header` parameter of `PublisherSpec`.

src/fundus/publishers/at/__init__.py

@@ -1,7 +1,5 @@
-import re
-
 from fundus.publishers.base_objects import Publisher, PublisherGroup
-from fundus.scraping.url import NewsMap, RSSFeed, Sitemap
+from fundus.scraping.url import NewsMap, RSSFeed, Sitemap, numeric_sort_key
 
 from .derstandard import DerStandardParser
 from .die_presse import DiePresseParser
@@ -59,7 +57,7 @@ class AT(metaclass=PublisherGroup):
             NewsMap("https://www.sn.at/news-artikel.sitemap.xml"),
             Sitemap(
                 "https://www.sn.at/portal-artikel.sitemap.xml",
-                sort_predicate=re.compile(r"(article-)\d{4}_\d{2}-\d(.)"),
+                sort_key=numeric_sort_key(r"article-(\d{4})_(\d{2})-(\d)", reverse=True),
             ),
         ],
     )

src/fundus/publishers/us/__init__.py

@@ -1,6 +1,6 @@
 from fundus.publishers.base_objects import Publisher, PublisherGroup
 from fundus.scraping.filter import inverse, lor, regex_filter
-from fundus.scraping.url import NewsMap, RSSFeed, Sitemap
+from fundus.scraping.url import NewsMap, RSSFeed, Sitemap, numeric_sort_key
 
 from .ap_news import APNewsParser
 from .business_insider import BusinessInsiderParser
@@ -261,6 +261,7 @@ class US(metaclass=PublisherGroup):
             Sitemap(
                 "https://www.voanews.com/sitemap.xml",
                 sitemap_filter=inverse(regex_filter(r"sitemap_[\d_]*\.xml\.gz")),
+                sort_key=numeric_sort_key(r"sitemap_\d+_(\d+)\.xml"),
             ),
         ],
     )

src/fundus/publishers/us/voice_of_america.py

@@ -16,11 +16,18 @@
 
 class VOAParser(ParserProxy):
     class V1(BaseParser):
-        _paragraph_selector = CSSSelector("#article-content > div > p")
+        VALID_UNTIL = datetime.date(2026, 7, 20)
+
+        _paragraph_selector: XPath = CSSSelector("#article-content > div > p")
+        _subheadline_selector: Optional[XPath] = None
 
         @attribute
         def body(self) -> Optional[ArticleBody]:
-            return extract_article_body_with_selector(self.precomputed.doc, paragraph_selector=self._paragraph_selector)
+            return extract_article_body_with_selector(
+                self.precomputed.doc,
+                paragraph_selector=self._paragraph_selector,
+                subheadline_selector=self._subheadline_selector,
+            )
 
         @attribute
         def publishing_date(self) -> Optional[datetime.datetime]:
@@ -46,3 +53,7 @@ def images(self) -> List[Image]:
                 upper_boundary_selector=XPath("//h1"),
                 lower_boundary_selector=XPath("//div[@id='ymla-section']"),
             )
+
+    class V1_1(V1):
+        _paragraph_selector = XPath("//div[@id='article-content']/div/p[not(strong)]")
+        _subheadline_selector = XPath("//div[@id='article-content']/div/p[strong]")

src/fundus/scraping/url.py

@@ -2,19 +2,21 @@
 import gzip
 import itertools
 import lzma
+import re
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
-from functools import cached_property, partial
+from functools import cached_property
 from typing import (
+    Any,
     Callable,
     ClassVar,
     Dict,
     Iterable,
     Iterator,
     List,
     Optional,
-    Pattern,
     Set,
+    Tuple,
 )
 from urllib.parse import unquote, urlparse
 
@@ -79,6 +81,7 @@ def __init__(self):
         self.archive_mapping: Dict[str, Callable[[bytes], bytes]] = {
             "application/octet-stream": self._decompress_octet_stream,
             "application/x-gzip": CompressionFormats.GZIP,
+            "application/gzip": CompressionFormats.GZIP,
             "gzip": CompressionFormats.GZIP,
         }
 
@@ -176,12 +179,34 @@ def fetch(self, session: InterruptableSession, headers: Dict[str, str]) -> Itera
                 yield clean_url(url)
 
 
+def numeric_sort_key(pattern: str, reverse: bool = False) -> Callable[[str], Tuple[int, ...]]:
+    """Build a <sort_key> ordering sitemaps by the integer capture groups of <pattern>.
+
+    Groups are compared numerically rather than as text, so unpadded indices order as
+    1, 2, ..., 10 instead of 1, 10, 2. Pass reverse=True to negate them, putting the
+    highest value first - use it when the number grows with recency (e.g. a date),
+    and leave it off when it grows with age (e.g. a sitemap chunk counted from newest).
+
+    Raises ValueError for a URL the pattern doesn't match; <sitemap_filter> is applied
+    first, so the key only ever sees the sitemaps that were kept.
+    """
+    compiled = re.compile(pattern)
+    sign = -1 if reverse else 1
+
+    def key(url: str) -> Tuple[int, ...]:
+        if match := compiled.search(url):
+            return tuple(sign * int(group) for group in match.groups())
+        raise ValueError(f"<sort_key> pattern {pattern!r} does not match sitemap URL {url!r}")
+
+    return key
+
+
 @dataclass
 class Sitemap(URLSource):
     recursive: bool = True
     reverse: bool = False
     sitemap_filter: URLFilter = lambda url: not bool(url)
-    sort_predicate: Optional[Pattern[str]] = None
+    sort_key: Optional[Callable[[str], Any]] = None
 
     _decompressor: ClassVar[_ArchiveDecompressor] = _ArchiveDecompressor()
     _sitemap_selector: ClassVar[XPath] = XPath("//*[local-name()='sitemap']/*[local-name()='loc']")
@@ -226,20 +251,11 @@ def yield_recursive(sitemap_url: str) -> Iterator[str]:
             elif self.recursive:
                 sitemap_locs = [node.text for node in self._sitemap_selector(tree)]
 
-                if self.sort_predicate is not None:
-
-                    def _extract_predicate(text: str, pattern: Pattern[str]) -> str:
-                        if match := pattern.search(text):
-                            return match.group()
-                        raise NotImplementedError("<sort_predicate> must match in all sitemap URLs")
+                filtered_locs = list(filter(inverse(self.sitemap_filter), sitemap_locs))
 
-                    sitemap_locs = sorted(
-                        sitemap_locs,
-                        key=partial(_extract_predicate, pattern=self.sort_predicate),
-                        reverse=True,
-                    )
+                if self.sort_key is not None:
+                    filtered_locs.sort(key=self.sort_key)
 
-                filtered_locs = list(filter(inverse(self.sitemap_filter), sitemap_locs))
                 for loc in reversed(filtered_locs) if self.reverse else filtered_locs:
                     yield from yield_recursive(loc)