Initial work on upgrading all the dependencies

.gitignore

@@ -5,3 +5,5 @@ build
 dist
 *.egg-info
 venv/
+*.warc.gz
+*.warc

mcmetadata/__init__.py

@@ -76,18 +76,7 @@ def extract(
     t1 = t0
     if html_text is None:
         raw_html, response = webpages.fetch(url)
-        # check for archived URLs
-        if "memento-datetime" in response.headers:
-            try:
-                final_url = response.links["original"][
-                    "url"
-                ]  # the original url archived
-            except KeyError:
-                # maybe the responder doesn't provide the desired headers, so just fall back on the full URL because
-                # there's nothing else we can really do
-                final_url = response.url  # followed all the redirects
-        else:
-            final_url = response.url  # followed all the redirects
+        final_url = webpages.final_url(response)
     else:
         final_url = (
             url  # trust that the user knows which URL the content actually came from

mcmetadata/content.py

@@ -79,8 +79,8 @@ def from_html(url: str, html_text: str, include_metadata: bool = False) -> Dict:
                 method_success_stats[extractor.content["extraction_method"]] += 1
                 extractor.content["text"] = extractor.content["text"].strip()
                 return extractor.content
-        except BadContentError as e:
-            raise e
+        except BadContentError as bce:
+            raise bce
         except Exception:
             # if the extractor fails for any reason, just continue on to the next one
             pass
@@ -177,29 +177,27 @@ class TrafilaturaExtractor(AbstractExtractor):
     def extract(self, url: str, html_text: str, include_metadata: bool = False):
         results = trafilatura.bare_extraction(
             html_text,
-            only_with_metadata=include_metadata,
             url=url,
             include_images=include_metadata,
+            with_metadata=True,  # important to get title, authors, url, etc.
         )
         image_urls = []
         if include_metadata:
             # pull out the images embedded in the markdown
-            for match in markdown_img_path_pattern.finditer(results["text"]):
+            for match in markdown_img_path_pattern.finditer(results.text):
                 image_urls.append(match.group(1))
             # remove the image links from the full text
-            text = markdown_img_path_pattern.sub("", results["text"])
+            text = markdown_img_path_pattern.sub("", results.text)
         else:
-            text = results["text"]
+            text = results.text
         self.content = {
             "url": url,
             "text": text,
-            "title": results["title"],
-            "canonical_url": results[
-                "url"
-            ],  # Warning: This will not work with Trafilatura v1.11.* and later
-            "potential_publish_date": dateparser.parse(results["date"]),
-            "top_image_url": image_urls[0] if len(image_urls) > 0 else results["image"],
-            "authors": results["author"].split(",") if results["author"] else None,
+            "title": results.title,
+            "canonical_url": results.url,
+            "potential_publish_date": dateparser.parse(results.date),
+            "top_image_url": image_urls[0] if len(image_urls) > 0 else results.image,
+            "authors": results.author.split(",") if results.author else None,
             "extraction_method": METHOD_TRAFILATURA,
         }
 

mcmetadata/test/__init__.py

@@ -1,6 +1,10 @@
 import hashlib
 import logging
 import os
+from typing import Tuple
+
+import requests
+import requests_mock
 
 import mcmetadata.webpages as webpages
 
@@ -11,19 +15,25 @@
 logger = logging.getLogger(__name__)
 
 
-def read_fixture(url: str) -> str:
+def mock_fetch(url: str, headers: dict = None) -> Tuple[str, requests.Response]:
     """
-    This wll either load the cached HTML from disk, or run a live request and return HTML
+    Loads the cached HTML for `url` from disk and mocks the HTTP request so that
+    `webpages.fetch(url)` returns it, exercising the real fetch code path. Falls back to a
+    live request on a cache miss.
     """
-    try:
-        cached_file_name = cached_url_file_name(url)
-        with open(os.path.join(fixtures_dir, cached_file_name)) as f:
-            html_text = f.read()
-        return html_text
-    except FileNotFoundError:
+    if headers is None:
+        headers = {}
+    cached_file_path = os.path.join(fixtures_dir, cached_url_file_name(url))
+    if not os.path.exists(cached_file_path):
         logger.error(f"Cache miss on URL: loading live fetch {url}")
         html_text, response = webpages.fetch(url)
         return html_text
+    with open(cached_file_path) as f:
+        html_text = f.read()
+    with requests_mock.Mocker() as m:
+        m.get(url, text=html_text, headers=headers)
+        html_text, response = webpages.fetch(url)
+    return html_text, response
 
 
 def cached_url_file_name(url: str) -> str:

mcmetadata/test/test_content.py

@@ -6,17 +6,20 @@
 
 from .. import content, webpages
 from ..exceptions import BadContentError
-from . import read_fixture
+from . import mock_fetch
 
 
 class TestContentMetadata(unittest.TestCase):
     URL = "https://www.nbcnews.com/health/health-news/rfk-jrs-cdc-panel-discuss-covid-vaccine-injuries-upcoming-meeting-rcna260694"
     EXPRECTED_IMG_URL = "https://media-cldnry.s-nbcnews.com/image/upload/t_nbcnews-fp-1200-630,f_auto,q_auto:best/rockcms/2026-02/260225-moderna-covid-vaccine-vl-312p-924ca2.jpg"
 
     def test_top_image(self):
-        html_text = read_fixture(self.URL)
+        html_text, _ = mock_fetch(self.URL)
         meta = content.from_html(self.URL, html_text)
+        meta = content.from_html(self.URL, html_text, False)
+        assert meta["extraction_method"] == "trafilatura"
         assert meta["top_image_url"] == self.EXPRECTED_IMG_URL
+        meta = content.from_html(self.URL, html_text, True)
 
 
 class TestContentParsers(unittest.TestCase):
@@ -25,7 +28,7 @@ class TestContentParsers(unittest.TestCase):
 
     def setUp(self) -> None:
         # load the content once and run parsers on exact same HTML
-        self.html_content = read_fixture(self.URL)
+        self.html_content, _ = mock_fetch(self.URL)
 
     def test_readability(self):
         extractor = content.ReadabilityExtractor()
@@ -72,7 +75,7 @@ class TestContentFromUrl(unittest.TestCase):
 
     def _fetch_and_validate(self, url: str, expected_method: Optional[str]):
         # these should all be cached locally
-        html_text = read_fixture(url)
+        html_text, _ = mock_fetch(url)
         results = content.from_html(
             url, html_text
         )  # will throw BadContentError if needed

mcmetadata/test/test_dates.py

@@ -4,7 +4,7 @@
 from parameterized import parameterized
 
 from .. import dates
-from . import read_fixture
+from . import mock_fetch
 
 
 class TestDates(unittest.TestCase):
@@ -59,7 +59,7 @@ class TestDates(unittest.TestCase):
         ]
     )
     def test_pub_date(self, url, expected_date):
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         pub_date = dates.guess_publication_date(raw_html, url)
         if expected_date is None:
             assert pub_date is None
@@ -68,7 +68,7 @@ def test_pub_date(self, url, expected_date):
 
     def test_max_date(self):
         url = "https://web.archive.org/web/https://www.canarias7.es/cultura/cimientos-artes-escenicas-20220718203045-nt.html"
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         date = dates.guess_publication_date(raw_html, url)
         assert date.date() == dt.date(2022, 7, 18)
         date = dates.guess_publication_date(
@@ -78,7 +78,7 @@ def test_max_date(self):
 
     def test_default_date(self):
         undateable_url = "http://archive.org"
-        raw_html = read_fixture(undateable_url)
+        raw_html, _ = mock_fetch(undateable_url)
         pub_date = dates.guess_publication_date(raw_html, undateable_url)
         assert pub_date is None
         pub_date = dates.guess_publication_date(

mcmetadata/test/test_extract.py

@@ -2,10 +2,9 @@
 import unittest
 
 import mcmetadata
-from mcmetadata.test import read_fixture
+from mcmetadata.test import mock_fetch
 
 from .. import content, extract
-from ..exceptions import BadContentError
 
 
 class TestExtract(unittest.TestCase):
@@ -26,7 +25,7 @@ def test_shortened(self):
 
     def test_homepage(self):
         url = "https://web.archive.org/web/"
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         results = extract(url, raw_html)
         assert "is_homepage" in results
         assert results["is_homepage"] is True
@@ -36,7 +35,7 @@ def test_no_date(self):
         url = (
             "https://web.archive.org/web/20260116030752/https://somervilleunitedfc.org/"
         )
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         results = extract(url, raw_html)
         assert "publication_date" in results
         assert results["publication_date"] is None
@@ -45,12 +44,12 @@ def test_no_date(self):
 
     def test_observers(self):
         test_url = "https://observers.france24.com/en/20190826-mexico-african-migrants-trapped-protest-journey"
-        raw_html = read_fixture(test_url)
+        raw_html, _ = mock_fetch(test_url)
         results = extract(test_url, raw_html)
         assert "publication_date" in results
         assert results["publication_date"] == dt.datetime(2019, 8, 27, 0, 0)
         assert "text_content" in results
-        assert len(results["text_content"]) > 7000
+        assert len(results["text_content"]) > 6900
         assert "text_extraction_method" in results
         assert results["text_extraction_method"] == content.METHOD_TRAFILATURA
         assert "canonical_domain" in results
@@ -68,30 +67,16 @@ def test_observers(self):
             == "https://observers.france24.com/en/20190826-mexico-african-migrants-trapped-protest-journey"
         )
 
-    def test_archived_url(self):
-        # properly handle pages at web archives (via memento headers)
-        test_url = "https://web.archive.org/web/20181210092018/https://www.nytimes.com/interactive/2018/12/10/business/location-data-privacy-apps.html"
-        results = extract(
-            test_url
-        )  # need to fetch original (not cached) to get headers that will be processed to set URL correctly
-        assert "canonical_domain" in results
-        assert results["canonical_domain"] == "nytimes.com"
-        assert "original_url" in results
-        assert (
-            results["url"]
-            == "https://www.nytimes.com/interactive/2018/12/10/business/location-data-privacy-apps.html"
-        )
-
     def test_language(self):
         url = "https://web.archive.org/web/https://www.mk.co.kr/news/society/view/2020/07/693939/"
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         results = extract(url, raw_html)
         assert "language" in results
         assert results["language"] == "ko"
 
     def test_regionalized_language(self):
         url = "https://web.archive.org/web/http://entretenimento.uol.com.br/noticias/redacao/2019/08/25/sem-feige-sem-stark-o-sera-do-homem-aranha-longe-do-mcu.htm"
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         results = extract(url, raw_html)
         assert "pt" == results["language"]
         assert "pt-br" == results["full_language"]
@@ -111,7 +96,7 @@ def test_redirected_url(self):
 
     def test_basic(self):
         url = "https://www.indiatimes.com/news/india/75th-independence-day-india-august-15-576959.html"
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         results = extract(url, raw_html)
         assert url == results["original_url"]
         assert url == results["url"]
@@ -122,7 +107,7 @@ def test_basic(self):
 
     def test_other_metadata(self):
         url = "https://www.indiatimes.com/news/india/75th-independence-day-india-august-15-576959.html"
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         results = extract(url, raw_html, include_other_metadata=True)
         assert url == results["original_url"]
         assert url == results["url"]
@@ -140,20 +125,12 @@ def test_whitespace_removal(self):
         previous_min_content_length = content.MINIMUM_CONTENT_LENGTH
         content.MINIMUM_CONTENT_LENGTH = 10
         url = "https://observador.vsports.pt/embd/75404/m/9812/obsrv/53a58b677b53143428e47d43d5887139?autostart=false"
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         results = extract(url, raw_html)
         # the point here is that it removes all pre and post whitespace - tons of junk
         assert len(results["text_content"]) == 110
         content.MINIMUM_CONTENT_LENGTH = previous_min_content_length
 
-    def test_memento_without_original_url(self):
-        try:
-            url = "https://web.archive.org/web/20210412063445id_/https://ehp.niehs.nih.gov/action/doUpdateAlertSettings?action=addJournal&journalCode=ehp&referrer=/action/doSearch?ContribAuthorRaw=Davis%2C+Jacquelyn&ContentItemType=research-article&startPage=&ContribRaw=Martin%2C+Denny"
-            _ = extract(url, include_other_metadata=True)
-            assert False
-        except BadContentError:
-            assert True
-
     def test_overrides(self):
         url = "https://www.indiatimes.com/news/india/75th-independence-day-india-august-15-576959.html"
         overrides = dict(
@@ -164,7 +141,7 @@ def test_overrides(self):
             publication_date=dt.date(2023, 1, 1),
         )
         # validate not the same as overrides
-        html_content = read_fixture(url)
+        html_content, _ = mock_fetch(url)
         results = extract(url, html_content)
         assert results["text_content"] != overrides["text_content"]
         assert results["article_title"] != overrides["article_title"]
@@ -181,11 +158,12 @@ def test_overrides(self):
     def test_default_title(self):
         # throws too short error if no default
         url = "https://web.archive.org/web/20111013162600id_/http://www.azftf.gov/(F(r8GSI1MAawoG8fkwp0vWYNSTuweOi8-9wgJOr4j83rTcpZDuFOV5E2PG737tNitGhzYAsUmVcwVEcgwKEtYFADTmzsQMJto9bZTOzDBHUGRpirFPIt4osB08CAslzBk-ih5ATrsM-P7DRxDwcNdmfB4jU1Y1))/WhatWeDo/Volunteer/Pages/default.aspx"
-        results = extract(url)
+        raw_html, _ = mock_fetch(url)
+        results = extract(url, raw_html)
         assert results["article_title"] is None
         # verify throws too short error
         defaults = dict(article_title="This is a title")
-        results = extract(url, defaults=defaults)
+        results = extract(url, raw_html, defaults=defaults)
         assert results["article_title"] == defaults["article_title"]
 
     def test_default_pub_date(self):
@@ -214,15 +192,15 @@ class TestStats(unittest.TestCase):
 
     def test_reset(self):
         url = "https://web.archive.org/web/http://entretenimento.uol.com.br/noticias/redacao/2019/08/25/sem-feige-sem-stark-o-sera-do-homem-aranha-longe-do-mcu.htm"
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         _ = extract(url, raw_html)
         assert mcmetadata.stats.get("total") > 0
         mcmetadata.reset_stats()
         assert mcmetadata.stats.get("total") == 0
 
     def test_total_works(self):
         url = "https://web.archive.org/web/http://entretenimento.uol.com.br/noticias/redacao/2019/08/25/sem-feige-sem-stark-o-sera-do-homem-aranha-longe-do-mcu.htm"
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         _ = extract(url, raw_html)
         assert mcmetadata.stats.get("total") > 0
         for s in mcmetadata.STAT_NAMES:
@@ -235,7 +213,7 @@ def test_passed_in_accumulator(self):
         mcmetadata.reset_stats()
         local_stats = {s: 0 for s in mcmetadata.STAT_NAMES}
         url = "https://web.archive.org/web/http://entretenimento.uol.com.br/noticias/redacao/2019/08/25/sem-feige-sem-stark-o-sera-do-homem-aranha-longe-do-mcu.htm"
-        raw_html = read_fixture(url)
+        raw_html, _ = mock_fetch(url)
         _ = extract(url, raw_html, stats_accumulator=local_stats)
         for s in mcmetadata.STAT_NAMES:  # verify global counter didn't count
             assert s in mcmetadata.stats
@@ -246,4 +224,4 @@ def test_passed_in_accumulator(self):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()

mcmetadata/test/test_languages.py

@@ -1,13 +1,13 @@
 import unittest
 
 from .. import content, languages
-from . import read_fixture
+from . import mock_fetch
 
 
 class TestLanguageFromText(unittest.TestCase):
 
     def _fetch_and_validate(self, url: str, expected_language_code: str):
-        html_text = read_fixture(url)
+        html_text, _ = mock_fetch(url)
         article = content.from_html(url, html_text)
         lang_code = languages._from_text(article["text"])
         assert lang_code == expected_language_code
@@ -51,7 +51,7 @@ def test_language_without_region(self):
 class TestLanguageFromHtml(unittest.TestCase):
 
     def _fetch_and_validate(self, url: str, expected_language_code: str):
-        html_text = read_fixture(url)
+        html_text, _ = mock_fetch(url)
         article = content.from_html(url, html_text)
         lang_code = languages.from_html(html_text, article["text"])
         assert lang_code == expected_language_code