mediacloud · rahulbot · Aug 25, 2026 · Aug 24, 2026 · Aug 24, 2026 · Aug 25, 2026
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,6 +8,7 @@ repos:
       - id: end-of-file-fixer
       - id: check-yaml
       - id: check-added-large-files
+        exclude: ^mcmetadata/test/fixtures/
       - id: check-json
       - id: check-toml
 #  - repo: http://github.com/pre-commit/mirrors-mypy

diff --git a/conftest.py b/conftest.py
diff --git a/mcmetadata/test/__init__.py b/mcmetadata/test/__init__.py
@@ -1,19 +1,30 @@
+import hashlib
+import logging
 import os
-import re
+
+import mcmetadata.webpages as webpages
 
 test_dir = os.path.dirname(os.path.realpath(__file__))
 fixtures_dir = os.path.join(test_dir, "fixtures")
 
 # mcmetadata.webpages.DEFAULT_TIMEOUT_SECS = 500  # reset to a longer timeout for tests
+logger = logging.getLogger(__name__)
 
 
-def read_fixture(filename: str) -> str:
-    with open(os.path.join(fixtures_dir, filename)) as f:
-        html_text = f.read()
-    return html_text
+def read_fixture(url: str) -> str:
+    """
+    This wll either load the cached HTML from disk, or run a live request and return HTML
+    """
+    try:
+        cached_file_name = cached_url_file_name(url)
+        with open(os.path.join(fixtures_dir, cached_file_name)) as f:
+            html_text = f.read()
+        return html_text
+    except FileNotFoundError:
+        logger.error(f"Cache miss on URL: loading live fetch {url}")
+        html_text, response = webpages.fetch(url)
+        return html_text
 
 
-def filesafe_url(url):
-    url = re.sub('"', "", url)
-    safe_url = "cached-" + re.sub(r"\W+", "", url)
-    return safe_url
+def cached_url_file_name(url: str) -> str:
+    return hashlib.md5(url.encode("utf-8")).hexdigest() + ".html"
diff --git a/...baenmarzoasesinaronatresmujeresnid1884942 → ...res/04b563732023d6ce22260200242071eb.html b/...baenmarzoasesinaronatresmujeresnid1884942 → ...res/04b563732023d6ce22260200242071eb.html
@@ -1,14 +1,14 @@
-<!DOCTYPE html><html lang="es"><head><script type="text/javascript" src="https://web-static.archive.org/_static/js/bundle-playback.js?v=t1Bf4PY_" charset="utf-8"></script>
-<script type="text/javascript" src="https://web-static.archive.org/_static/js/wombat.js?v=txqj7nKC" charset="utf-8"></script>
-<script>window.RufflePlayer=window.RufflePlayer||{};window.RufflePlayer.config={"autoplay":"on","unmuteOverlay":"hidden"};</script>
+<!DOCTYPE html><html lang="es"><head><script type="text/javascript" src="https://web-static.archive.org/_static/js/bundle-playback.js?v=2N_sDSC0" charset="utf-8"></script>
+<script type="text/javascript" src="https://web-static.archive.org/_static/js/wombat.js?v=o9P2E1FK" charset="utf-8"></script>
+<script>window.RufflePlayer=window.RufflePlayer||{};window.RufflePlayer.config={"autoplay":"on","unmuteOverlay":"hidden","showSwfDownload":true};</script>
 <script type="text/javascript" src="https://web-static.archive.org/_static/js/ruffle/ruffle.js"></script>
 <script type="text/javascript">
-  __wm.init("https://web.archive.org/web");
+    __wm.init("https://web.archive.org/web");
   __wm.wombat("https://www.lanacion.com.ar/seguridad/cordoba-en-marzo-asesinaron-a-tres-mujeres-nid1884942/","20220724231339","https://web.archive.org/","web","https://web-static.archive.org/_static/",
 	      "1658704419");
 </script>
-<link rel="stylesheet" type="text/css" href="https://web-static.archive.org/_static/css/banner-styles.css?v=S1zqJCYt" />
-<link rel="stylesheet" type="text/css" href="https://web-static.archive.org/_static/css/iconochive.css?v=qtvMKcIJ" />
+<link rel="stylesheet" type="text/css" href="https://web-static.archive.org/_static/css/banner-styles.css?v=1utQkbB3" />
+<link rel="stylesheet" type="text/css" href="https://web-static.archive.org/_static/css/iconochive.css?v=3PDvdIFv" />
 <!-- End Wayback Rewrite JS Include -->
 <meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1.0,minimum-scale=0.5,maximum-scale=5.0,user-scalable=yes"/><meta name="theme-color" content="#ffffff"/><title>En marzo hubo tres feminicidios en Córdoba - LA NACION</title><link rel="preload" as="font" type="font/woff2" href="/web/20220724231339/https://www.lanacion.com.ar/pf/resources/fonts/suecaslab-bold-webfont.woff2?d=945" crossorigin=""/><link rel="preload" as="font" type="font/woff2" href="/web/20220724231339/https://www.lanacion.com.ar/pf/resources/fonts/suecaslab-medium-webfont.woff2?d=945" crossorigin=""/><script defer="">
         const loadFont = ({family, url, config}) => {
@@ -438,20 +438,21 @@
           })
     });</script></body></html><!--
      FILE ARCHIVED ON 23:13:39 Jul 24, 2022 AND RETRIEVED FROM THE
-     INTERNET ARCHIVE ON 15:11:40 Mar 25, 2024.
+     INTERNET ARCHIVE ON 10:46:10 Aug 24, 2026.
      JAVASCRIPT APPENDED BY WAYBACK MACHINE, COPYRIGHT INTERNET ARCHIVE.
 
      ALL OTHER CONTENT MAY ALSO BE PROTECTED BY COPYRIGHT (17 U.S.C.
      SECTION 108(a)(3)).
 -->
 <!--
 playback timings (ms):
-  exclusion.robots: 0.207
-  exclusion.robots.policy: 0.188
-  cdx.remote: 0.17
-  esindex: 0.015
-  LoadShardBlock: 484.593 (6)
-  PetaboxLoader3.datanode: 250.615 (7)
-  load_resource: 128.1
-  PetaboxLoader3.resolve: 37.832
+  captures_list: 0.639
+  exclusion.robots: 0.084
+  exclusion.robots.policy: 0.069
+  esindex: 0.009
+  cdx.remote: 66.394
+  LoadShardBlock: 219.039 (3)
+  PetaboxLoader3.datanode: 110.544 (4)
+  PetaboxLoader3.resolve: 113.418 (2)
+  load_resource: 124.814
 -->
diff --git a/...0190823whatthefedcouldlearnfromcanadahtml → ...res/05856a4821f5af38887e6f3d84bfb1a6.html b/...0190823whatthefedcouldlearnfromcanadahtml → ...res/05856a4821f5af38887e6f3d84bfb1a6.html
@@ -1,14 +1,14 @@
-<!doctype html><html lang="en" data-theme="heavy-blue" data-reactroot=""><head><script type="text/javascript" src="https://web-static.archive.org/_static/js/bundle-playback.js?v=t1Bf4PY_" charset="utf-8"></script>
-<script type="text/javascript" src="https://web-static.archive.org/_static/js/wombat.js?v=txqj7nKC" charset="utf-8"></script>
-<script>window.RufflePlayer=window.RufflePlayer||{};window.RufflePlayer.config={"autoplay":"on","unmuteOverlay":"hidden"};</script>
+<!doctype html><html lang="en" data-theme="heavy-blue" data-reactroot=""><head><script type="text/javascript" src="https://web-static.archive.org/_static/js/bundle-playback.js?v=2N_sDSC0" charset="utf-8"></script>
+<script type="text/javascript" src="https://web-static.archive.org/_static/js/wombat.js?v=o9P2E1FK" charset="utf-8"></script>
+<script>window.RufflePlayer=window.RufflePlayer||{};window.RufflePlayer.config={"autoplay":"on","unmuteOverlay":"hidden","showSwfDownload":true};</script>
 <script type="text/javascript" src="https://web-static.archive.org/_static/js/ruffle/ruffle.js"></script>
 <script type="text/javascript">
-  __wm.init("https://web.archive.org/web");
+    __wm.init("https://web.archive.org/web");
   __wm.wombat("https://www.thestar.com/business/economy/opinion/2019/08/23/what-the-fed-could-learn-from-canada.html","20220721144440","https://web.archive.org/","web","https://web-static.archive.org/_static/",
 	      "1658414680");
 </script>
-<link rel="stylesheet" type="text/css" href="https://web-static.archive.org/_static/css/banner-styles.css?v=S1zqJCYt" />
-<link rel="stylesheet" type="text/css" href="https://web-static.archive.org/_static/css/iconochive.css?v=qtvMKcIJ" />
+<link rel="stylesheet" type="text/css" href="https://web-static.archive.org/_static/css/banner-styles.css?v=1utQkbB3" />
+<link rel="stylesheet" type="text/css" href="https://web-static.archive.org/_static/css/iconochive.css?v=3PDvdIFv" />
 <!-- End Wayback Rewrite JS Include -->
 <meta charset="utf-8"/><meta content="width=device-width, initial-scale=1.0, maximum-scale=2.0, user-scalable=yes" name="viewport"/><meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/><meta name="msapplication-TileColor" content="#FFFFFF"/><meta name="msapplication-TileImage" content="/favicon-thestar-144.png"/><meta name="msapplication-square70x70logo" content="/favicon-thestar-70.png"/><meta name="msapplication-square150x150logo" content="/favicon-thestar-150.png"/><meta name="msapplication-square310x310logo" content="/favicon-thestar-310.png"/><meta name="apple-mobile-web-app-title" content="thestar.com"/><meta name="application-name" content="thestar.com"/><meta property="fb:pages" content="184906186150"/><meta property="fb:pages" content="228549210541681"/><meta property="fb:pages" content="163075187111670"/><meta property="fb:pages" content="134392236659479"/><meta property="article:content_tier" content="metered"/><link rel="icon" sizes="16x16 32x32 48x48 64x64" href="/web/20220721144440im_/https://www.thestar.com/favicon-thestar.ico"/><link rel="apple-touch-icon-precomposed" href="/web/20220721144440im_/https://www.thestar.com/favicon-thestar-152.png"/><link rel="apple-touch-icon-precomposed" sizes="152x152" href="/web/20220721144440im_/https://www.thestar.com/favicon-thestar-152.png"/><link rel="apple-touch-icon-precomposed" sizes="144x144" href="/web/20220721144440im_/https://www.thestar.com/favicon-thestar-144.png"/><link rel="apple-touch-icon-precomposed" sizes="120x120" href="/web/20220721144440im_/https://www.thestar.com/favicon-thestar-120.png"/><link rel="apple-touch-icon-precomposed" sizes="114x114" href="/web/20220721144440im_/https://www.thestar.com/favicon-thestar-114.png"/><link rel="apple-touch-icon-precomposed" sizes="72x72" href="/web/20220721144440im_/https://www.thestar.com/favicon-thestar-72.png"/><link rel="apple-touch-icon-precomposed" href="/web/20220721144440im_/https://www.thestar.com/favicon-thestar-57.png"/><link rel="preconnect" href="https://web.archive.org/web/20220721144440/https://fonts.googleapis.com/"/><link rel="preconnect" href="https://web.archive.org/web/20220721144440/https://fonts.gstatic.com/"/><link href="https://web.archive.org/web/20220721144440cs_/https://fonts.googleapis.com/css2?family=Roboto:wght@500&amp;display=swap" rel="stylesheet"/><link rel="preload" href="/web/20220721144440/https://www.thestar.com/assets/fonts/TorstarTextO3-Roman.ttf" as="font" type="font/ttf" crossorigin="anonymous"/><link rel="preload" href="/web/20220721144440/https://www.thestar.com/assets/fonts/TorstarTextO3-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><link rel="preload" href="/web/20220721144440/https://www.thestar.com/assets/fonts/TorstarTextO3-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><link rel="preload" href="/web/20220721144440/https://www.thestar.com/assets/fonts/TorstarDeckCondensed-Roman.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><link rel="preload" href="/web/20220721144440/https://www.thestar.com/assets/fonts/TorstarDeckCondensed-Semibold.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><link rel="preload" href="/web/20220721144440/https://www.thestar.com/assets/fonts/merriweather/MerriweatherSans-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><link rel="preload" href="/web/20220721144440/https://www.thestar.com/assets/fonts/merriweather/MerriweatherSans-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><link rel="preload" href="/web/20220721144440/https://www.thestar.com/assets/fonts/merriweather/MerriweatherSans-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><link rel="preload" href="/web/20220721144440/https://www.thestar.com/assets/fonts/merriweather/MerriweatherSans-BoldItalic.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><link rel="preload" href="/web/20220721144440/https://www.thestar.com/assets/fonts/merriweather/MerriweatherSans-Black.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><link rel="preconnect" href="//web.archive.org/web/20220721144440/https://prebid.the-ozone-project.com/"/><link rel="dns-prefetch" href="//web.archive.org/web/20220721144440/https://prebid.the-ozone-project.com/"/><link rel="preconnect" href="//web.archive.org/web/20220721144440/https://elb.the-ozone-project.com/"/><link rel="dns-prefetch" href="//web.archive.org/web/20220721144440/https://elb.the-ozone-project.com/"/><script src="https://web.archive.org/web/20220721144440js_/https://prebid.the-ozone-project.com/hw/torstar/toronto-star-adunits.js"></script><script src="https://web.archive.org/web/20220721144440js_/https://e377.thestar.com/script.js"></script>
 
@@ -518,20 +518,21 @@
 </script>
 <script id="__LOADABLE_REQUIRED_CHUNKS__" type="application/json">[23]</script><script id="__LOADABLE_REQUIRED_CHUNKS___ext" type="application/json">{"namedChunks":["StnSmartPlayer"]}</script><script async="" data-chunk="bundle" src="/web/20220721144440js_/https://www.thestar.com/static/vendors~bundle.chunk.js?v=d333aec9"></script><script async="" data-chunk="bundle" src="/web/20220721144440js_/https://www.thestar.com/static/bundle.js?v=f1d63e4b"></script><script async="" data-chunk="StnSmartPlayer" src="/web/20220721144440js_/https://www.thestar.com/static/StnSmartPlayer.chunk.js?v=d92b0b75"></script></body></html><!--
      FILE ARCHIVED ON 14:44:40 Jul 21, 2022 AND RETRIEVED FROM THE
-     INTERNET ARCHIVE ON 15:09:58 Mar 25, 2024.
+     INTERNET ARCHIVE ON 10:50:19 Aug 24, 2026.
      JAVASCRIPT APPENDED BY WAYBACK MACHINE, COPYRIGHT INTERNET ARCHIVE.
 
      ALL OTHER CONTENT MAY ALSO BE PROTECTED BY COPYRIGHT (17 U.S.C.
      SECTION 108(a)(3)).
 -->
 <!--
 playback timings (ms):
-  exclusion.robots: 0.224
-  exclusion.robots.policy: 0.208
-  cdx.remote: 0.142
-  esindex: 0.016
-  LoadShardBlock: 625.012 (6)
-  PetaboxLoader3.datanode: 323.672 (7)
-  load_resource: 221.659
-  PetaboxLoader3.resolve: 123.866
+  captures_list: 0.385
+  exclusion.robots: 0.047
+  exclusion.robots.policy: 0.039
+  esindex: 0.007
+  cdx.remote: 5.861
+  LoadShardBlock: 63.744 (3)
+  PetaboxLoader3.datanode: 42.658 (4)
+  PetaboxLoader3.resolve: 397.165 (2)
+  load_resource: 422.216
 -->