From c4ded12ed39350582988242ecf3deffbf8fb99f5 Mon Sep 17 00:00:00 2001 From: Adrian Breiding Date: Wed, 19 Aug 2026 09:55:52 +0200 Subject: [PATCH 1/3] feat(publisher): add VOAParser V1_1 with enhanced selectors Introduce a new version of the VoiceOfAmerica parser with improved content extraction. - Define V1_1 parser with distinct paragraph and subheadline selectors - Update filter configuration for reverse ordering in sitemaps - Enhance handling of gzip content types in URL scraper - Add comprehensive VOA test data for validation and new articles --- src/fundus/publishers/us/__init__.py | 1 + src/fundus/publishers/us/voice_of_america.py | 15 +- src/fundus/scraping/url.py | 1 + .../parser/test_data/us/VoiceOfAmerica.json | 231 ++++++++++++++++++ .../us/VoiceOfAmerica_2026_08_19.html.gz | Bin 0 -> 17886 bytes tests/resources/parser/test_data/us/meta.info | 4 + 6 files changed, 250 insertions(+), 2 deletions(-) create mode 100644 tests/resources/parser/test_data/us/VoiceOfAmerica_2026_08_19.html.gz diff --git a/src/fundus/publishers/us/__init__.py b/src/fundus/publishers/us/__init__.py index 0e5d285ca..d7ed69780 100644 --- a/src/fundus/publishers/us/__init__.py +++ b/src/fundus/publishers/us/__init__.py @@ -261,6 +261,7 @@ class US(metaclass=PublisherGroup): Sitemap( "https://www.voanews.com/sitemap.xml", sitemap_filter=inverse(regex_filter(r"sitemap_[\d_]*\.xml\.gz")), + reverse=True, ), ], ) diff --git a/src/fundus/publishers/us/voice_of_america.py b/src/fundus/publishers/us/voice_of_america.py index b0715df48..148119665 100644 --- a/src/fundus/publishers/us/voice_of_america.py +++ b/src/fundus/publishers/us/voice_of_america.py @@ -16,11 +16,18 @@ class VOAParser(ParserProxy): class V1(BaseParser): - _paragraph_selector = CSSSelector("#article-content > div > p") + VALID_UNTIL = datetime.date(2026, 7, 20) + + _paragraph_selector: XPath = CSSSelector("#article-content > div > p") + _subheadline_selector: Optional[XPath] = None @attribute def body(self) -> Optional[ArticleBody]: - return extract_article_body_with_selector(self.precomputed.doc, paragraph_selector=self._paragraph_selector) + return extract_article_body_with_selector( + self.precomputed.doc, + paragraph_selector=self._paragraph_selector, + subheadline_selector=self._subheadline_selector, + ) @attribute def publishing_date(self) -> Optional[datetime.datetime]: @@ -46,3 +53,7 @@ def images(self) -> List[Image]: upper_boundary_selector=XPath("//h1"), lower_boundary_selector=XPath("//div[@id='ymla-section']"), ) + + class V1_1(V1): + _paragraph_selector = XPath("//div[@id='article-content']/div/p[not(strong)]") + _subheadline_selector = XPath("//div[@id='article-content']/div/p[strong]") diff --git a/src/fundus/scraping/url.py b/src/fundus/scraping/url.py index c6d2c1922..33e4ee01d 100644 --- a/src/fundus/scraping/url.py +++ b/src/fundus/scraping/url.py @@ -79,6 +79,7 @@ def __init__(self): self.archive_mapping: Dict[str, Callable[[bytes], bytes]] = { "application/octet-stream": self._decompress_octet_stream, "application/x-gzip": CompressionFormats.GZIP, + "application/gzip": CompressionFormats.GZIP, "gzip": CompressionFormats.GZIP, } diff --git a/tests/resources/parser/test_data/us/VoiceOfAmerica.json b/tests/resources/parser/test_data/us/VoiceOfAmerica.json index 2c355f31d..5a9a736f0 100644 --- a/tests/resources/parser/test_data/us/VoiceOfAmerica.json +++ b/tests/resources/parser/test_data/us/VoiceOfAmerica.json @@ -92,5 +92,236 @@ "USA", "United States" ] + }, + "V1_1": { + "authors": [ + "Rikar Hussein", + "Zhiyar Mohammed" + ], + "body": { + "summary": [], + "sections": [ + { + "headline": [], + "paragraphs": [ + "Meta, the parent company of Facebook, has removed two popular Kurdish Facebook pages accused of spreading misinformation that helped convince thousands of Kurds to mass along the border of Belarus and Poland late last year.", + "The two accounts, one from a Kurdish lawmaker with 143,000 followers and another belonging to a Kurdish journalist with nearly 270,000 followers, were spreading misinformation that falsely claimed Kurds who went to the Belarus-Poland border would be allowed into the European Union.", + "There was no such immigration plan. Instead, frustrated crowds clashed with border guards and thousands were later deported.", + "The false posts were among many seen by Kurds who traveled to the border area and were interviewed by VOA.", + "\"We followed the crowd towards the Polish borders after rumors on Facebook. It resulted in nothing more than adversity for this destitute people,\" said Hersh Saeed Ahmad, a Kurdish migrant in Belarus.", + "But the accounts on Facebook continued to publish widely read posts until earlier this month when VOA contacted Meta asking if the pages were violating the company's policies.", + "\"Meta has decided both pages violated our policies for misinformation under Violence & Incitement Community Standards, and both have been taken down,\" a spokesperson from the company told VOA in an email.", + "The episode illustrates how the social media network continues to struggle to police even well-known spreaders of misinformation who are involved in high-profile news events, especially when misinformation is being published in languages other than English." + ] + }, + { + "headline": [ + "Spreading misinformation" + ], + "paragraphs": [ + "The Belarus-EU border crisis began last July and worsened by November, when thousands of migrants from the Middle East, North Africa and Iraqi Kurdistan, attempted to cross into the EU from Belarus.", + "In mid-November, violence broke out at the Polish border when security forces used tear gas and water cannons to prevent migrants from breaking the border fencing. Polish police at the time reported several injuries in their ranks from migrants throwing stones at them.", + "At the time, Facebook told news outlets that it was working to shut down information about human trafficking in the region.", + "But two prominent Kurdish Facebook pages continued to traffic in misinformation about the situation at the border until earlier this month.", + "Sirwan Baban, a member of the Kurdistan Regional Parliament, and Ranj Pshdary, a Kurdish journalist based in Greece, used their pages on Facebook to tell their followers in early November that the EU and Germany had decided to open their borders to let in migrants stranded on a Belarus-Poland border point. It was a claim denied by EU officials at the time, but thousands of migrants, mostly Kurds, still stormed the Polish border fence and clashed with police.", + "VOA interviews with officials in the Kurdistan Region of Iraq and Kurdish migrants in Belarus confirmed that misinformation on Facebook, including from Baban and Pshdary, helped to lure thousands of people towards the Polish border.", + "Hersh Saeed Ahmad, a 36-year-old Kurd from Sulaimani province, is among those migrants who, after reading posts on social media, took his wife and 4-year-old child to the Polish border in November.", + "\"Our situation after the storming turned from bad to worse,\" he said. \"Our admission by Germany was nothing more than lies and rumors on Facebook.\"", + "Another migrant, Bahadin Muhsin Qadir, said they were told the Polish security at the border had announced through loudspeakers that they will be transferred through buses to Germany.", + "The standoff at the border during extreme weather conditions left more than a dozen people dead, according to human rights activists, who say the total number is likely higher but hard to confirm due to restricted access to the area.", + "Ari Jalal, the head of Kurdish foundation Lutka for Refugees and Migrants, told VOA that his group registered two deaths among the Kurds at the border. He said about 4,000 migrants have since returned to the Kurdistan Region of Iraq, with about 1,100 people remaining in Belarus camps.", + "\"In addition to bodily and material damage, the Kurdish migrants are also psychologically broken down completely,\" said Jalal, while expressing his frustration at \"how gullible Kurdish youth can be manipulated by livestream videos on social media.\"" + ] + }, + { + "headline": [ + "Facebook's efforts" + ], + "paragraphs": [ + "Working with independent experts, Meta says it works to detect and remove harmful false claims that could contribute to the risk of imminent violence or physical harm ­— such as claims in Arabic and Kurdish that either the Polish border is open to migrants or Germany is sending buses for the migrants to the border.", + "In November, The New York Times reported that an account belonging to a Kurdish-German influencer widely known online as Karwan Rawanduzy was disabled on Facebook for frequently promoting bogus stories that fueled the crisis.", + "In early December, Meta released a threat report saying it removed 38 Facebook accounts, five groups and four Instagram accounts linked to the Belarusian KGB that were inflaming the migrant crisis. It also reported taking down 31 Facebook accounts, four groups, two Facebook events and four Instagram accounts that originated in Poland and targeted Belarus and Iraq.", + "Despite those high-profile takedowns, the social media giant missed other far-reaching pages that were still being used to mislead migrants." + ] + }, + { + "headline": [ + "False claim of open border" + ], + "paragraphs": [ + "In the case of Sirwan Baban, a lawmaker, and Ranj Pshdary, who calls himself a journalist, both with thousands of followers, Facebook for months served as their main medium to encourage migrants to amass at the Polish border.", + "Pshdary, 32, from Iraqi Kurdistan's Qaladza town, has gained recognition in Kurdish media for his role in covering the Belarus crisis. On November 13, he went live on Facebook to tell migrants \"the great news\" that the EU was going to open its doors to let them in. In the video, viewed and shared by thousands, people claiming to be migrants in Belarus or their relatives, joined Pshdary's call in encouraging the migrants to prepare to cross into Poland \"in the next couple of days.\"", + "\"The Polish border will be opened to migrant on November 15 and the migrants will be sent to Germany via buses,\" Pshdary said.", + "Three days later, thousands of migrants, mostly Kurds, headed from a Belarus forest to the Polish border, anticipating a crossing into Poland. They clashed with the Polish police, but no migrants crossed over.", + "Pshdary, in another live Facebook video titled \"I confess that we failed,\" admitted he had intentionally misled people.", + "\"I don't want to conceal from you that I was the organizer of the crowd. On Friday, [Nov. 12, 2021], I met with the representatives of the migrants... Seeing that the Belarus police were torturing a lot of young migrants, there was no option but to encourage those people to cut the barbed wire fence so that those young people can be seen as perpetrators and violated against [by the Polish security].\"", + "Pshdary said in his Facebook Live he believed that falsely saying the border was open would have created a spectacle with women, children and older migrants out in the cold, thus embarrassing EU politicians and forcing them to open their doors on a humanitarian basis.", + "When reached by VOA, Pshdary insisted that his plan was \"good intentioned\" and aimed at helping the migrants who desperately reached out to him for a way out." + ] + }, + { + "headline": [ + "Lawmaker resigns from diaspora committee" + ], + "paragraphs": [ + "Lawmaker Sirwan Baban, who served as a member of the Kurdistan Regional Parliament's diaspora committee at the time of the crisis, gave similar false hopes to migrants on Facebook and on TV.", + "On November 8, he appeared in a live interview with the Kurdish media network Rudaw, which was streamed live on Facebook with 755,000 views, claiming he had access to a \"proclamation\" from the EU: \"It says the migrant situation in Belarus has escalated and become tragedy and a global issue. Therefore, the European Union has met tonight, telling Poland, 'Let Belarus continue its dictatorship. You open your borders and allow the migrants in. Once in Poland, we will distribute them among other European countries.'\"", + "In an interview with VOA, Baban denied coordinating his false information about border openings with Pshdary, claiming that he had received reliable information that the EU's refugee committee and some German officials had made a \"recommendation\" to let in the migrants. However, he declined to share the source of his information with VOA.", + "\"This issue is portrayed this way in Kurdistan only to implicate me,\" he added.", + "In addition to using his Facebook page, Baban also went live on several Facebook groups and other social media pages to promote the story which — at the time — was also denied by Kurdish Foreign Relations officer Safin Dizayee in an interview with VOA Kurdish Service.", + "Among videos Baban posted of alleged migrants celebrating and thanking him for his efforts to influence EU officials is a young girl introducing herself as Saya and saying, \"Mr. Sirwan Baban, the parliamentarian, thank you very much for such a great news … I will see you in Germany.\"", + "The Kurdistan Regional Parliament's diaspora committee in an urgent statement on November 11 accused Baban of spreading misinformation \"that pushes the youth into harm's way.\"", + "The head of the committee, Rebwar Babkai, told VOA that Baban has since been forced to resign from the committee due to his role in spreading the misinformation.", + "\"I hope this is a lesson to all of us holding a public position to feel the responsibility of our jobs,\" Babkai said, adding it was unclear if his region's government was going to take further action against Baban." + ] + }, + { + "headline": [ + "Limits of Facebook's enforcement?" + ], + "paragraphs": [ + "Some social media observers say Facebook's failure to detect those pages after months of misinformation shows \"a gap\" that needs to be filled particularly in non-English content.", + "\"I think Facebook needs to do more when it comes to content in local languages,\" said Dlshad Othman, a Kurdish cybersecurity expert based in Washington.", + "Othman said the social media company has improved over the years in moderating content in major Middle Eastern languages such as Arabic, often at the expense of languages for smaller populations like the Kurds. A Facebook representative declined to answer VOA questions about how many Kurdish-literate moderators the company employs.", + "Moustafa Ayad is the executive director for Africa, Middle East and Asia at the UK-based Institute for Strategic Dialogue, a research group that monitors online extremism and disinformation. He told VOA that while Facebook's technical advances such as the development of artificial intelligence have been helpful in countering disinformation, the company needs to do more.", + "\"I believe all of these issues can be fixed with effective moderation in those languages,\" he said. \"It is not only just language skills but also an understanding of what is happening in those countries in a geopolitical and social level.\"", + "Facebook says it works with law enforcement, academics, non-profit organizations and others to detect and remove harmful false claims, ads, posts, pages and groups about people smuggling over international borders, and did so during the crisis in Belarus. It uses technology, human review and \"reports from our users and trusted partners to detect and remove such content,\" said a Meta spokesperson in an email to VOA.", + "\"We remove this content as soon as we become aware of it regardless of who posts the content,\" the spokesperson said. While their pages were removed, the journalist and lawmaker still have a presence on Facebook.", + "As of January 19, Pshdary, the journalist, has a personal account with 3,600 followers.", + "The lawmaker Baban's personal account has over 15,000 followers. On a recent post, he invites viewers to like his new page created on January 22." + ] + } + ] + }, + "images": [ + { + "versions": [ + { + "url": "https://gdb.voanews.com/5710b053-9f6a-4469-8775-f734a688ac38_w250_r1_s.jpg", + "query_width": null, + "size": null, + "type": "image/jpeg" + } + ], + "is_cover": true, + "description": "FILE - Polish police and border guards stand near the barbed wire as migrants from the Middle East and elsewhere gather at the Belarus-Poland border near Grodno Grodno, Belarus, Nov. 9, 2021.", + "caption": "FILE - Polish police and border guards stand near the barbed wire as migrants from the Middle East and elsewhere gather at the Belarus-Poland border near Grodno Grodno, Belarus, Nov. 9, 2021.", + "authors": [], + "position": 329 + }, + { + "versions": [ + { + "url": "https://gdb.voanews.com/C43C2D60-9DB1-4E2F-B9B1-F380925E6AD8_w250_r0.jpg", + "query_width": null, + "size": null, + "type": "image/jpeg" + } + ], + "is_cover": false, + "description": "FILE - A migrant man sleeps in a camp near Bruzgi-Kuznica checkpoint on the Belarusian-Polish border in the Grodno region, Belarus, Nov. 18, 2021.", + "caption": "FILE - A migrant man sleeps in a camp near Bruzgi-Kuznica checkpoint on the Belarusian-Polish border in the Grodno region, Belarus, Nov. 18, 2021.", + "authors": [], + "position": 349 + }, + { + "versions": [ + { + "url": "https://gdb.voanews.com/3EC5B4E3-2405-485C-BD7A-3623D75B9064_w250_r0.jpg", + "query_width": null, + "size": null, + "type": "image/jpeg" + } + ], + "is_cover": false, + "description": "FILE - Polish servicemen spray tear gas during clashes with migrants and Polish border guards at the Belarus-Poland border near Grodno, Belarus, Nov. 16, 2021.", + "caption": "FILE - Polish servicemen spray tear gas during clashes with migrants and Polish border guards at the Belarus-Poland border near Grodno, Belarus, Nov. 16, 2021.", + "authors": [], + "position": 364 + }, + { + "versions": [ + { + "url": "https://gdb.voanews.com/4E61F2FE-6E85-48F3-9062-F4A73304CA4D_w250_r0.jpg", + "query_width": null, + "size": null, + "type": "image/jpeg" + } + ], + "is_cover": false, + "description": "FILE - A migrant holds his child as he waits to get meal during a snowfall outside a logistics center at the checkpoint \"Kuznitsa\" at the Belarus-Poland border near Grodno, Belarus, Nov. 23, 2021.", + "caption": "FILE - A migrant holds his child as he waits to get meal during a snowfall outside a logistics center at the checkpoint \"Kuznitsa\" at the Belarus-Poland border near Grodno, Belarus, Nov. 23, 2021.", + "authors": [], + "position": 381 + }, + { + "versions": [ + { + "url": "https://gdb.voanews.com/1D62A8C0-F11B-4339-A56D-47B5CEE310DF_w250_r0.jpg", + "query_width": null, + "size": null, + "type": "image/jpeg" + } + ], + "is_cover": false, + "description": "Ari Jalal, the head of Kurdish foundation Lutka for Refugees and Migrants, is seen in this undated photo. (Courtesy of Ari Jalal)", + "caption": "Ari Jalal, the head of Kurdish foundation Lutka for Refugees and Migrants, is seen in this undated photo. (Courtesy of Ari Jalal)", + "authors": [], + "position": 389 + }, + { + "versions": [ + { + "url": "https://gdb.voanews.com/270ED6A7-3E48-4438-917C-26128924BC2D_w250_r0.jpg", + "query_width": null, + "size": null, + "type": "image/jpeg" + } + ], + "is_cover": false, + "description": "Ranj Pshdary, a Kurdish journalist based in Greece, is seen in this undated photo. (Courtesy of Ranj Pshdary)", + "caption": "Ranj Pshdary, a Kurdish journalist based in Greece, is seen in this undated photo. (Courtesy of Ranj Pshdary)", + "authors": [], + "position": 408 + }, + { + "versions": [ + { + "url": "https://gdb.voanews.com/73701B57-30C7-4826-9612-9894864EECD0_w250_r0.jpg", + "query_width": null, + "size": null, + "type": "image/jpeg" + } + ], + "is_cover": false, + "description": "Sirwan Baban, a member of the Kurdistan Regional Parliament, is seen in this undated photo. (Ahmad Zebari/VOA)", + "caption": "Sirwan Baban, a member of the Kurdistan Regional Parliament, is seen in this undated photo. (Ahmad Zebari/VOA)", + "authors": [], + "position": 426 + }, + { + "versions": [ + { + "url": "https://gdb.voanews.com/1D1A3D73-6F25-4735-A699-F03512858C32_w250_r0.jpg", + "query_width": null, + "size": null, + "type": "image/jpeg" + } + ], + "is_cover": false, + "description": "Rebwar Babkai, of the Kurdistan Regional Parliament's diaspora committee, is seen in this undated photo. (Courtesy of Rebwar Babkai)", + "caption": "Rebwar Babkai, of the Kurdistan Regional Parliament's diaspora committee, is seen in this undated photo. (Courtesy of Rebwar Babkai)", + "authors": [], + "position": 438 + } + ], + "publishing_date": "2022-01-22 14:22:22+00:00", + "title": "Facebook Removes Kurdish Pages Linked to Misinformation on Belarus Migrant Crisis ", + "topics": [ + "East Asia", + "Technology", + "Extremism Watch", + "facebook" + ] } } diff --git a/tests/resources/parser/test_data/us/VoiceOfAmerica_2026_08_19.html.gz b/tests/resources/parser/test_data/us/VoiceOfAmerica_2026_08_19.html.gz new file mode 100644 index 0000000000000000000000000000000000000000..da53fa6073d1c433e228263b0072c2068a31f95d GIT binary patch literal 17886 zcmV)VK(D_aiwFSbV1;S||LuL*b{n^r=zFi#|A6q^6G|NxXHkPqrDa)8+{aT#cG9QQ zS!%EfB+DvR@f4~=GdW(XKcc_kzTKa=pVD7)_uc?%<|IqbaHYf&S%qO^V`I+%{_!6k zfARG7`P;v~xnR?1=AHiIKRm|2%;WxKSQCDYIc_+tc~Mwnw#TI$*8ISFFKbj%a0mV} z{qxv!{c9EqZ&;Jj(i3tjL{wwbP>hGQ)_LGZ!jD>^kn_No?u}@D|N5*YeCypMezNFO z)0Xn!(>nE0H+?J?i{>(j zqj)5!3wIpynYiL}x6U{=oi>$i^@Y;b_j+T9-fJ~%%7so0c!5lkko4-N8>Cmd*G%snYe z#omPC#ubY#pCO}S@5_{JGmymrTI?~7bt`|OwWHU2DI-w77Y@19*i{mhrY z|DFBv@zJWXO~Sbzdg@hnO40NJhZ0-oZN#(o01;vm%44W*mq(U+yH$} z1v7xwRf9m_VZONPr|Ad}-X-IzSGN0Z?(j&p3lIT@qA4uP?fOYUhB4_*qu??Mfxh~y^_KkSC#)*c=T~UgUBou00lfJ*2}(R z7qbyCgoI({SFymCZ^8gLW8@0N4=um6P!ukUV7XoWzA-JQT=& zxz7~$Z{}+&-l3sOrUOUJz8EIofs+@;Ux2jf$}872V%jUM1Wi`zZRKPYo2BnpHcH|T zI2pAWcl}xc7EQDj%>izHs6YXGA)>k@WCFIhZ3`)BF~agfBCKraEi5WicHw#+NH}i^ z%+~d2Ju?Z*$&Q#E22xr<=mP&|duF9)q#BN3zoHAA3Dy4#&E!}X{|W0WVKsqI81Q5a ze}yBQCVLeukP5_yx#!w$1Q3HFfZfn|r-6+8GEtbhDluJMoEUc=*5uvFHMntV)dM8G zTGe7(Imvqb-x}axnjCjL76=~O6U|98^IgUxH3e$Yiu3YGpwraXsu@M)lwG=@ZO}v* zx(dQ;*}w_2fm$UW3z#ruLs$lQ1`wNUZeBIO->=m4u<|?s2Yj@-i3n&#q>W_gFhLm= zDK5pVD1jb9XzhtG+^=9-6$`TgA07M(Re;|IwjGC|a2gC0;t?>orO_U9iwyL0T6!?L zB68MosVI!a2Bx-Yo~y@|*zYB|?rT-6&R+ zvmRGDnk$V1HA&PzHXyt}byOq8YdunHAs~3EERj#19HcFL9=buZAdHE>E_hKT&$3^+ zI7e0NsVk|B1~##g^jDY*=yJ+KDWYL5j>gtejkQjTc!F4+svHvk$K(@R&nKG{RW`{Dm5!sB5q;msuEfROVLibw=VHC^bB;6e^KtT{q~ z3=P$ot?bxg?QD$3(cfa2Xk06790oH6hHQhqgsP1!C6SQumq=e`01coV9035k2mpDZ z1y+(h1r?I$PLhfy=kUhmhDqD~0%Rha1`UrlVJyO^#a;WV1uEi2q zr6V(gyWl{+XVQgnycPD;54>QqY_N+D5rFN=8T$db5h$m6IB6%@hS5Tr2Vsd$mO0TZSH|7(`*|8$hL@9xF!A;yEntyzgh3jtaC(aMBIP9Jco^>wkT_TfSMxQULS zn)2rWy9j(2rl+_xv%}`|xhJeyFmmC~LX0eUVA0yj-~bj#VVIj*kYH{Blp|OoNMg|xffFwMZU(-WEw#=P9H#^%X99(gcq5pHgM;*4@gJUHyMN9}{2 zbvzz$tKT0STSte72iEwo*XM(yBX0MOt`?ncyL~ks^xDn$^GRW7#B5i0j8&^@oRFAU zT@N~|tTq+yWE!om2Ryvp?jE+=2mRyTVYim&_`nOUTxWId@7!z9alVbE6ktPZZgUoc zfw~5_|2}n>Q0rwd<+GV^wyh@P*^GzFE0RxF3XQsI1cnzdffJjDYk>V{J(0y)NNk^p z67n5yAccG3ta%9jFswB{tZLp`WrkcFS|V zdM|NkzZjNwsXTwRlUk~%)*3Zeo{`On`r)%6d=?5(J3%A0QNz#)zzN^)h0dUF{{{Ca zF;e(Rjlx>+Fj2Bkc2}+&HLf_s$&bI({+VRpN$rgY5fhCXN}H3_lSVaLA+jrLs@PR) zG;yzPplX&{PHG4`t;>_Nm*@E4_2@ldx%RP9Bf)f1V16GPw}t9w;)kH|DSp^;qwppy z!ByazXkv5rK8Qmf)NC}f(f0&v^FLzb=jkL$5u%yV+%&Etutk2?>$iK(VP|9=I-{=D zcWm1_>i0(0ako7Xqrs?e9}I}er3flA`j0=spuiw_qvfvyWxDNd*J^jHZkKiXC*3am z|L-+b>}BA%V|QJtRFK1Z0ZW$OF-Khtc@R6D>u+Nwho1pps(V2Cf@7k-xkBQF^~f` zf*zlsP|o$i%7Aq|%eK|3{gljSQE6j)&=Ny<_HlK30EQ3q_DYo|jA%z!X4ul6iWzSP z;iQ(w*qPRK8|09`@If{$Uyt7?2J-7EwHp`lXMiBj34YsB`9H${@Uy%|FUL;%sGY7y zF!&p{;zC<$UCX-Mj4fKfu3TyHvGq*<}YRcXsYl)w*4mhusf{yP}oq?9?5) zV3A`760PE$Iy*T0aB#S`vpfoqVaktpCKOd`hi4v-Kbg>-Q?WM8H>nz>w4lG zEQm1U+5FR(es*c%xpFEbO?(P9S22THg)tXOsrV1s;^Z4<7JD1a*<}=j7~%WzyUWX; z5*+fm+p6pZ-54`IU)zUnLl|Fn^Yyb{L#uQf28^`w^C|05EGJSTMdf%$x8e8sioiJtpfBaD4{^&UX`s{m8Wp2;g%e}YSRdDQG5 zkye7r2%zi64Ff$zn1Bmt_2Nl7rn@6*>+3b=0P@ISKGNi?GjNySz%n~!jF!s;}KDBiBnc3ajzKH7fZ)kDqqRh&bF28>Kc zgx^G%qgzAAVlh<-Q@%2VhNf6$O4nYJ<+*4$4cYJ-?*1=}AHAQ3`0MX~M?m3&0`8YO zfY+1}EY$i5=H))cgw*}!L_}|aukGzO6k@2@dszn>M2N>j2aD(oaQS5UmQT=Ou_yO` zZ2#1F53g;&UnF!1c;0Zlq4&kGo?xf`b^S|kSpUj9sn;L9Z&KR(kaBbqY#-lK6-+3K&*x^aD2s?if7?7jDm@&2Qa`#NQ=VG>{( zb)A^s$UxMn!!od59ld;W_@=(UI)f@hq>o#A>%!wv;4IC?2GEUeEI+U??-tg%ZC`=0 zKND;|u{2A}e6%cJ0Wd}&VlSTgY#zG6l03dl%EnSy5RFVaLqKNsh23!nS-}1x{eDUZ8SkuV$ zkY!F?#}O1LP!Pa}1gJmkN`6?&zuUFEUgUgXS*cV7fZ2P=ZbT?iMiYiDtC%VBskB{{ zc8pt%uAC|p;Fph$ypaG#0BWR5!9%5uDh+i88t(L@*6tXE6_gwE4zvO&+v%%s3n8xQ z^(Bu$S>a2iaJb_=98xgedn-UUh-W|^&%$7~52RZ+&Yv`^p7=F;7Ui?YCjFB{{Gq` z<(3~ASa5I!5Y!{+IoBvYg{De)_*j(#zL%z}X0`)hj`o*dCT%toYY32ElfFBji>S8$ zOGN?PPoWrhy{f|~T>et*G}mY5OF1olm=wN97Vq@mA#k z=8u%rya$c#4yT$p;5D`vJ3{W&OdoF%btxFN{Z*i8Y ze&y|4ctF3jtxNC8qJ&S%-h~hFzw-PKhM&93o8DlUwe;1)!SLzO9?<8>kkAd-_;Ys* z{}((K@C0Any%zu)%{sRfemVfxM1h!oBz;N_=}$*#R|j2Pen7L^ZEJ6ju1dq}sPWkc z!_;Y`-XGxW1NC-Vd~0_OhWXa0ct@A-4>kFymwkNK*A@FiJtAF^UUt;mn{IpP2fE)csKq)GZz7i1* z3sXj~a9)V7Ch+-k2x~?Ia}3W_ZT;W?5veJrJ?sxOp>*0f`gc&&I>YtGYK(hPR;Mzy zOD8pGI_Rj)7$%x~T=)(JGB{&w^=F)Y< zX;N9!i46=7D18)sV89GRbRx_GhkLm;OJ%RFTt5nfY#%^XE6Z6)!efra9*^QE0>P}S zjUu1HKg$Y(*e5SYrX74)Zr8^o7ZbvHg&e z$@u6ngdmo;pLtJ=cbj`vlPtf{hgw6-PZuJ4PGC7-OM ziAvE&Dm-PqRK%DuzKpI)XhpY1vE^zhp_GR?MWEP%yV0t0dg`(z%e#gur(_lDolPpo zmMI=hpX&>GN+U3vv!wDK9uq0Q$N`pA0uF%DSqfn|+BoUVjBKlNEsiD53o z6_j0FT>+efKtEg4Of^a>$4`%ev^Ojg;{?xC%py!L8}8h=G6##A83aXFZ9v8YtR)WnD|uX^>WTewoubEdUO z3nNb`zMu?yO%y9nKqu?1AirW`!8|I9+1OiC+?s$Knpe(+=bC!t@4BLrEc4m2=H?Q+Bm_FgFaeGZ z$?@FzDBz(}R3Gv~NIg7xh^9_xb=yq+v8<6hsRBs7PGG5~LQ`nQKhWU9GL=*nN5EM2 z3mWk*?ki@FHTFccv8EOn`P*ncq8krz=Es&Dg`W1r$%PxiLJdK2bSD!5#V9)>`_fV; zz?B;?Bof2xkl@pen6WX^M$J>*sCp)Dpg|jT#N1=#O4!3zMvSr6X+>uHhgpDE#RuB8 z!xX%#Oy;w-)@idA0EBp({)0Kt<;w2y?ON+;C1=c#3{B76_srTcQ$SO=-L&p99Th!S zEP+xrpQqKZcl@5IABkDvE10%!D5G_a7IYHRE(jXjXv%FHs(!qAYc{GEK9+wQ|9NbLFf2D zZ_n0z|68y<^?;IUK+1S5X^oxYTlBhT89B+_Euc@4tVdH#ZaA^jbLY6-Esa=lt(`|r zwKDyqr56UCda(_!Q!H{VZ20rehu03HgGA*9*_rSs?Lc&bg`WW*AFpyJR<8BjYD=1t z@yJ3FuYy*KsoPOe^&99eA+K2Sgsk2*FPpThwoF@4%u4-)Kx9c-MQQg8CHFjwc7R&R z=_(rLGtXmZv^O+4(4`1_Mdi~{wd^of2czz786EY0yUVBoHvIV^2tDWj_vORKd1@T5 zAVGSbUqPzu=UG0A#$^I0p09eAJS;)0-{j%V*7+<@3l3HgZq!M@PLA%VfEk|V;Y|vE z9$;hK&Xq!{zZSt%#R`DZv(EzM%yk@3ScusK)R**(UC^^4yqOAlpfj~T3*-`b_0+Nj z@G|b@sp1k}F-mb^=c>R8BB^TRbXE1cl?ZpN7OvX22x>1Lq#;{T8XtZV2Hdf6HN4Sp zYmA+ipWaqy{7gejvtCdgz@4Nrw|CW?`h%l068383acr6Psrmt$XK2>sivPxj4Ks*I z>-oIZi5HBx*KH*Oeu27c#J-{D>>LwUXa_Mnt$wp&kX2_nZ}1uCU&Dyjyic=rR+m`Y zeYI^`Wn0R?cDc8W6{q4w2jzBAjZpESXMyJh3-%5(!VbzzfLBmzPGJvp-HQEgiqbxouP95QB0Lc{i`az5ur;{ z?oJV!w0cJf)$QFWLRXwwRf)~#rtC(1x@pNf#b~D8JHlzw;%%|IO=ijV@?aarpUh*m zDpA?464BdAl`2G;z3{l}bgf&u_>(9oTa2v5Pj8RiRqs|lFGeMzU~Z}XK(`G`lv*%Z zT=*QHny5s^A%CCog<`*|bmh}eh&kw+khYCF>;%-EL?TqX!JL^77OZ>Hk&;l*hKbxF z=unx^QxRaKePf)`c+eiK6@k3Eq7wP9Ie;+~LJ#4!N>2Y)znO_>8o*l$u!I3}a5akB zX)z>T%7!myjGDCfEP{0uKiw>*fGM@5Zv?0@y6S2gcn+=`O(zt>Z8n=VeIduSSi-v` z1FQles@Fu^0K>NQffj)ewb2^~)Hr5)^t1_&&I3o?h|}rqvtQUVHx$VhkGFiH_Hz3Q z+`YO2tlnV%3lDn(9B__ilmC1%g*EoP2qOLO@cLZo_Krp($1sr9eS(#I?T`WJ%Em`VD3&Z>uer|rAo5&qh)nIZA2;j1>sZ;Jga_e z8ID%=pK4CG)eL2{oAQ8GJhsZrG(X-dD`E>BW)uJ~UTGw3QZ7_&tzYQ%ohlRRtwH#1 zn#8Sj2s1-ki@1CJ!HoG=QX%d^c~B#|H~rzWY7SfPOxacFQ_^8`;*N`wHho)Tctn$n zH&lE4=jA3*dpi)WY&=vGHy^DAgry%*`=CQOw+3z9mW0gNG^hEihm*VD@Z_rW7U%Rv z;Q7Rw1~XCLXQ#~SlnyAu^I1Ubbf5d#i{<_ni*R`?B29L=N$^yQbPGH$;8>+A7YiikaCor5;M zGkyG6Hz!Bbf1+zy>tpwWaO#zZIjN=ZA`13J~0%h@~SrOQ7t&ww#y(6LScCz{jJUhKi!I6 zOxFL3=az*%06C8U=ZS!x1`CGIck|h0l7?>Q%KH3+$HuD$kI!#4c-(-q> zQ*`2GlAw@e)%@HV;N=DBJxJ=b4xFT9$_l`#{gm56rh@IA#Gg--{b>a*V1=VZ#}l$-lTFfn+SrAoLNZ{=?bNCDg}OH?tGf7qHCGh*pp>- zs!FCf8l?dL2G&Vg%V^iZe2+>h$0{i>FoQG-S`VCwWW;b{4$a>6iXU&L6E0IOphC^ZJD8 zTZIPK^MHF33%5x@C{3DhFs08@enYvDJAYwWbi%|O=n1dUqYJMH87%chGd({jsG zNO6e5u@;C`G$By-Z+v9ooy^`+&j>5MRnLtd69YWz2uoQ0HMUh2gHj%evg6SERqqJS zfMSw#?&F@*Y&VqMvsG$J#gQj^7Ot-5!5p(Vx6f@HtZYiFU6%+RC`N;An_P8L2@DWs z=31~)PkM`W&8VlY4eg#keU*aH%g!psUa7_b$|JQo^f<`>jbA|D3d4 zl~aC)q^-yFD^m81yIvHWe2sy*-w+|on6980-iNB0!(J7oT_92liuzi@+KQ9}wt8QL zL?G`@MjKDMx^>2Ra-_`KQuFj66*`lMl#P-BR*3QKs%_I0(eXf~b|Z{8u8t{|p({@q z{q78|j?0%nr&}?MMlo?lIph7{u+tv34|>+|c)+cGe{gIa9UdN7q|>3CSig4;_j30K+q1@DSh>Y{bJ6 zFrtMExZ#p%&MX;T>q-SOr;Jf+!job#6;OP_;eE(>MCCNMwV?k@r_|3kVc__I{?jle z8|+nZ(`3h#!qjQ5J`H_szSiz{!BpJ90xIV38+j|T|j|D?7U= zP_rl}0`NuNd6VjFx0OB%%-}MaYsuY&1!+F^0v>_JM7vSOBkON* z5aG3c)k)P0xzPNsn1l#Y59<7IcKPk|SKqvS{Yt~}KmYaLCSgp|$m99xOMzE!BjqKR z+|1zR67Q2V8drl3fmcc%XjqE?mc={<>Pu9xL^mVN54SYNQqY~0a*pm46uEgN5V{14 zp9*h|w~ht=jqAfYi>5&=f$k+%#m1$&FHu69UI2~IT2pHRN*KKkSBhRR9>@JGc_R=r-tR zN$H0X<|8^ng31|3=02;2f{ID*I_z60J!O|%K!az~8BRhJf zmQk`kT<%-Bxx(M#5rkD#MO%QebF6oBMG6d_spjQkx%SuEzR#M22}&(mT=GTfbq?I_nAQ|6s7 zKx~KUypc`-HR8#{6EufPNynoD0O6izUHf>nxst4OYxTH@8k88E1j0?=-6-}vbthB2 zs$%Ryv$(98x*xs4aG(ef5C~0vp)nV4$RV$S*(ZX{rDx44b%7ZEg}VPdi;KrF6p(AD zm#Y*3SR|@M0f9#9U%WGGPkGS*m5Qd_Qcc(d3k_wU!Gm z-w;Scot94RJ`efdTrGlth=6_~pssnOZwd+nDKmP`g|Xi%Q+(><=Ke zHL4_PTQ+hVT#bC0&WRI~A5JNZQYnlHUbH@+nPsCyZfPW!1kPYP8Q!}%KX}r==vm!< z`@rfS9h_TFo*tfAy+OD4^zh)xaeL7JO~~-yDt_-vpr56St&vJ{5;Mf9fjxMFRul}u zjmUyQp`ooaIR*AX_g!QeE8RoFfTjV@Ab`eW*H#i-sW3`+$ucQYE71IfRSY&ZaE}!R za8AY0O=GhK&aMC;fd?wTeimhU&nhJ(3)Q6?E?@*t_=xx~NKNFR$jq^!gf+A8Xgt82 z-e7YaRRAUKzvKRU_C`+eNRmdiB|G9$Nwsf6A#9if#bJrFsUc@IEF3h*Vf{X}o}s=n zh7Frn^QA=tg7S?B$-bL{8fH`H3!cJ^E2L_&MlFJZDTu}uL`+I-(~uQ2 zAD%0c;mXj4u6vn;>CMl^L5qO$>0BqL87rw!ay2DktgAI`K%nO}Qj z5_1S+&U{v4COgqeq0$ztUehUXnre&ljgX;`rVa|GqRiNeNV8*CgUF*Isb;3P0kv!| zCRIg=bEG`QR16Vm#|}QczsRICb22ae0zE zseziAavA^tQl!HM^{-*T;W*|P(-Y$(h;QV}w7-KKvDNqReSNH*S%{g4jAgH;o=pBo za%1V;QX}a0w$r%!7lY2T?z0PPaB+kh*R!4lN>HCP9FYqpUX+RJ5N#&}#wVkds+mo~HFU zrK^SNcre`QeFIoTPdb?M64hUn&Orpo_hZs8T+AuBM$Zmf_vo}ojSr8ZLhDpXJOGs3 z2%!K4!XI+M|GcJv`{$xvAO(X!+WCq4UQ9($fe;QuB$&spfQf8ieYR741m>NbHG zWa4AYRZ%kQMI2po@;QAc#_>dGf$}m@fdKL3qbIGzMQ@rzKCAgOh=L~DI|tn|5|ZLf zNw@nOm__I5pnG<7-nO1~I!~;AuXk*n9SojY{lg~*=NA{fPW$PzUzb_@>KMH1^?io} zN}Pf!^an#yH`Sw^C4mR(*rJ1cBGuc3j`8GpOOt)8G$3t4^(nNLsCVSLloU}6h1R9? zT3)3dHwG<EQ0TjkgMLEG#!8F-K<7=8{K+AtsyA{CKm9C*DXGarf*n|1&nw#uwfacX4p zvvKU{C?c74TA8#dE=o6Xx+4sRdFS;$9V>1RzX>QcDc1za){q_%B?ln9_VqrVdm@Jae94OfK;GnPy)+L;R^ zJoht>-p67_qWYCsu)m@+{VmwA@^K3ur_M_P73Hvsnyd(LTM9Vzq(cfKgBP$QXQ(^0&6TyT~hg=vOO$OjFdelfsw6vonzoB=O!j% zjKP=WE}=BgcX z*B~~Ulu<#9wLlYGfMS?yu4C0&v!tj!C0WDcY`O*V%q-%c3OUE4eU+h|3#DNA2qBk^ za%^YfQXCR2SELA9Bx5kHVTNonT9t1p#hdb_sA!9FLE*xY9*QNiCYqJ(C*cCP#MQ;| zY6(`cq36n%Vgt!w@uG*#WQkO{&rlUbo{bj5ObQRQ;*2A$p<-*q-f~WnOj<~8Rgkw& zlA~9UsiC}Jboe5oFrFa_U>QOes4!+316PSN(Ha-aA6c8djazdUr7PS1Ce5{b*uHo= zI6Jg@7yTov-|rn+$DPA-t2^j)kB+~!6GNwi69lP490P~|D5hyqEz;C5 zoJvVg$ge0FofNu+12v4YOB_}1JNiiZ;2MCS>S zt6O1c|Ad*(=j3xC*NPJ9Anid#aFV>AbQb6P|=@F1Lkee-h~Dx*w4TE)*o7iO)&epJy-b+~5drv}uX zi73S#P%umiZri7YU>Zs^k!F)!3ZaNU@47Qt1ln63M#fp_1ras_@KPp7VmxOPGXvs} zDgu=i{z~~u;w#M7(Zy_bx{5Z*z(9mGRA+_-DBF)1J9GzT*Ic0O80hrZB>jjZ{{)Ks zakQ**_5V~}qSMUGCIMFK=^6JT;(}P9%5x=~-$v$1T0Um% zMBIk)&}1!wnE}+k}4GASmH>#+ywUOlrm&F8o1~>S5}pDmM(>D|dQC73mqxGfI|+i! zLJ^KxN~pb4&alq2F0o+LR2|CU!kqDC0zsh23e;DUejNH*;=?Mv`i04thGpPRe4XF| zf>VO!f;)3&w4gj zKw~8}f?7H}I3VvA1%(@_1lKGyb!Br+FkqKKA-$JZ5lfQnb@A@Ru_LAD5vPLZ-WJ@XfgxX#E`S*+e3{y-NHTb64fg5I=Ac{RMyOdf zV}v_SVV6xIToEVnf?PVAnOky6=ImLM?V1$NI7k(5%sKnCGcVBoU6JIxEVZ;lr+QXe z<9ABS!k*KdtR(3UTB=j=TCJkMxG|DXg&xpb6 zFozmHW(cWGZNQ<$L^v5ENqAxuYU_*4NbX-NWX=*7f{An{fKEDjpm9#QGaYHnq zL1}VgTZ!{fc@rg!ZmE*5l~UZ$^!hSv7E~#{txvNL(mopsHnN9c^1!IVAO;@`7F0y%$lhD}Mke|tOiOwCRlPLQ&os@|NCGtN87mO?cZ>oyh|R|SyQc^|1fCcdvX^B5otW6nqN z)sjS=U4`6f&k$|IaCBM?GU@CdA-(c--c8oMnyzXkUb8E^%HxC9n!+3LPP_$6$udf} z8Z)#{09fT(q3t;~GglsBv9S=I_T;E*Fh{P3X^I#l0*m2&FiMk~W6d2TTx#>OzOwDI3o`jGaf5H>4KZBT_GncbI-v2>ph zja__Q8OgdtT_-1nVMu6QgB^q!Q<=8V6&EXitPZDZf0EkMOC8xo{cqBlG z@OjpI?F*rHa*trxAWo3HX)`=-BKMYRZ>(?W%~9qA#H#tmRMzWM-$)vE0+mNbyMbKV z#jaEh-4<=g3NU`3Y|F|ePdxP`>VK~zP@w&6_`_a~Q1I&PRh5O1210(|bj?(hCr;H$ zk`kLozu6*(>jf0GkYdhACrV5K8IiYr7C1sH6qzB+_{M(N;?KD(kauOo9Oq$Bp6paCsgPv4_70H_Tq}FT7=p|Yj`YFN81jq6luK2bd9W1 zv_b6^HqDQr=|D?6WGpicOkcuM( z=oYji2+g~>GBVS?6Mv0z(@jL6y}O8ivJ(ln9%8*BwJBK<`D`Q_w8s}&-i&FiU|P;x zTQ-Q*QEo(JLJ)e`hl20b}JRdgmgNRTAi67ygKAWd(Krl46S$06kE1) zF5;X?OCXyY<2uKU)X|dfLq*3qQ()s_&>uu*d$@n@;r_XY`{y3+pL@7}?&1Eqhx_Lq?w@D1!^0qMQo)Hi%mKBEGT^n($4Yn0D zb4#o2_UmhdJgKz99`g;XvLfpxr=EZR`s@`*oMa6qQd&C;t``RACstTwC_h3aYhH@2rWo&Ak7XH>wu zowzDJ?ylx6GrHf!wBeG&omufn(9~gCC@wqNcUL`ue^tddPTta#yVJ?jt=`Mz>5=_z zCeAen$z@kvt9sUwL+3Wmo12!r)0s=!y_dO5#&?_9TdVr!Z`I7Iw6|AZ?_XU_CVwk? z1gx=DrpNZ0Xwqd7oh3^or@N%t%h@=KH4pI&?y2iI!nZ!q9s@#g2AdnPsM^rD3w+yi z?Q0nHnC&69a3hx+iZink-tx%>-AFVX4q1I1`;|O4J zUT6MgKuWbw9!Jw4lop;t>CtK7xq+$`R<+$XxuO*}0MVhNxr)<+B;{Ao^jNfEGW*4{ zXe*74E*09Bh%DFS`37Uq<_}J*VJ>yOI+T&&@m1B>cS;gyHKeq<$wW@iJ z2&$UX9s%5Yh5N2HT71=Bf)$RY&xV9<=MIqYHrXZf`@};3=;+`sXd&4l_XCgY>D}NW zBQkeM+n)hvmb2>HmDSe#kr&w4rS12Rr`;+xfLp`q5F}XR-4#xPio2&DS9-cML|I*~ za3%F1x&69puB3)fvyn9k-Ib>aWQN5yZAHnHy}B}YR&TS7q{UlvIzP$6opF}hS!P_U zbiRa;ku`G3H;eb(a@=+Ov#5)X_4;&_WE2$-uCirBcDRv*Y~ zGnd1SQS$(Q#d130_O-MYw5=9HP9j`d;~-4v$cjSTgY01ZN%KcTpPEG?r472><9nbq z#arG0zz8pUW8hWKPuLFxHpZ^-P@plLGN2DPdaJ-jGhT5r%Mh4iX3)f0`%w{R_ukJUkrz;_oLjY=)8fBeW##f?+kS8B^}9fmK>3~4P(2jb{`0ABZ0-f zMOSHLGd55hXz7W`V!IW zBe%*HNm+wRN0UtXJoL9)Z`hs!{dyQX#Zl_UEc0-OMl+uMcX9_iBi>$}U#jktkIyI_TiZ#&=+Z!o%-q0R%x<}#8Q(}o^w9PZqd+kVJNm2HQu zE8KGEyGZn{_VI4Jm$zyH5;TzpG!NYwo+6fLY~~Z)ms%B9t5kq0;CSbgk$8z`UCUKy z)>Eyv%|(1vE9GZS#Z3r?Q9R$6wXNGychjtC3U6uLwlVHFrFQ0CEVr$%a`sD2%v^tE z8`TfDQQj()wA%Wd;f!9aDipRX6xG}$PI_>BoLY*POlfAP`qS#gf2-cWbhPM%M~lw{ z9(S7beCB!VjLv>MGjC2y&sIRQVbxVyil2D}{Sbtn^RROMNGper^8`FLR6AZns<-Y; zEr3*_;=z??$-~p^*E{Z7fUvZ;N?`}S zb3Z)uQl+XqC>(zzwz=xyjd>WL?Uk`6@zC)%`YBL&_Uqj)IRg@aKw*&_T5<3{O{wS$yE|e_Rf))MRxkce<9(lBXFSzrl}JUG08k0p5;-cP|Ka!%MYDK>qVW z7i9V_m~BQ;s&qeN5Ds4TfpIJZzc~qRQqV7wcN+mti~gSZjq)SV)d>>XdEhvIBvTwU zrQT6qR=ySCj54L(!AvT{=uPO}aC?cFrLK*}evs;4Lx z*u};bnv}eI7363z$V2swpq}*cH0(6@{r=FEKcf%6S)5$=Rx}mX0-bBEey7vx9kvS^ z6L8xzDoIy{kQ+{Gj>X5Hajpm5!d#zYT86D>`!w_I?t!X=DfMzt9$0ceDSA5cPd*bs zfTY9DaUMzUBwm~HouHe#Hp{Ji@q2+A3_l-;$QG;h15i_ZDB-H3b-syVj9aM)74_~+ z{N5sBwNoZkA&p4y;!ZzWbSht1+U8h3<|ZjuJ1rlYvR%fN%AqP09q})RyyPiq_N8^yh=sg77apr9Ap#da7V7bb0r9yn9Oj7m(ez7xI=BP^_p6F z39Yo;{eAXJ9&L^`S(`wD7wV80`DFQ)PhRnv*sI}TR<->f+dnmF4)7uSYRGDMQ5x+; z8INn3RM3xw5zK-S-VQYig9Y7~h1-Pkfa|Tq-#xovU*N`J48V@@LcTit@=JW&bmdEG zr35-w&lYHMf4Ns4;irt!m6jSpc943Y*{=d`G`rh=5xE=t{%FX!lFAg-;x-{9$qz)4 zt4OcVL}}w+Lh_dDJ9s;H5U#t@_r@h99g}rQo3EYz+vO(iVCu-3sdM>q+d4DH)Oq=I z+dAIF)OqpErdxM6F5GNqH#}q`9$sO30~;#3sPVb`+^L+&L;RzYCkqDJPQ3y$HA?!nY*ADlhqZq*F{!WCXT)^au&fu-eBhq<`YRDB0=R N{{#MB>`upI0RXnmzlHz+ literal 0 HcmV?d00001 diff --git a/tests/resources/parser/test_data/us/meta.info b/tests/resources/parser/test_data/us/meta.info index e9befb3ec..f8b3aeabb 100644 --- a/tests/resources/parser/test_data/us/meta.info +++ b/tests/resources/parser/test_data/us/meta.info @@ -99,6 +99,10 @@ "url": "https://www.voanews.com/a/america-s-child-care-crisis-is-holding-back-moms-without-college-degrees-/7585813.html", "crawl_date": "2024-04-28 11:54:11.503866" }, + "VoiceOfAmerica_2026_08_19.html.gz": { + "url": "https://www.voanews.com/a/facebook-removes-kurdish-pages-linked-to-misinformation-on-belarus-migrant-crisis-/6407509.html", + "crawl_date": "2026-08-19 09:53:10.034542" + }, "WashingtonPost_2024_04_26.html.gz": { "url": "https://www.washingtonpost.com/weather/2024/04/26/storms-tornadoes-iowa-kansas-nebraska-oklahoma/", "crawl_date": "2024-04-26 17:38:15.805977" From abe5c35ff1f1ebaa66824d2b2d615764b4ffa493 Mon Sep 17 00:00:00 2001 From: Max Dallabetta Date: Fri, 28 Aug 2026 14:12:49 +0200 Subject: [PATCH 2/3] refactor(sitemap): replace `sort_predicate` with callable `sort_key` --- docs/how_to_add_a_publisher.md | 17 +++++++++++ src/fundus/publishers/at/__init__.py | 6 ++-- src/fundus/publishers/us/__init__.py | 4 +-- src/fundus/scraping/url.py | 45 ++++++++++++++++++---------- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/docs/how_to_add_a_publisher.md b/docs/how_to_add_a_publisher.md index 66a335c50..d914ca199 100644 --- a/docs/how_to_add_a_publisher.md +++ b/docs/how_to_add_a_publisher.md @@ -255,6 +255,23 @@ sitemap_filter=inverse(regex_filter("sitemap-content-")) ```` will exclude all sitemap URLs not containing the substring `sitemap-content-`. +#### Ordering sitemaps + +Some indices list their sitemaps in an order that is neither ascending nor descending by date, e.g. numbered sitemaps ordered as text, where `sitemap_10` follows directly after `sitemap_1`. +Use the `sort_key` parameter to reorder them. +It is handed to `list.sort`, so ordering is ascending, and `numeric_sort_key` builds one from a regular expression by reading its capture groups as integers: + +````python +Sitemap( + "https://www.voanews.com/sitemap.xml", + sitemap_filter=inverse(regex_filter(r"sitemap_[\d_]*\.xml\.gz")), + sort_key=numeric_sort_key(r"sitemap_\d+_(\d+)\.xml"), +) +```` + +Pass `reverse=True` to `numeric_sort_key` if the number grows with recency instead. +Unlike `Sitemap`'s `reverse`, which is applied afterwards and also flips the URLs within each sitemap, this only reorders the sitemaps. + ### Finishing the Publisher Specification 1. If your publisher requires to use custom request headers to work properly you can alter it by using the `request_header` parameter of `PublisherSpec`. diff --git a/src/fundus/publishers/at/__init__.py b/src/fundus/publishers/at/__init__.py index 87023b830..836012a5c 100644 --- a/src/fundus/publishers/at/__init__.py +++ b/src/fundus/publishers/at/__init__.py @@ -1,7 +1,5 @@ -import re - from fundus.publishers.base_objects import Publisher, PublisherGroup -from fundus.scraping.url import NewsMap, RSSFeed, Sitemap +from fundus.scraping.url import NewsMap, RSSFeed, Sitemap, numeric_sort_key from .derstandard import DerStandardParser from .die_presse import DiePresseParser @@ -59,7 +57,7 @@ class AT(metaclass=PublisherGroup): NewsMap("https://www.sn.at/news-artikel.sitemap.xml"), Sitemap( "https://www.sn.at/portal-artikel.sitemap.xml", - sort_predicate=re.compile(r"(article-)\d{4}_\d{2}-\d(.)"), + sort_key=numeric_sort_key(r"article-(\d{4})_(\d{2})-(\d)", reverse=True), ), ], ) diff --git a/src/fundus/publishers/us/__init__.py b/src/fundus/publishers/us/__init__.py index d7ed69780..0dcc27c7d 100644 --- a/src/fundus/publishers/us/__init__.py +++ b/src/fundus/publishers/us/__init__.py @@ -1,6 +1,6 @@ from fundus.publishers.base_objects import Publisher, PublisherGroup from fundus.scraping.filter import inverse, lor, regex_filter -from fundus.scraping.url import NewsMap, RSSFeed, Sitemap +from fundus.scraping.url import NewsMap, RSSFeed, Sitemap, numeric_sort_key from .ap_news import APNewsParser from .business_insider import BusinessInsiderParser @@ -261,7 +261,7 @@ class US(metaclass=PublisherGroup): Sitemap( "https://www.voanews.com/sitemap.xml", sitemap_filter=inverse(regex_filter(r"sitemap_[\d_]*\.xml\.gz")), - reverse=True, + sort_key=numeric_sort_key(r"sitemap_\d+_(\d+)\.xml"), ), ], ) diff --git a/src/fundus/scraping/url.py b/src/fundus/scraping/url.py index 33e4ee01d..1b60bb54d 100644 --- a/src/fundus/scraping/url.py +++ b/src/fundus/scraping/url.py @@ -2,10 +2,12 @@ import gzip import itertools import lzma +import re from abc import ABC, abstractmethod from dataclasses import dataclass, field -from functools import cached_property, partial +from functools import cached_property from typing import ( + Any, Callable, ClassVar, Dict, @@ -13,8 +15,8 @@ Iterator, List, Optional, - Pattern, Set, + Tuple, ) from urllib.parse import unquote, urlparse @@ -177,12 +179,34 @@ def fetch(self, session: InterruptableSession, headers: Dict[str, str]) -> Itera yield clean_url(url) +def numeric_sort_key(pattern: str, reverse: bool = False) -> Callable[[str], Tuple[int, ...]]: + """Build a ordering sitemaps by the integer capture groups of . + + Groups are compared numerically rather than as text, so unpadded indices order as + 1, 2, ..., 10 instead of 1, 10, 2. Pass reverse=True to negate them, putting the + highest value first - use it when the number grows with recency (e.g. a date), + and leave it off when it grows with age (e.g. a sitemap chunk counted from newest). + + Raises ValueError for a URL the pattern doesn't match; is applied + first, so the key only ever sees the sitemaps that were kept. + """ + compiled = re.compile(pattern) + sign = -1 if reverse else 1 + + def key(url: str) -> Tuple[int, ...]: + if match := compiled.search(url): + return tuple(sign * int(group) for group in match.groups()) + raise ValueError(f" pattern {pattern!r} does not match sitemap URL {url!r}") + + return key + + @dataclass class Sitemap(URLSource): recursive: bool = True reverse: bool = False sitemap_filter: URLFilter = lambda url: not bool(url) - sort_predicate: Optional[Pattern[str]] = None + sort_key: Optional[Callable[[str], Any]] = None _decompressor: ClassVar[_ArchiveDecompressor] = _ArchiveDecompressor() _sitemap_selector: ClassVar[XPath] = XPath("//*[local-name()='sitemap']/*[local-name()='loc']") @@ -227,20 +251,11 @@ def yield_recursive(sitemap_url: str) -> Iterator[str]: elif self.recursive: sitemap_locs = [node.text for node in self._sitemap_selector(tree)] - if self.sort_predicate is not None: - - def _extract_predicate(text: str, pattern: Pattern[str]) -> str: - if match := pattern.search(text): - return match.group() - raise NotImplementedError(" must match in all sitemap URLs") + filtered_locs = list(filter(inverse(self.sitemap_filter), sitemap_locs)) - sitemap_locs = sorted( - sitemap_locs, - key=partial(_extract_predicate, pattern=self.sort_predicate), - reverse=True, - ) + if self.sort_key is not None: + filtered_locs.sort(key=self.sort_key) - filtered_locs = list(filter(inverse(self.sitemap_filter), sitemap_locs)) for loc in reversed(filtered_locs) if self.reverse else filtered_locs: yield from yield_recursive(loc) From 064a52c5616ca7faafa69050f84b2f3ce085cebe Mon Sep 17 00:00:00 2001 From: Adrian Breiding Date: Sat, 29 Aug 2026 11:09:13 +0200 Subject: [PATCH 3/3] fix(publisher): refine VOA V1_1 selectors for improved content extraction --- src/fundus/publishers/us/voice_of_america.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fundus/publishers/us/voice_of_america.py b/src/fundus/publishers/us/voice_of_america.py index 148119665..f0de97af8 100644 --- a/src/fundus/publishers/us/voice_of_america.py +++ b/src/fundus/publishers/us/voice_of_america.py @@ -55,5 +55,5 @@ def images(self) -> List[Image]: ) class V1_1(V1): - _paragraph_selector = XPath("//div[@id='article-content']/div/p[not(strong)]") - _subheadline_selector = XPath("//div[@id='article-content']/div/p[strong]") + _paragraph_selector = XPath("//div[@id='article-content']/div[@class='wsw']//p[not(strong) and not(@class)]") + _subheadline_selector = XPath("//div[@id='article-content']/div[@class='wsw']//p[strong]")