misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-07 16:15:32 +02:00
Code Releases Activity

fix nytimes-3

Browse source
This commit is contained in:
Jan Lukas Gernert 2023-03-31 10:38:04 +02:00
parent c42ffa57a2
commit c46d93058f
6 changed files with 933 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

267
resources/tests/readability/nytimes-3/expected.html Normal file
View file

@ -0,0 +1,267 @@
<article><DIV id="readability-page-1"><article id="story">
<header>
<p>
New York’s aging below-street infrastructure is tough to maintain, and the corrosive rock salt and “freeze-thaw” cycles of winter make it even worse.
</p>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemscope="itemscope" itemprop="associatedMedia" itemid="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<p><span>Image</span><img alt="" src="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale" srcset="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-articleLarge.jpg?quality=90&amp;auto=webp 600w,https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-jumbo.jpg?quality=90&amp;auto=webp 1024w,https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-superJumbo.jpg?quality=90&amp;auto=webp 2048w" sizes="((min-width: 600px) and (max-width: 1004px)) 84vw, (min-width: 1005px) 80vw, 100vw" itemprop="url" itemid="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale">
</p>
<figcaption itemprop="caption description">
<span>A Con Edison worker repairing underground cables this month in Flushing, Queens. The likely source of the problem was water and rock salt that had seeped underground.</span><span itemprop="copyrightHolder"><span>Credit</span><span><span>Credit</span><span>Chang W. Lee/The New York Times</span></span></span>
</figcaption>
</figure>
</div>
<div>
<div>
<p><a href="https://www.nytimes.com/by/corey-kilgannon" target="_blank"><img alt="Corey Kilgannon" title="Corey Kilgannon" src="https://static01.nyt.com/images/2018/02/20/multimedia/author-corey-kilgannon/author-corey-kilgannon-thumbLarge.jpg"></a>
</p>
</div>
<ul>
<li>
<time datetime="2019-02-21">Feb. 21, 2019</time>
</li>
<li>
</li>
</ul>
</div>
</header>
<section name="articleBody" itemprop="articleBody">
<div>
<p>
<em>[What you need to know to start the day:</em> <a href="https://www.nytimes.com/newsletters/newyorktoday?module=inline" title="" target="_blank"><em>Get New York Today in your inbox</em></a><em>.]</em>
</p>
<p>
A series of recent <a href="https://nypost.com/2019/02/17/manhole-fires-force-theater-evacuations-in-midtown/" title="" rel="noopener noreferrer" target="_blank">manhole fires</a> in the heart of Manhattan forced the evacuation of several theaters and was a stark reminder that the subway is not the only creaky infrastructure beneath the streets of New York City.
</p>
<p>
Underground lies a chaotic assemblage of utilities that, much like the subway, are lifelines for the city: a sprawling tangle of water mains, power cables, gas and steam lines, telecom wires and sewers.
</p>
<p>
The city has one of the oldest and largest networks of subterranean infrastructure in the world, with some portions dating more than a century and prone to leaks and cracks.
</p>
<p>
And winter — from the corrosive rock salt used on streets and sidewalks to “freeze-thaw” cycles that weaken pipes — makes infrastructure problems even worse.
</p>
</div>
<div>
<p>
In the late 1800s, many of the city’s overhead utilities were buried to lessen the exposure to winter weather. “People think it’s all protected and safe, but it’s really not,” said Patrick McHugh, vice president of electrical engineering and planning for Con Edison, which maintains about 90,000 miles of underground cable in the city.
</p>
<p>
“You have water, sewage, electricity and gas down there, and people don’t appreciate the effort that goes into keeping all that working,” he added.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<p><span>Image</span><img alt="" src="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale" srcset="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-articleLarge.jpg?quality=90&amp;auto=webp 600w,https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-jumbo.jpg?quality=90&amp;auto=webp 1024w,https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-superJumbo.jpg?quality=90&amp;auto=webp 2000w" sizes="((min-width: 600px) and (max-width: 1004px)) 84vw, (min-width: 1005px) 80vw, 100vw" itemprop="url" itemid="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale">
</p>
<figcaption itemprop="caption description">
<span>In the late 1800s, overhead utilities were buried to lessen the exposure to winter weather.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Kirsten Luce for The New York Times</span></span>
</figcaption>
</figure>
</div>
<div>
<h2 id="link-32de4094">
Rock salt on icy streets can cause mayhem below them
</h2>
<p>
When rock salt melts ice, and the water seeps down manholes and into electrical units, it can set off fires and explosions strong enough to pop a 300-pound manhole cover five stories into the air.
</p>
<p>
For days after a storm, Con Edison officials say, they often deal with scores of electrical fires caused by the rock salt eating away at electrical cable insulation. The wet salt can create sparking that burns the insulation, producing both fire and gases that can combust and pop the manhole lids.
</p>
<p>
To alleviate the threat, the officials said, the utility switched most of its manhole covers to vented ones that allow gases to escape, “so they cannot form a combustible amount,” Mr. McHugh said.
</p>
</div>
<div>
<p>
“It also lets smoke escape, which can tip off the public to notify the authorities,” he added.
</p>
<p>
Winter can also bring an increase in gas-line breakages. Con Edison, which maintains 4,300 miles of gas mains in and around New York City, records about 500 leaks — most of them nonemergencies — in a typical month, but many more in winter.
</p>
<p>
Even this past January, which was unseasonably mild, there were 750 leaks, Con Edison officials said.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2019/02/15/nyregion/00winterutilitiesOAK11/merlin_94083158_9e622a52-ec2f-4fbd-845c-5d530e94bc82-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<div>
<p><span>Image</span></p>
</div>
<figcaption itemprop="caption description">
<span>There are typically between 400 and 600 water main breaks each year in New York City, an official said.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Michael Appleton for The New York Times</span></span>
</figcaption>
<img src="https://static01.nyt.com/images/2019/02/15/nyregion/00winterutilitiesOAK11/merlin_94083158_9e622a52-ec2f-4fbd-845c-5d530e94bc82-articleLarge.jpg?quality=90&amp;auto=webp"></figure>
</div>
<div>
<h2 id="link-470a509">
‘Freeze-thaws’ linked to climate change can imperil utilities
</h2>
<p>
The extreme temperature swings that many researchers link to climate change are adding to the challenges of winter.
</p>
<p>
Officials monitor weather forecasts closely for freeze-thaw cycles, when they put extra repair crews on call.
</p>
<p>
During a polar vortex in late January, for instance, single-digit temperatures in the city quickly ballooned into the 50s. The thaw, much welcomed by many New Yorkers, worried Tasos Georgelis, deputy commissioner for water and sewer operations at the Department of Environmental Protection, which operates the city’s <a href="https://www.nytimes.com/interactive/2016/03/24/nyregion/how-nyc-gets-its-water-new-york-101.html?action=click&amp;contentCollection=New%20York&amp;region=Footer&amp;module=inline&amp;version=WhatsNext&amp;contentID=WhatsNext&amp;moduleDetail=undefined&amp;pgtype=Multimedia" title="" target="_blank">water system</a>.
</p>
<p>
“When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” Mr. Georgelis said.
</p>
<p>
Along the city’s roughly 6,500 miles of water mains, there are typically between 400 and 600 breaks a year, he added. The majority occur in winter, when the cold can make older cast-iron mains brittle.
</p>
</div>
<div>
<p>
Environmental Protection officials said the department repaired 75 water-main breaks in January, including one in Lower Manhattan that disrupted rush-hour subway service and another on the West Side that snarled traffic and left nearby buildings without water for hours.
</p>
<p>
The city’s 7,500 miles of sewer lines are less affected by cold weather because they are generally buried deeper than other utilities, below the frost line, agency officials said.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK13/merlin_77720485_1f5be529-e659-49c3-a0bc-a3f312835d14-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<div>
<p><span>Image</span></p>
</div>
<figcaption itemprop="caption description">
<span>In 1978, a water main break caused severe flooding in Bushwick, Brooklyn.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Fred R. Conrad/The New York Times</span></span>
</figcaption>
<img src="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK13/merlin_77720485_1f5be529-e659-49c3-a0bc-a3f312835d14-articleLarge.jpg?quality=90&amp;auto=webp"></figure>
</div>
<div>
<h2 id="link-4c678ca">
How to replace century-old utilities: slowly, and carefully
</h2>
<p>
Upgrading the city’s below-street utilities is a slow, painstaking process, “because you have such a fixed-in-place system,” said Rae Zimmerman, a research professor of planning and public administration at New York University.
</p>
<p>
But there is progress. Con Edison officials said they had begun replacing the city’s nearly 1,600 miles of natural gas lines — which were made of either cast iron or unprotected steel — with plastic piping. The plastic is less susceptible to corrosion, cracks and leaks, said the officials, who added that they were swapping about 100 miles of line each year.
</p>
<p>
The city is also replacing older, leak-prone water and sewer mains.
</p>
<p>
Some pipes that are more than a century old hold up because they were built with a thicker grade of cast iron, according to Environmental Protection Department officials. For less healthy ones, the agency has invested more than $1 billion in the past five years — with an additional $1.4 billion budgeted over the next five years — for upgrades and replacements. New pipes will be made of a more durable, graphite-rich cast iron known as ductile iron.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2017/12/07/nyregion/00winter4/00winter4-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<div>
<p><span>Image</span></p>
</div>
<figcaption itemprop="caption description">
<span>Matt Cruz snowboarded through Manhattan’s Lower East Side after a snowstorm in 2016 left the streets coated in slush and rock salt.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Hiroko Masuike/The New York Times</span></span>
</figcaption>
<img src="https://static01.nyt.com/images/2017/12/07/nyregion/00winter4/00winter4-articleLarge.jpg?quality=90&amp;auto=webp"></figure>
</div>
<div>
<h2 id="link-132c3a6b">
Construction during the winter isn’t always a good idea
</h2>
<p>
Of course, winter also poses problems aboveground. Most nonemergency repair and construction work involving concrete is halted because concrete and some types of dirt, used to fill in trenches, freeze in colder temperatures, said Ian Michaels, a spokesman for the city’s Department of Design and Construction.
</p>
<p>
Digging by hand is also a challenge in frozen ground, so many excavations that are close to pipes and other utilities are put off, Mr. Michaels said.
</p>
</div>
<div>
<p>
And asphalt is harder to obtain because it must be kept and transported at high temperatures, he added.
</p>
<p>
In the extreme cold, city officials will not risk shutting down water mains for construction because spillage into the street could freeze, Mr. Michaels said. He added that stopping the water flow could freeze the private water-service connections that branch off the mains, he said.
</p>
<p>
Even the basic task of locating utilities under the street can be complicated because infrastructure has been added piecemeal over the decades.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2019/02/15/nyregion/00winterutilitiesOAK12/merlin_77751796_7f1b7a7a-23e9-49b2-b12e-1edfed680911-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<div>
<p><span>Image</span></p>
</div>
<figcaption itemprop="caption description">
<span>A water main break in Manhattan in 2014. “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” said Tasos Georgelis of the city's Department of Environmental Protection.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Ãngel Franco/The New York Times</span></span>
</figcaption>
<img src="https://static01.nyt.com/images/2019/02/15/nyregion/00winterutilitiesOAK12/merlin_77751796_7f1b7a7a-23e9-49b2-b12e-1edfed680911-articleLarge.jpg?quality=90&amp;auto=webp"></figure>
</div>
<div>
<h2 id="link-7ba21106">
Almost 1.8 million potholes filled since de Blasio took office
</h2>
<p>
Street surfaces are affected by winter weather, too: Last year, the city filled 255,904 potholes.
</p>
<p>
And should anyone forget that filling potholes, like snow removal, is a sacred staple of constituent services, transportation officials have compiled the number of potholes the city has filled — more than 1,786,300 — since Mayor Bill de Blasio took office in 2014.
</p>
<p>
Potholes form when water and salt seep into cracks, freeze and expand, creating a larger crevice, said Joe Carbone, who works for the Transportation Department, where he is known as the pothole chief.
</p>
<p>
Simply put, more freeze-thaw cycles <a href="https://www.nytimes.com/2011/02/07/nyregion/07pothole.html?module=inline" title="" target="_blank">result in more potholes</a>, he said. Currently, the department has 25 crews repairing potholes. During peak pothole-repair season in early March, that number can expand to more than 60.
</p>
</div>
<div>
<p>
Still, the department is continually resurfacing the city’s more than 6,000 miles of streets and 19,000 lane miles. Each year, agency officials said, it uses more than one million tons of asphalt to repave more than 1,300 lane-miles of street.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2019/02/12/nyregion/00winterutilities2/merlin_150504129_e893b874-01eb-4d8f-8158-18512153e414-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<div>
<p><span>Image</span></p>
</div>
<figcaption itemprop="caption description">
<span>Workers learning how to fix water main breaks at a training center in Queens.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Chang W. Lee/The New York Times</span></span>
</figcaption>
<img src="https://static01.nyt.com/images/2019/02/12/nyregion/00winterutilities2/merlin_150504129_e893b874-01eb-4d8f-8158-18512153e414-articleLarge.jpg?quality=90&amp;auto=webp"></figure>
</div>
<div>
<h2 id="link-13d0a396">
So, where do people learn how to fix some of these issues?
</h2>
<p>
Of the 400 city laborers who work on water mains, many learn the finer points of leak repair at a training center in Queens, where underground pipes are made to spring leaks for repair drills.
</p>
<p>
Workers from the Department of Environmental Protection recently gathered around a muddy hole as a co-worker, Nehemiah Dejesus, scrambled to apply a stainless-steel repair clamp around a cracked segment that was spewing water.
</p>
<p>
“Don’t get nervous,” instructed Milton Velez, the agency’s district supervisor for Queens.
</p>
<p>
“I’m not,” Mr. Dejesus said as he secured the clamp and stopped the leak. “It’s ‘Showtime at the Apollo.’”
</p>
</div>
</section>
<div>
<div>
<p>
Corey Kilgannon is a Metro reporter covering news and human interest stories. His writes the <a href="https://www.nytimes.com/column/character-study" target="_blank">Character Study</a> column in the Sunday Metropolitan section. He was also part of the team that won the 2009 Pulitzer Prize for Breaking News. <span><a href="https://twitter.com/coreykilgannon" rel="noopener noreferrer" target="_blank"><span>@</span>coreykilgannon</a> <span>•</span> <a href="https://www.facebook.com/corey.kilgannon.9" rel="noopener noreferrer" target="_blank">Facebook</a></span>
</p>
</div>
<p>
A version of this article appears in print on , on Page A22 of the New York edition with the headline: Under the City’s Streets, A Battle Against Winter<span>. <a href="http://www.nytreprints.com/" target="_blank">Order Reprints</a> | <a href="http://www.nytimes.com/pages/todayspaper/index.html" target="_blank">Today’s Paper</a> | <a href="https://www.nytimes.com/subscriptions/Multiproduct/lp8HYKU.html?campaignId=48JQY" target="_blank">Subscribe</a></span>
</p>
</div>
</article></DIV></article>

224
resources/tests/readability/nytimes-4/expected.html Normal file
View file

@ -0,0 +1,224 @@
<article><DIV id="readability-page-1"><article id="story">
<header>
<p>
Tax cuts, spending increases and higher interest rates could make it harder to respond to future recessions and deal with other needs.
</p>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemscope="itemscope" itemprop="associatedMedia" itemid="https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<p><span>Image</span><img alt="" src="https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale" srcset="https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-articleLarge.jpg?quality=90&amp;auto=webp 600w,https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-jumbo.jpg?quality=90&amp;auto=webp 1024w,https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-superJumbo.jpg?quality=90&amp;auto=webp 2048w" sizes="((min-width: 600px) and (max-width: 1004px)) 84vw, (min-width: 1005px) 60vw, 100vw" itemprop="url" itemid="https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale">
</p>
<figcaption itemprop="caption description">
<span>Interest payments on the federal debt could surpass the Defense Department budget in 2023.</span><span itemprop="copyrightHolder"><span>Credit</span><span><span>Credit</span><span>Jeon Heon-Kyun/EPA, via Shutterstock</span></span></span>
</figcaption>
</figure>
</div>
<div>
<ul>
<li>
<time datetime="2018-09-25">Sept. 25, 2018</time>
</li>
<li>
</li>
</ul>
</div>
</header>
<section name="articleBody" itemprop="articleBody">
<div>
<p>
The federal government could soon pay more in interest on its debt than it spends on the military, Medicaid or children’s programs.
</p>
<p>
The run-up in borrowing costs is a one-two punch brought on by the need to finance a fast-growing budget deficit, worsened by tax cuts and steadily rising interest rates that will make the debt more expensive.
</p>
<p>
With less money coming in and more going toward interest, political leaders will find it harder to address pressing needs like fixing crumbling roads and bridges or to make emergency moves like pulling the economy out of future recessions.
</p>
<p>
Within a decade, more than $900 billion in interest payments will be due annually, easily outpacing spending on myriad other programs. Already the fastest-growing major government expense, the cost of interest is on track to hit $390 billion next year, nearly 50 percent more than in 2017, according to the Congressional Budget Office.
</p>
</div>
<div>
<p>
“It’s very much something to worry about,” said C. Eugene Steuerle, a fellow at the Urban Institute and a co-founder of the Urban-Brookings Tax Policy Center in Washington. “Everything else is getting squeezed.”
</p>
<p>
Gradually rising interest rates would have made borrowing more expensive even without additional debt. But the tax cuts passed late last year have created a deeper hole, with the deficit increasing faster than expected. A budget bill <a href="https://www.nytimes.com/2018/02/08/us/politics/congress-budget-deal-vote.html?module=inline" title="" target="_blank">approved in February that raised</a> spending by $300 billion over two years will add to the financial pressure.
</p>
<p>
The deficit is expected to total nearly $1 trillion next year — the first time it has been that big since 2012, when the economy was still struggling to recover from the financial crisis and interest rates were near zero.
</p>
</div>
<div>
<p>
Deficit hawks have gone silent, even proposing changes that would exacerbate the deficit. House Republicans introduced legislation this month that would make the tax cuts permanent.
</p>
<p>
“The issue has just disappeared,” said Senator Mark Warner, a Virginia Democrat. “There’s collective amnesia.”
</p>
</div>
<div>
<p>
The combination, say economists, marks a journey into mostly uncharted financial territory.
</p>
<p>
In the past, government borrowing expanded during recessions and waned in recoveries. That countercyclical policy has been a part of the standard Keynesian toolbox to combat downturns since the Great Depression.
</p>
<p>
The deficit is soaring now as the economy booms, meaning the stimulus is pro-cyclical. The risk is that the government would have less room to maneuver if the economy slows.
</p>
</div>
<div>
<p>
Aside from wartime or a deep downturn like the 1930s or 2008-9, “this sort of aggressive fiscal stimulus is unprecedented in U.S. history,” said Jeffrey Frankel, an economist at Harvard.
</p>
<p>
Pouring gasoline on an already hot economy has resulted in faster growth — the economy expanded at an annualized rate of 4.2 percent in the second quarter. But Mr. Frankel warns that when the economy weakens, the government will find it more difficult to cut taxes or increase spending.
</p>
<p>
Lawmakers might, in fact, feel compelled to cut spending as tax revenue falls, further depressing the economy. “There will eventually be another recession, and this increases the chances we will have to slam on the brakes when the car is already going too slowly,” Mr. Frankel said.
</p>
<h2 id="link-cb5ee12">
Interest costs make it harder for the government to do other things
</h2>
</div>
<div>
<p>
Finding the money to pay investors who hold government debt will crimp other parts of the budget. In a decade, interest on the debt will eat up 13 percent of government spending, up from 6.6 percent in 2017.
</p>
<p>
“By 2020, we will spend more on interest than we do on kids, including education, food stamps and aid to families,” said Marc Goldwein, senior policy director at the Committee for a Responsible Federal Budget, a research and advocacy organization.
</p>
</div>
<div>
<p>
Interest costs already dwarf spending on many popular programs. For example, grants to students from low-income families for college total roughly $30 billion — about one-tenth of what the government will pay in interest this year. Interest payments will overtake Medicaid in 2020 and the Department of Defense budget in 2023.
</p>
<p>
What’s more, the heavy burden of interest payments could make it harder for the government to repair aging infrastructure or take on other big new projects.
</p>
<p>
Mr. Trump has called for spending $1 trillion on infrastructure, but Congress has not taken up that idea.
</p>
</div>
<div>
<p>
More about the federal debt and the economy
</p>
</div>
<div>
<h2 id="link-1a1f1f7a">
The U.S. hasn’t faced this issue for years
</h2>
<p>
Until recently, ultralow interest rates, set by the Federal Reserve to support the economy, allowed lawmakers to borrow without fretting too much about the cost of that debt.
</p>
<p>
But as the economy has strengthened, the Fed has gradually raised rates, starting in December 2015. The central bank is expected to push rates up again on Wednesday, and more increases are in store.
</p>
<p>
“When rates went down to record lows, it allowed the government to take on more debt without paying more interest,” Mr. Goldwein said. “That party is ending.”
</p>
</div>
<div>
<p>
Since the beginning of the year, the yield on the 10-year Treasury note has risen by more than half a percentage point, to 3.1 percent. The Congressional Budget Office estimates that the yield will climb to 4.2 percent in 2021. Given that the total public debt of the United States stands at nearly $16 trillion, even a small uptick in rates can cost the government billions.
</p>
</div>
<div>
<p>
There’s no guarantee that these forecasts will prove accurate. If the economy weakens, rates might fall or rise only slightly, reducing interest payments. But rates could also overshoot the budget office forecast.
</p>
<p>
Some members of Congress want to set the stage for even more red ink. Republicans in the House want to make last year’s tax cuts permanent, instead of letting some of them expire at the end of 2025. That would reduce federal revenue by an additional $631 billion over 10 years, according to the <a href="https://www.taxpolicycenter.org/publications/analysis-protecting-family-and-small-business-tax-cuts-act-2018" title="" rel="noopener noreferrer" target="_blank">Tax Policy Center</a>.
</p>
<h2 id="link-7b39a7bd">
No, the United States isn’t at risk of becoming the next Greece
</h2>
</div>
<div>
<p>
Deficit hawks have warned for years that a day of reckoning is coming, exposing the United States to the kind of economic crisis that overtook profligate borrowers in the past like Greece or Argentina.
</p>
<p>
But most experts say that isn’t likely because the dollar is the world’s reserve currency. As a result, the United States still has plenty of borrowing capacity left because the Fed can print money with fewer consequences than other central banks.
</p>
<p>
And interest rates plunged over the last decade, even as the government turned to the market for trillions each year after the recession. That’s because Treasury bonds are still the favored port of international investors in any economic storm.
</p>
<p>
“We exported a financial crisis a decade ago, and the world responded by sending us money,” said William G. Gale, a senior fellow at the Brookings Institution.
</p>
<p>
But that privileged position has allowed politicians in both parties to avoid politically painful steps like cutting spending or raising taxes.
</p>
</div>
<div>
<p>
That doesn’t mean rapidly rising interest costs and a bigger deficit won’t eventually catch up with us.
</p>
<p>
Charles Schultze, chairman of the Council of Economic Advisers in the Carter administration, once summed up the danger of deficits with a metaphor. “It’s not so much a question of the wolf at the door, but termites in the woodwork.”
</p>
<h2 id="link-12df6cc7">
But Washington doesn’t want to hear about the potential problems
</h2>
<p>
Rather than simply splitting along party lines, lawmakers’ attitudes toward the deficit also depend on which party is in power. Republicans pilloried the Obama administration for proposing a large stimulus in the depths of the recession in 2009 and complained about the deficit for years.
</p>
<p>
In 2013, Senator Mitch McConnell of Kentucky called the debt and deficit “the transcendent issue of our era.” By 2017, as Senate majority leader, he quickly shepherded the tax cut through Congress.
</p>
<p>
Senator James Lankford, an Oklahoma Republican who warned of the deficit’s dangers in the past, nevertheless played down that threat on the Senate floor as the tax billed neared passage.
</p>
<p>
“I understand it’s a risk, but I think it’s an appropriate risk to be able to say let’s allow Americans to keep more of their own money to invest in this economy,” he said.
</p>
<p>
He also claimed the tax cuts would pay for themselves even as the Congressional Budget Office estimated that they would add $250 billion to the deficit on average from 2019 to 2024.
</p>
</div>
<div>
<p>
In an interview, Mr. Lankford insisted that the jury was still out on whether the tax cuts would generate additional revenue, citing the strong economic growth recently.
</p>
<p>
While the Republican about-face has been much more striking, Democrats have adjusted their position, too.
</p>
<p>
Mr. Warner, the Virginia Democrat, called last year’s tax bill “the worst piece of legislation we have passed since I arrived in the Senate.” In 2009, however, when Congress passed an $800 billion stimulus bill backed by the Obama administration, he called it “a responsible mix of tax cuts and investments that will create jobs.”
</p>
<p>
The difference, Mr. Warner said, was that the economy was near the precipice then.
</p>
<p>
“There was virtual unanimity among economists that we needed a stimulus,” he said. “But a $2 trillion tax cut at the end of a business cycle with borrowed money won’t end well.”
</p>
</div>
</section>
<div>
<div>
<p>
Nelson D. Schwartz has covered economics since 2012. Previously, he wrote about Wall Street and banking, and also served as European economic correspondent in Paris. He joined The Times in 2007 as a feature writer for the Sunday Business section. <span><a href="https://twitter.com/NelsonSchwartz" rel="noopener noreferrer" target="_blank"><span>@</span>NelsonSchwartz</a></span>
</p>
</div>
<p>
A version of this article appears in print on , on Page A1 of the New York edition with the headline: What May Soon Exceed Cost of U.S. Military? Interest on U.S. Debt <span>. <a href="http://www.nytreprints.com/" target="_blank">Order Reprints</a> | <a href="http://www.nytimes.com/pages/todayspaper/index.html" target="_blank">Today’s Paper</a> | <a href="https://www.nytimes.com/subscriptions/Multiproduct/lp8HYKU.html?campaignId=48JQY" target="_blank">Subscribe</a></span>
</p>
</div>
</article></DIV></article>

406
resources/tests/readability/nytimes-5/expected.html Normal file
View file

@ -0,0 +1,406 @@
<article><div aria-hidden="false" id="readability-page-1">
<main id="site-content">
<section id="collection-espanol">
<div>
<div id="collection-highlights-container">
<h2>
Highlights
</h2>
<ol>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/25/espanol/juegos-olimpicos-pekin-2022.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/13/world/00xi-olympics-30-esp-1/00xi-olympics-30-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/13/world/00xi-olympics-30-esp-1/00xi-olympics-30-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/13/world/00xi-olympics-30-esp-1/00xi-olympics-30-mediumThreeByTwo440.jpg 440w" alt="Xi Jinping, el líder de China, arriesgó su prestigio personal cuando su país se postuló para organizar los Juegos de Invierno 2022; hasta ahora el país ha cumplido sus promesas."></a>
<figcaption>
<span><span>Credit</span>Kevin Frayer/Getty Images</span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/25/espanol/opinion/beatles-get-back.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/19/opinion/19max/19max-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/19/opinion/19max/19max-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/19/opinion/19max/19max-mediumThreeByTwo440.jpg 440w" alt=""></a>
<figcaption>
<span><span>Credit</span>Ellen Surrey</span>
</figcaption>
</figure>
<div>
<h3>
Ensayo invitado
</h3>
<h2>
<a data-rref="" href="http://fakehost/es/2022/01/25/espanol/opinion/beatles-get-back.html" target="_blank">El día que renuncié a los Beatles</a>
</h2>
<p>
Estaba obsesionado con el cuarteto de Liverpool: celebraba sus cumpleaños, leía todo sobre la banda y memoricé todas sus canciones. Cuando me desintoxiqué de ellos descubrí un nuevo mundo musical.
</p>
<p>
<span>Por <span itemprop="name">Josh Max</span></span>
</p>
</div>
</article>
</li>
<li>
<ol>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/25/espanol/microsoft-meta-activision-blizzard.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/18/business/18OnTech-Microsoft-NL/18OnTech-Microsoft-NL-videoLarge.png" srcset="https://static01.nyt.com/images/2022/01/18/business/18OnTech-Microsoft-NL/18OnTech-Microsoft-NL-videoLarge.png 768w, https://static01.nyt.com/images/2022/01/18/business/18OnTech-Microsoft-NL/18OnTech-Microsoft-NL-mediumThreeByTwo440.png 440w" alt=""></a>
<figcaption>
<span><span>Credit</span>Erik Carter</span>
</figcaption>
</figure>
<div>
<h3>
On Tech
</h3>
<h2>
<a data-rref="" href="http://fakehost/es/2022/01/25/espanol/microsoft-meta-activision-blizzard.html" target="_blank">¿Qué tienen que ver los videojuegos con el metaverso?</a>
</h2>
<p>
Las compañías tecnológicas creen que los videojuegos son el camino para avanzar más rápido hacia un internet inmersivo. La adquisición de Activision Blizzard por Microsoft es una muestra de esta tendencia.
</p>
<p>
<span>Por <span itemprop="name">Shira Ovide</span></span>
</p>
</div>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/25/espanol/bebidas-funcionales-beneficios.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/20/fashion/20SKIN-BEAUTYDRINKS1-ESP-1/20SKIN-BEAUTYDRINKS1-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/20/fashion/20SKIN-BEAUTYDRINKS1-ESP-1/20SKIN-BEAUTYDRINKS1-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/20/fashion/20SKIN-BEAUTYDRINKS1-ESP-1/20SKIN-BEAUTYDRINKS1-mediumThreeByTwo440.jpg 440w" alt=""></a>
<figcaption>
<span><span>Credit</span>Ted + Chelsea Cavanaugh para The New York Times</span>
</figcaption>
</figure>
<div>
<h3>
Skin Deep
</h3>
<h2>
<a data-rref="" href="http://fakehost/es/2022/01/25/espanol/bebidas-funcionales-beneficios.html" target="_blank">Bebidas con beneficios: ¿de verdad funcionan?</a>
</h2>
<p>
Hay un nuevo mercado de bebidas que prometen beneficios como la salud intestinal, una mente relajada y piel más brillante. El problema es que ninguno de esos efectos ha sido respaldado científicamente.
</p>
<p>
<span>Por <span itemprop="name">Rachel Strugatz</span></span>
</p>
</div>
</article>
</li>
</ol>
</li>
</ol>
</div>
<section>
<header>
<h2>
<a href="http://fakehost/es/section/opinion" target="_blank">Opinión<svg viewbox="0 0 5.6 9.5" width="5.6" height="9.5" xml:space="preserve" xmlns="http://www.w3.org/1999/xlink" x="0px" y="0px">
<path d="M5.6,4.2L1.8,0L0,1.3l3.1,3.5L0,8.3l1.8,1.2l3.8-4.1V4.2z"></path></svg></a>
</h2>
<a href="http://fakehost/es/section/opinion" target="_blank">Más en Opinión ›</a>
</header>
<ol>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/24/espanol/opinion/covid-variantes-omicron.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/20/opinion/20Nkengasong/20Nkengasong-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/20/opinion/20Nkengasong/20Nkengasong-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/20/opinion/20Nkengasong/20Nkengasong-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/20/opinion/20Nkengasong/20Nkengasong-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/20/opinion/20Nkengasong/20Nkengasong-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/20/opinion/20Nkengasong/20Nkengasong-hpSmall.jpg 163w" alt=""></a>
<figcaption>
<span><span>Credit</span>Danielle Chenette</span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/23/espanol/opinion/descansar-trabajo.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/23/opinion/sunday/20kukla/20kukla-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/23/opinion/sunday/20kukla/20kukla-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/23/opinion/sunday/20kukla/20kukla-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/23/opinion/sunday/20kukla/20kukla-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/23/opinion/sunday/20kukla/20kukla-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/23/opinion/sunday/20kukla/20kukla-hpSmall.jpg 163w" alt=""></a>
<figcaption>
<span><span>Credit</span>Cari Vander Yacht</span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/20/espanol/opinion/adicciones-como-ayudar.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/16/opinion/sunday/15Fisher/15Fisher-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/16/opinion/sunday/15Fisher/15Fisher-thumbLarge-v2.jpg 150w, https://static01.nyt.com/images/2022/01/16/opinion/sunday/15Fisher/15Fisher-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/16/opinion/sunday/15Fisher/15Fisher-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/16/opinion/sunday/15Fisher/15Fisher-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/16/opinion/sunday/15Fisher/15Fisher-hpSmall.jpg 163w" alt="  "></a>
<figcaption>
<span><span>Credit</span>Bianca Bagnarelli</span>
</figcaption>
</figure>
</article>
</li>
<li>
<ol>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/19/espanol/opinion/omicron-covid.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/19/multimedia/19Hanage/19Hanage-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/19/multimedia/19Hanage/19Hanage-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/19/multimedia/19Hanage/19Hanage-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/19/multimedia/19Hanage/19Hanage-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/19/multimedia/19Hanage/19Hanage-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/19/multimedia/19Hanage/19Hanage-hpSmall.jpg 163w" alt=""></a>
<figcaption>
<span><span>Credit</span>Kim Raff for The New York Times</span>
</figcaption>
</figure>
</article>
</li>
</ol>
</li>
</ol>
</section>
<section>
<header>
<h2>
<a href="http://fakehost/es/spotlight/especial" target="_blank">Especial<svg viewbox="0 0 5.6 9.5" width="5.6" height="9.5" xml:space="preserve" xmlns="http://www.w3.org/1999/xlink" x="0px" y="0px">
<path d="M5.6,4.2L1.8,0L0,1.3l3.1,3.5L0,8.3l1.8,1.2l3.8-4.1V4.2z"></path></svg></a>
</h2>
<a href="http://fakehost/es/spotlight/especial" target="_blank">Más en Especial ›</a>
</header>
<ol>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/19/espanol/desafio-come-bien.html" target="_blank"><img src="https://static01.nyt.com/images/2021/12/30/well/30well-eatmore-challenge1/merlin_199522602_493a2c6c-4ff1-4e6e-a378-6e4810ba997b-videoLarge.jpg" srcset="https://static01.nyt.com/images/2021/12/30/well/30well-eatmore-challenge1/30well-eatmore-challenge1-thumbLarge.jpg 150w, https://static01.nyt.com/images/2021/12/30/well/30well-eatmore-challenge1/merlin_199522602_493a2c6c-4ff1-4e6e-a378-6e4810ba997b-videoLarge.jpg 768w, https://static01.nyt.com/images/2021/12/30/well/30well-eatmore-challenge1/merlin_199522602_493a2c6c-4ff1-4e6e-a378-6e4810ba997b-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2021/12/30/well/30well-eatmore-challenge1/merlin_199522602_493a2c6c-4ff1-4e6e-a378-6e4810ba997b-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2021/12/30/well/30well-eatmore-challenge1/merlin_199522602_493a2c6c-4ff1-4e6e-a378-6e4810ba997b-hpSmall.jpg 163w" alt=" "></a>
<figcaption>
<span><span>Credit</span>Photo Illustration by Andrew B. Myers for The New York Times</span>
</figcaption>
</figure>
<div>
<h2>
<a href="http://fakehost/es/2022/01/19/espanol/desafio-come-bien.html" target="_blank">El desafío Come bien</a>
</h2>
<p>
Actualiza tus hábitos alimenticios este año, sin necesidad de hacer dieta.
</p>
<p>
<span>By <span itemprop="name">Tara Parker-Pope</span></span>
</p>
</div>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/08/espanol/desafio-come-bien.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/04/well/30well-NL-getready/30well-NL-getready-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/04/well/30well-NL-getready/30well-NL-getready-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/04/well/30well-NL-getready/30well-NL-getready-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/04/well/30well-NL-getready/30well-NL-getready-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/04/well/30well-NL-getready/30well-NL-getready-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/04/well/30well-NL-getready/merlin_199256238_147af738-0898-4260-99c1-2e371133ab11-hpSmall.jpg 163w" alt=""></a>
<figcaption>
<span><span>Credit</span>Photo Illustration by Andrew B. Myers for The New York Times</span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/10/espanol/balance-comida-antojo.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/11/well/10sciWell-EatWell-Cravings-Week-ESP-1/merlin_199522599_79543c1f-9951-4849-b1fd-395b1b97351c-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/11/well/10sciWell-EatWell-Cravings-Week-ESP-1/10well-cravings-challenge2-thumbLarge-v2.jpg 150w, https://static01.nyt.com/images/2022/01/11/well/10sciWell-EatWell-Cravings-Week-ESP-1/merlin_199522599_79543c1f-9951-4849-b1fd-395b1b97351c-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/11/well/10sciWell-EatWell-Cravings-Week-ESP-1/merlin_199522599_79543c1f-9951-4849-b1fd-395b1b97351c-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/11/well/10sciWell-EatWell-Cravings-Week-ESP-1/merlin_199522599_79543c1f-9951-4849-b1fd-395b1b97351c-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/11/well/10sciWell-EatWell-Cravings-Week-ESP-1/merlin_199522599_79543c1f-9951-4849-b1fd-395b1b97351c-hpSmall.jpg 163w" alt=""></a>
<figcaption>
<span><span>Credit</span>Fotoilustraciones de Andrew B. Myers para The New York Times</span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/17/espanol/dieta-alimentacion-intuitiva.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/18/well/17Well-EatWell-Intuitive-onsite-Week3-ESP-1/merlin_200240556_6593abcf-64b4-460e-909b-37d65dcbee10-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/18/well/17Well-EatWell-Intuitive-onsite-Week3-ESP-1/17well-eatwell-giveup-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/18/well/17Well-EatWell-Intuitive-onsite-Week3-ESP-1/merlin_200240556_6593abcf-64b4-460e-909b-37d65dcbee10-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/18/well/17Well-EatWell-Intuitive-onsite-Week3-ESP-1/merlin_200240556_6593abcf-64b4-460e-909b-37d65dcbee10-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/18/well/17Well-EatWell-Intuitive-onsite-Week3-ESP-1/merlin_200240556_6593abcf-64b4-460e-909b-37d65dcbee10-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/18/well/17Well-EatWell-Intuitive-onsite-Week3-ESP-1/merlin_200240556_6593abcf-64b4-460e-909b-37d65dcbee10-hpSmall.jpg 163w" alt=""></a>
<figcaption>
<span><span>Credit</span>Fotoilustración de Andrew B. Myers para The New York Times</span>
</figcaption>
</figure>
</article>
</li>
</ol>
</section>
<section>
<header>
<h2>
<a href="http://fakehost/es/spotlight/el-brote-de-coronavirus" target="_blank">El brote de Coronavirus<svg viewbox="0 0 5.6 9.5" width="5.6" height="9.5" xml:space="preserve" xmlns="http://www.w3.org/1999/xlink" x="0px" y="0px">
<path d="M5.6,4.2L1.8,0L0,1.3l3.1,3.5L0,8.3l1.8,1.2l3.8-4.1V4.2z"></path></svg></a>
</h2>
<a href="http://fakehost/es/spotlight/el-brote-de-coronavirus" target="_blank">Más en El brote de Coronavirus ›</a>
</header>
<ol>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/21/espanol/covid-vacuna-inmunidad.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/11/well/00well-covid-super-immunity/00well-covid-super-immunity-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/11/well/00well-covid-super-immunity/00well-covid-super-immunity-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/11/well/00well-covid-super-immunity/00well-covid-super-immunity-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/11/well/00well-covid-super-immunity/00well-covid-super-immunity-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/11/well/00well-covid-super-immunity/00well-covid-super-immunity-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/11/well/00well-covid-super-immunity/00well-covid-super-immunity-hpSmall.jpg 163w" alt=""></a>
<figcaption>
<span><span>Credit</span></span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/20/espanol/omicron-covid-prolongada.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/19/science/19omicron-longcovid-explainer1-esp-1/19omicron-longcovid-explainer1-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/19/science/19omicron-longcovid-explainer1-esp-1/19omicron-longcovid-explainer1-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/19/science/19omicron-longcovid-explainer1-esp-1/19omicron-longcovid-explainer1-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/19/science/19omicron-longcovid-explainer1-esp-1/19omicron-longcovid-explainer1-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/19/science/19omicron-longcovid-explainer1-esp-1/19omicron-longcovid-explainer1-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/19/science/19omicron-longcovid-explainer1-esp-1/19omicron-longcovid-explainer1-hpSmall.jpg 163w" alt="Largas filas para hacerse pruebas de coronavirus en Jonesboro, Georgia, este mes. La variante ómicron se identificó a finales de noviembre, por lo que es demasiado pronto para decir cuánto tiempo pueden persistir los síntomas."></a>
<figcaption>
<span><span>Credit</span>Dustin Chambers para The New York Times</span>
</figcaption>
</figure>
<div>
<h2>
<a href="http://fakehost/es/2022/01/20/espanol/omicron-covid-prolongada.html" target="_blank">¿Ómicron puede causar covid prolongada?</a>
</h2>
<p>
Los científicos dicen que aún es muy pronto para saber si quienes se infectan con la nueva variante tendrán síntomas persistentes. Una infección leve no necesariamente es señal de que hay menos riesgo.
</p>
<p>
<span>By <span itemprop="name">Pam Belluck</span></span>
</p>
</div>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/20/espanol/vacunas-arnm-covid.html" target="_blank"><img src="https://static01.nyt.com/images/2021/12/01/science/00VIRUS-mRNA-promo/00VIRUS-mRNA-promo-videoLarge-v3.jpg" srcset="https://static01.nyt.com/images/2021/12/01/science/00VIRUS-mRNA-promo/00VIRUS-mRNA-promo-thumbLarge.jpg 150w, https://static01.nyt.com/images/2021/12/01/science/00VIRUS-mRNA-promo/00VIRUS-mRNA-promo-videoLarge-v3.jpg 768w, https://static01.nyt.com/images/2021/12/01/science/00VIRUS-mRNA-promo/00VIRUS-mRNA-promo-mediumThreeByTwo225-v3.jpg 225w, https://static01.nyt.com/images/2021/12/01/science/00VIRUS-mRNA-promo/00VIRUS-mRNA-promo-mediumThreeByTwo440-v3.jpg 440w, https://static01.nyt.com/images/2021/12/01/science/00VIRUS-mRNA-promo/00VIRUS-mRNA-promo-hpSmall.jpg 163w" alt="A 3-D plaster model of a coronavirus spike protein in the office of Dr. Barney Graham of the Vaccine Research Center of the National Institutes of Health."></a>
<figcaption>
<span><span>Credit</span>Johnathon Kelso for The New York Times</span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/18/espanol/prueba-covid-saliva.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/14/science/14virus-saliva1-esp-1/14virus-saliva1-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/14/science/14virus-saliva1-esp-1/14virus-saliva1-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/14/science/14virus-saliva1-esp-1/14virus-saliva1-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/14/science/14virus-saliva1-esp-1/14virus-saliva1-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/14/science/14virus-saliva1-esp-1/14virus-saliva1-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/14/science/14virus-saliva1-esp-1/14virus-saliva1-hpSmall.jpg 163w" alt="Un centro de pruebas de COVID-19 realizadas con saliva en la Universidad de Minnesota, en Mineápolis"></a>
<figcaption>
<span><span>Credit</span>Jenn Ackerman para The New York Times</span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/17/espanol/n95-covid-mascarillas.html" target="_blank"><img src="https://static01.nyt.com/images/2022/02/13/well/13well-n95mask-guide1-esp-1/13well-n95mask-guide1-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/02/13/well/13well-n95mask-guide1-esp-1/13well-n95mask-guide1-thumbLarge-v2.jpg 150w, https://static01.nyt.com/images/2022/02/13/well/13well-n95mask-guide1-esp-1/13well-n95mask-guide1-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/02/13/well/13well-n95mask-guide1-esp-1/13well-n95mask-guide1-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/02/13/well/13well-n95mask-guide1-esp-1/13well-n95mask-guide1-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/02/13/well/13well-n95mask-guide1-esp-1/13well-n95mask-guide1-hpSmall-v2.jpg 163w" alt=""></a>
<figcaption>
<span><span>Credit</span>Charlie Rubin para The New York Times</span>
</figcaption>
</figure>
</article>
</li>
</ol>
</section>
<section>
<header>
<h2>
<a href="http://fakehost/es/spotlight/elecciones-2020" target="_blank">Estados Unidos<svg viewbox="0 0 5.6 9.5" width="5.6" height="9.5" xml:space="preserve" xmlns="http://www.w3.org/1999/xlink" x="0px" y="0px">
<path d="M5.6,4.2L1.8,0L0,1.3l3.1,3.5L0,8.3l1.8,1.2l3.8-4.1V4.2z"></path></svg></a>
</h2>
<a href="http://fakehost/es/spotlight/elecciones-2020" target="_blank">Más en Estados Unidos ›</a>
</header>
<ol>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/23/espanol/texas-operacion-estrella-solitaria.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/14/us/00texas-guards-esp-1/00texas-guards-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/14/us/00texas-guards-esp-1/00texas-guards-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/14/us/00texas-guards-esp-1/00texas-guards-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/14/us/00texas-guards-esp-1/00texas-guards-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/14/us/00texas-guards-esp-1/00texas-guards-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/14/us/00texas-guards-esp-1/00texas-guards-hpSmall.jpg 163w" alt="Una presentación sobre la Operación Estrella Solitaria, en Weslaco, Texas, el año pasado"></a>
<figcaption>
<span><span>Credit</span>Christopher Lee para The New York Times</span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/07/espanol/donald-trump-republicanos-capitolio.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/05/us/politics/05trump-1-esp-1/merlin_182049705_b7c15186-9167-4b6e-931d-2f98e7ba04ef-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/05/us/politics/05trump-1-esp-1/05trump-1-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/05/us/politics/05trump-1-esp-1/merlin_182049705_b7c15186-9167-4b6e-931d-2f98e7ba04ef-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/05/us/politics/05trump-1-esp-1/merlin_182049705_b7c15186-9167-4b6e-931d-2f98e7ba04ef-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/05/us/politics/05trump-1-esp-1/merlin_182049705_b7c15186-9167-4b6e-931d-2f98e7ba04ef-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/05/us/politics/05trump-1-esp-1/merlin_182049705_b7c15186-9167-4b6e-931d-2f98e7ba04ef-hpSmall.jpg 163w" alt="Una multitud se reunió en el National Mall el 6 de enero de 2021, cuando el expresidente Donald Trump cuestionó los resultados de las elecciones de 2020."></a>
<figcaption>
<span><span>Credit</span>Pete Marovich para The New York Times</span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/06/espanol/6-enero-capitolio-ataque.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/05/us/politics/06dc-factcheck-ESP-1/merlin_182053380_725e614e-df1a-4fdd-8963-20af3e9ab5cf-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/05/us/politics/06dc-factcheck-ESP-1/merlin_182053380_725e614e-df1a-4fdd-8963-20af3e9ab5cf-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/05/us/politics/06dc-factcheck-ESP-1/merlin_182053380_725e614e-df1a-4fdd-8963-20af3e9ab5cf-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/05/us/politics/06dc-factcheck-ESP-1/merlin_182053380_725e614e-df1a-4fdd-8963-20af3e9ab5cf-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/05/us/politics/06dc-factcheck-ESP-1/merlin_182053380_725e614e-df1a-4fdd-8963-20af3e9ab5cf-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/05/us/politics/06dc-factcheck-ESP-1/merlin_182053380_725e614e-df1a-4fdd-8963-20af3e9ab5cf-hpSmall.jpg 163w" alt="Ninguna de las más de 729 personas acusadas en relación con los disturbios del Capitolio tiene hasta ahora ninguna conexión con los antifa, según una base de datos de NPR sobre registros de detenciones."></a>
<figcaption>
<span><span>Credit</span>Jason Andrew para The New York Times</span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/05/espanol/asalto-capitolio-investigacion.html" target="_blank"><img src="https://static01.nyt.com/images/2022/01/03/us/politics/03dc-riot-explainer-esp-1/merlin_190720395_49af63b7-f075-41dc-a3d3-5ed460494d3c-videoLarge.jpg" srcset="https://static01.nyt.com/images/2022/01/03/us/politics/03dc-riot-explainer-esp-1/merlin_190720395_49af63b7-f075-41dc-a3d3-5ed460494d3c-thumbLarge.jpg 150w, https://static01.nyt.com/images/2022/01/03/us/politics/03dc-riot-explainer-esp-1/merlin_190720395_49af63b7-f075-41dc-a3d3-5ed460494d3c-videoLarge.jpg 768w, https://static01.nyt.com/images/2022/01/03/us/politics/03dc-riot-explainer-esp-1/merlin_190720395_49af63b7-f075-41dc-a3d3-5ed460494d3c-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2022/01/03/us/politics/03dc-riot-explainer-esp-1/merlin_190720395_49af63b7-f075-41dc-a3d3-5ed460494d3c-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2022/01/03/us/politics/03dc-riot-explainer-esp-1/merlin_190720395_49af63b7-f075-41dc-a3d3-5ed460494d3c-hpSmall.jpg 163w" alt="El expresidente Donald Trump el año pasado. Liz Cheney, representante republicana por Wyoming, ha calificado su lenta respuesta al atentado del 6 de enero como una negligencia en el cumplimiento del deber."></a>
<figcaption>
<span><span>Credit</span>Cooper Neill para The New York Times</span>
</figcaption>
</figure>
</article>
</li>
<li>
<article>
<figure aria-label="media" role="group">
<span>Photo</span><a data-rref="" href="http://fakehost/es/2022/01/04/espanol/elizabeth-holmes-juicio.html" target="_blank"><img src="https://static01.nyt.com/images/2021/12/28/business/00holmes-siliconvalley1-esp-1/00holmes-siliconvalley1-videoLarge.jpg" srcset="https://static01.nyt.com/images/2021/12/28/business/00holmes-siliconvalley1-esp-1/00holmes-siliconvalley1-thumbLarge.jpg 150w, https://static01.nyt.com/images/2021/12/28/business/00holmes-siliconvalley1-esp-1/00holmes-siliconvalley1-videoLarge.jpg 768w, https://static01.nyt.com/images/2021/12/28/business/00holmes-siliconvalley1-esp-1/00holmes-siliconvalley1-mediumThreeByTwo225.jpg 225w, https://static01.nyt.com/images/2021/12/28/business/00holmes-siliconvalley1-esp-1/00holmes-siliconvalley1-mediumThreeByTwo440.jpg 440w, https://static01.nyt.com/images/2021/12/28/business/00holmes-siliconvalley1-esp-1/merlin_199542435_06a25cb4-18c0-45d3-b264-a9d8cccf3c33-hpSmall.jpg 163w" alt="Durante una década, Holmes engañó a inversionistas inteligentes, a cientos de empleados inteligentes, a un comité de figuras ilustres y a los medios de comunicación que estaban ansiosos por ungir a una nueva estrella."></a>
<figcaption>
<span><span>Credit</span>Jenny Hueston</span>
</figcaption>
</figure>
<div>
<h2>
<a href="http://fakehost/es/2022/01/04/espanol/elizabeth-holmes-juicio.html" target="_blank">El auge y la caída de Elizabeth Holmes</a>
</h2>
<p>
El caso de la fundadora de Theranos podría cambiar el estatus de culto que tienen algunos emprendedores tecnológicos a los que no se les cuestionan sus ambiciosos proyectos ni se les exige que rindan cuentas.
</p>
<p>
<span>By <span itemprop="name">David Streitfeld</span></span>
</p>
</div>
</article>
</li>
</ol>
</section>
</div>
<div>
<nav>
<ul role="tablist">
<li role="presentation">
<a href="#stream-panel" aria-controls="stream-panel" aria-selected="true" data-id="latest" role="tab">Lo más reciente</a>
</li>
<li role="presentation">
<a role="tab" href="#stream-panel" aria-controls="stream-panel" data-id="search" aria-selected="false"><svg viewbox="0 0 16 16">
<path fill="#333" d="M11.3,9.2C11.7,8.4,12,7.5,12,6.5C12,3.5,9.5,1,6.5,1S1,3.5,1,6.5S3.5,12,6.5,12c1,0,1.9-0.3,2.7-0.7l3.3,3.3c0.3,0.3,0.7,0.4,1.1,0.4s0.8-0.1,1.1-0.4c0.6-0.6,0.6-1.5,0-2.1L11.3,9.2zM6.5,10.3c-2.1,0-3.8-1.7-3.8-3.8c0-2.1,1.7-3.8,3.8-3.8c2.1,0,3.8,1.7,3.8,3.8C10.3,8.6,8.6,10.3,6.5,10.3z"></path></svg><span>Buscar</span>
</a>
</li>
</ul>
</nav>
<section id="stream-panel">
</section>
</div>
</section>
</main>
</div></article>

4
src/full_text_parser/mod.rs
View file

@ -21,6 +21,7 @@ use libxml::tree::{Document, Node, NodeType};
use libxml::xpath::Context; use libxml::xpath::Context;
use reqwest::header::HeaderMap; use reqwest::header::HeaderMap;
use reqwest::{Client, Url}; use reqwest::{Client, Url};
use std::collections::HashSet;
use std::path::Path; use std::path::Path;
use std::str::from_utf8; use std::str::from_utf8;
@ -493,8 +494,7 @@ impl FullTextParser {
if tag_name == "IMG" || tag_name == "PICTURE" { if tag_name == "IMG" || tag_name == "PICTURE" {
_ = node.set_attribute(copy_to, &val); _ = node.set_attribute(copy_to, &val);
} else if tag_name == "FIGURE" } else if tag_name == "FIGURE"
&& !Util::has_decendent_tag(&node, "img") && !Util::has_any_descendent_tag(&node, &HashSet::from(["IMG", "PICTURE"]))
&& !Util::has_decendent_tag(&node, "picture")
{ {
//if the item is a <figure> that does not contain an image or picture, create one and place it inside the figure //if the item is a <figure> that does not contain an image or picture, create one and place it inside the figure
//see the nytimes-3 testcase for an example //see the nytimes-3 testcase for an example

18
src/full_text_parser/readability/tests.rs
View file

@ -382,10 +382,20 @@ async fn nytimes_2() {
run_test("nytimes-2").await run_test("nytimes-2").await
} }
// #[tokio::test] #[tokio::test]
// async fn nytimes_3() { async fn nytimes_3() {
// run_test("nytimes-3").await run_test("nytimes-3").await
// } }
#[tokio::test]
async fn nytimes_4() {
run_test("nytimes-4").await
}
#[tokio::test]
async fn nytimes_5() {
run_test("nytimes-5").await
}
#[tokio::test] #[tokio::test]
async fn webmd_1() { async fn webmd_1() {

27
src/util.rs
View file

@ -1,3 +1,5 @@
use std::collections::HashSet;
use libxml::{ use libxml::{
tree::{Document, Node, NodeType}, tree::{Document, Node, NodeType},
xpath::Context, xpath::Context,
@ -340,14 +342,21 @@ impl Util {
1.0 - distance_b 1.0 - distance_b
} }
pub fn has_decendent_tag(node: &Node, tag_name: &str) -> bool { pub fn has_any_descendent_tag(node: &Node, tag_names: &HashSet<&str>) -> bool {
let mut node_iter = Self::next_node(node, false); let children = node.get_child_elements();
while let Some(node) = node_iter { let is_direct_child = children
if Self::has_tag_name(Some(&node), tag_name) { .iter()
.map(|node| node.get_name().to_uppercase())
.any(|name| tag_names.contains(name.as_str()));
if is_direct_child {
return true;
}
for child in children {
if Util::has_any_descendent_tag(&child, tag_names) {
return true; return true;
} }
node_iter = Util::next_node(&node, false);
} }
false false
@ -521,7 +530,11 @@ impl Util {
for mut node in nodes.into_iter().rev() { for mut node in nodes.into_iter().rev() {
if Util::get_class_weight(&node) < 0 { if Util::get_class_weight(&node) < 0 {
log::debug!("Removing header with low class weight: {} {}", node.get_name(), node.get_attribute("class").unwrap_or_default()); log::debug!(
"Removing header with low class weight: {} {}",
node.get_name(),
node.get_attribute("class").unwrap_or_default()
);
node.unlink(); node.unlink();
} }
} }