From c42ffa57a239370ea35ae3aaeff2613820f418d9 Mon Sep 17 00:00:00 2001 From: Jan Lukas Gernert Date: Fri, 31 Mar 2023 09:37:23 +0200 Subject: [PATCH] start adding nytimes tests --- .../tests/readability/nytimes-1/expected.html | 89 + .../tests/readability/nytimes-1/source.html | 5530 +++++++++++++++++ .../tests/readability/nytimes-2/expected.html | 96 + .../tests/readability/nytimes-2/source.html | 5236 ++++++++++++++++ .../tests/readability/nytimes-3/source.html | 1638 +++++ .../tests/readability/nytimes-4/source.html | 3896 ++++++++++++ .../tests/readability/nytimes-5/source.html | 1540 +++++ src/full_text_parser/mod.rs | 15 +- src/full_text_parser/readability/tests.rs | 15 + src/util.rs | 12 + 10 files changed, 18063 insertions(+), 4 deletions(-) create mode 100644 resources/tests/readability/nytimes-1/expected.html create mode 100644 resources/tests/readability/nytimes-1/source.html create mode 100644 resources/tests/readability/nytimes-2/expected.html create mode 100644 resources/tests/readability/nytimes-2/source.html create mode 100644 resources/tests/readability/nytimes-3/source.html create mode 100644 resources/tests/readability/nytimes-4/source.html create mode 100644 resources/tests/readability/nytimes-5/source.html diff --git a/resources/tests/readability/nytimes-1/expected.html b/resources/tests/readability/nytimes-1/expected.html new file mode 100644 index 0000000..3d978c8 --- /dev/null +++ b/resources/tests/readability/nytimes-1/expected.html @@ -0,0 +1,89 @@ +
+ + +
+
+ + + + + + + + + +
+
+ Photo +
+

+ + +
+
+ United Nations peacekeepers at a refugee camp in Sudan on Monday. In exchange for the lifting of United States trade sanctions, Sudan has said it will improve access for aid groups, stop supporting rebels in neighboring South Sudan and cooperate with American intelligence agents. + + Credit Ashraf Shazly/Agence France-Presse — Getty Images +
+
+

LONDON — After nearly 20 years of hostile relations, the American government plans to reverse its position on Sudan and lift trade sanctions, Obama administration officials said late Thursday.

+

Sudan is one of the poorest, most isolated and most violent countries in Africa, and for years the United States has imposed punitive measures against it in a largely unsuccessful attempt to get the Sudanese government to stop killing its own people.

+

On Friday, the Obama administration will announce a new Sudan strategy. For the first time since the 1990s, the nation will be able to trade extensively with the United States, allowing it to buy goods like tractors and spare parts and attract much-needed investment in its collapsing economy.

+

In return, Sudan will improve access for aid groups, stop supporting rebels in neighboring South Sudan, cease the bombing of insurgent territory and cooperate with American intelligence agents.

+

American officials said Sudan had already shown important progress on a number of these fronts. But to make sure the progress continues, the executive order that President Obama plans to sign on Friday, days before leaving office, will have a six-month review period. If Sudan fails to live up to its commitments, the embargo can be reinstated.

+ +

Analysts said good relations with Sudan could strengthen moderate voices within the country and give the Sudanese government incentives to refrain from the brutal tactics that have defined it for decades.

+

In 1997, President Bill Clinton imposed a comprehensive trade embargo against Sudan and blocked the assets of the Sudanese government, which was suspected of sponsoring international terrorism. In the mid-1990s, Osama bin Laden lived in Khartoum, the capital, as a guest of Sudan’s government.

+

In 1998, Bin Laden’s agents blew up the United States Embassies in Kenya and Tanzania, killing more than 200 people. In retaliation, Mr. Clinton ordered a cruise missile strike against a pharmaceutical factory in Khartoum.

+

Since then, American-Sudanese relations have steadily soured. The conflict in Darfur, a vast desert region of western Sudan, was a low point. After rebels in Darfur staged an uprising in 2003, Sudanese security services and their militia allies slaughtered tens of thousands of civilians, leading to condemnation around the world, genocide charges at the International Criminal Court against Sudan’s president, Omar Hassan al-Bashir, and a new round of American sanctions.

+

American officials said Thursday that the American demand that Mr. Bashir be held accountable had not changed. Neither has Sudan’s status as one of the few countries, along with Iran and Syria, that remain on the American government’s list of state sponsors of terrorism.

+

Sales of military equipment will still be prohibited, and some Sudanese militia and rebel leaders will still face sanctions.

+

But the Obama administration is clearly trying to open a door to Sudan. There is intense discontent across the country, and its economy is imploding. American officials have argued for years that it was time to help Sudan dig itself out of the hole it had created.

+

Officials divulged Thursday that the Sudanese government had allowed two visits by American operatives to a restricted border area near Libya, which they cited as evidence of a new spirit of cooperation on counterterrorism efforts.

+ +

In addition to continuing violence in Darfur, several other serious conflicts are raging in southern and central Sudan, along with a civil war in newly independent South Sudan, which Sudan has been suspected of inflaming with covert arms shipments.

+

Eric Reeves, one of the leading American academic voices on Sudan, said he was “appalled” that the American government was lifting sanctions.

+

He said that Sudan’s military-dominated government continued to commit grave human rights abuses and atrocities, and he noted that just last week Sudanese security services killed more than 10 civilians in Darfur.

+

“There is no reason to believe the guys in charge have changed their stripes,” said Mr. Reeves, a senior fellow at the François-Xavier Bagnoud Center for Health and Human Rights at Harvard University. “These guys are the worst of the worst.”

+

Obama administration officials said that they had briefed President-elect Donald J. Trump’s transition team, but that they did not know if Mr. Trump would stick with a policy of warmer relations with Sudan.

+

They said that Sudan had a long way to go in terms of respecting human rights, but that better relations could help increase American leverage.

+

Mr. Reeves said he thought that the American government was being manipulated and that the Obama administration had made a “deal with the devil.”

+ +

Continue reading the main story +

+
+ + + + + + + +
+ + +
+ + + +
+ + + + +
+ +
+ + + + + +
+ + + +
diff --git a/resources/tests/readability/nytimes-1/source.html b/resources/tests/readability/nytimes-1/source.html new file mode 100644 index 0000000..d8d1243 --- /dev/null +++ b/resources/tests/readability/nytimes-1/source.html @@ -0,0 +1,5530 @@ + + + + + + + + + + + + United States to Lift Sudan Sanctions - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + + +
+ + + + + + + + + + + +
+ + +
+
+ + + + +
+
+ +

+ Africa

+

United States to Lift Sudan Sanctions

+
+ + +

By JEFFREY GETTLEMAN + +

+ +
+
+ + + + +
+ +
+
+ +
+ +
+ + + + +
+
+
+ +
+ +
+ +
+ + +
+
+ United Nations peacekeepers at a refugee camp in Sudan on Monday. In exchange for the lifting of United States trade sanctions, Sudan has said it will improve access for aid groups, stop supporting rebels in neighboring South Sudan and cooperate with American intelligence agents. + + Ashraf Shazly/Agence France-Presse — Getty Images +
+
+

LONDON — After nearly 20 years of hostile relations, the American government plans to reverse its position on Sudan and lift trade sanctions, Obama administration officials said late Thursday.

+

Sudan is one of the poorest, most isolated and most violent countries in Africa, and for years the United States has imposed punitive measures against it in a largely unsuccessful attempt to get the Sudanese government to stop killing its own people.

+

On Friday, the Obama administration will announce a new Sudan strategy. For the first time since the 1990s, the nation will be able to trade extensively with the United States, allowing it to buy goods like tractors and spare parts and attract much-needed investment in its collapsing economy.

+

In return, Sudan will improve access for aid groups, stop supporting rebels in neighboring South Sudan, cease the bombing of insurgent territory and cooperate with American intelligence agents.

+

American officials said Sudan had already shown important progress on a number of these fronts. But to make sure the progress continues, the executive order that President Obama plans to sign on Friday, days before leaving office, will have a six-month review period. If Sudan fails to live up to its commitments, the embargo can be reinstated.

+ +

Analysts said good relations with Sudan could strengthen moderate voices within the country and give the Sudanese government incentives to refrain from the brutal tactics that have defined it for decades.

+

In 1997, President Bill Clinton imposed a comprehensive trade embargo against Sudan and blocked the assets of the Sudanese government, which was suspected of sponsoring international terrorism. In the mid-1990s, Osama bin Laden lived in Khartoum, the capital, as a guest of Sudan’s government.

+

In 1998, Bin Laden’s agents blew up the United States Embassies in Kenya and Tanzania, killing more than 200 people. In retaliation, Mr. Clinton ordered a cruise missile strike against a pharmaceutical factory in Khartoum.

+

Since then, American-Sudanese relations have steadily soured. The conflict in Darfur, a vast desert region of western Sudan, was a low point. After rebels in Darfur staged an uprising in 2003, Sudanese security services and their militia allies slaughtered tens of thousands of civilians, leading to condemnation around the world, genocide charges at the International Criminal Court against Sudan’s president, Omar Hassan al-Bashir, and a new round of American sanctions.

+

American officials said Thursday that the American demand that Mr. Bashir be held accountable had not changed. Neither has Sudan’s status as one of the few countries, along with Iran and Syria, that remain on the American government’s list of state sponsors of terrorism.

+

Sales of military equipment will still be prohibited, and some Sudanese militia and rebel leaders will still face sanctions.

+

But the Obama administration is clearly trying to open a door to Sudan. There is intense discontent across the country, and its economy is imploding. American officials have argued for years that it was time to help Sudan dig itself out of the hole it had created.

+

Officials divulged Thursday that the Sudanese government had allowed two visits by American operatives to a restricted border area near Libya, which they cited as evidence of a new spirit of cooperation on counterterrorism efforts.

+ +

In addition to continuing violence in Darfur, several other serious conflicts are raging in southern and central Sudan, along with a civil war in newly independent South Sudan, which Sudan has been suspected of inflaming with covert arms shipments.

+

Eric Reeves, one of the leading American academic voices on Sudan, said he was “appalled” that the American government was lifting sanctions.

+

He said that Sudan’s military-dominated government continued to commit grave human rights abuses and atrocities, and he noted that just last week Sudanese security services killed more than 10 civilians in Darfur.

+

“There is no reason to believe the guys in charge have changed their stripes,” said Mr. Reeves, a senior fellow at the François-Xavier Bagnoud Center for Health and Human Rights at Harvard University. “These guys are the worst of the worst.”

+

Obama administration officials said that they had briefed President-elect Donald J. Trump’s transition team, but that they did not know if Mr. Trump would stick with a policy of warmer relations with Sudan.

+

They said that Sudan had a long way to go in terms of respecting human rights, but that better relations could help increase American leverage.

+

Mr. Reeves said he thought that the American government was being manipulated and that the Obama administration had made a “deal with the devil.”

+
+
+ + +

A version of this article appears in print on January 13, 2017, on Page A7 of the New York edition with the headline: Sanctions on Sudan Will Be Lifted. Order Reprints| Today's Paper|Subscribe + +

+ +
+ +
+ +
+ + + +
+ + + + + + + +
+ + + + + +
+ +
+ +
+
+

Go to Home Page »

+

+ + + + The New York Times +

+ +
+ + + + +
+ + + +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+ + + diff --git a/resources/tests/readability/nytimes-2/expected.html b/resources/tests/readability/nytimes-2/expected.html new file mode 100644 index 0000000..3818957 --- /dev/null +++ b/resources/tests/readability/nytimes-2/expected.html @@ -0,0 +1,96 @@ +
+ + +
+
+ + + + + + + + + + + +
+
+ Photo +
+

+ + +
+
+ + Credit Harry Campbell +
+
+

Yahoo’s $4.8 billion sale to Verizon is a complicated beast, showing how different acquisition structures can affect how shareholders are treated.

+

First, let’s say what the Yahoo sale is not. It is not a sale of the publicly traded company. Instead, it is a sale of the Yahoo subsidiary and some related assets to Verizon.

+

The sale is being done in two steps. The first step will be the transfer of any assets related to Yahoo business to a singular subsidiary. This includes the stock in the business subsidiaries that make up Yahoo that are not already in the single subsidiary, as well as the odd assets like benefit plan rights. This is what is being sold to Verizon. A license of Yahoo’s oldest patents is being held back in the so-called Excalibur portfolio. This will stay with Yahoo, as will Yahoo’s stakes in Alibaba Group and Yahoo Japan.

+

It is hard to overestimate how complex an asset sale like this is. Some of the assets are self-contained, but they must be gathered up and transferred. Employees need to be shuffled around and compensation arrangements redone. Many contracts, like the now-infamous one struck with the search engine Mozilla, which may result in a payment of up to a $1 billion, will contain change-of-control provisions that will be set off and have to be addressed. Tax issues always loom large.

Continue reading the main story +

+
+ +
+ +

Continue reading the main story

+
+
+

In the second step, at the closing, Yahoo will sell the stock in the single subsidiary to Verizon. At that point, Yahoo will change its name to something without “Yahoo” in it. My favorite is simply Remain Co., the name Yahoo executives are using. Remain Co. will become a holding company for the Alibaba and Yahoo Japan stock. Included will also be $10 billion in cash, plus the Excalibur patent portfolio and a number of minority investments including Snapchat. Ahh, if only Yahoo had bought Snapchat instead of Tumblr (indeed, if only Yahoo had bought Google or Facebook when it had the chance).

+ +

Because it is a sale of a subsidiary, the $4.8 billion will be paid to Yahoo. Its shareholders will not receive any money unless Yahoo pays it out in a dividend (after paying taxes). Instead, Yahoo shareholders will be left holding shares in the renamed company.

+

Verizon’s Yahoo will then be run under the same umbrella as AOL. It is unclear whether there will be a further merger of the two businesses after the acquisition. Plans for Yahoo are still a bit in flux in part because of the abnormal sale process.

+

As for Remain Co., it will become a so-called investment company. This is a special designation for a company that holds securities for investment but does not operate a working business. Investment companies are subject to special regulation under the Investment Company Act of 1940. Remain Co. will probably just sit there, returning cash to shareholders and waiting for Alibaba to buy it in a tax-free transaction. (Alibaba says it has no plans to do this, but most people do not believe this).

+

The rights of Yahoo shareholders in this sale will be different from those in an ordinary sale, when an entire company is bought.

+

Ordinary sales are done in one of two ways: in a merger where the target is merged into a subsidiary of the buyer and the target shareholders receive the cash (or other consideration), or in a tender offer that gives the target shareholders a choice to tender into the offer or not. Then there will be a merger where the target is merged into the buyer’s subsidiary and the target shareholders are forcibly squeezed out, receiving the merger consideration. (if you want to know why you would choose one structure over another, I wrote a good primer in 2009.)

+

In either case, shareholders get a say. They either vote on the merger or decide whether to tender into the offer.

+

In both cases, there would be appraisal rights if the buyer pays cash. This means that shareholders can object to the deal by not voting for it or not tendering into the offer and instead asking a court to value their shares – this is what happened in Dell’s buyout in 2013.

+

The Yahoo deal, however, is not a sale of the public company. It is an asset sale, in which there is only a shareholder vote if there is a sale of “all” or “substantially all” of the assets of the company. Yahoo is not all of the assets or even “substantially all” – the Alibaba shares being left behind in Remain Co. are worth about $28 billion, or six times the value of the cash Verizon is paying for the Yahoo assets it is buying.

+ +

The courts have held that the definition of “substantially all” includes a change of business in a company because of an asset sale where the assets are “qualitatively vital.” And that certainly applies here. So there will be a vote – indeed, Yahoo has no problem with a vote – and shareholders are desperate to sell at this point.

+

There will be no appraisal rights, however. Again, in an asset sale, there are no appraisal rights. So anyone who votes against the deal and thinks this is a bum price is out of luck.

+

The different standards for voting and appraisal rights apply because the structure of the deal is a quirk of the law in Delaware, where Yahoo is incorporated, that allows lawyers to sometimes work around these issues simply by changing the way a deal is done.

+

In Yahoo’s case, this is not deliberate, though. It is simply the most expedient way to get rid of the assets.

+

Whether this is the most tax-efficient way is unclear to me as a nontax lawyer (email me if you know). Yahoo is likely to have a tax bill on the sale, possibly a substantial one. And I presume there were legal reasons for not using a Morris Trust structure, in which Yahoo would have been spun off and immediately sold to Verizon so that only Yahoo’s shareholders paid tax on the deal. In truth, the Yahoo assets being sold are only about 10 percent of the value of the company, so the time and logistics for such a sale when Yahoo is a melting ice cube may not have been worth it.

+

Finally, if another bidder still wants to acquire Yahoo, it has time. The agreement with Verizon allows Yahoo to terminate the deal and accept a superior offer by paying a $144 million breakup fee to Verizon. And if Yahoo shareholders change their minds and want to stick with Yahoo’s chief executive, Marissa Mayer, and vote down the deal, there is a so-called naked no-vote termination fee of $15 million payable to Verizon to reimburse expenses.

+

All in all, this was as hairy a deal as they come. There was the procedural and logistical complications of selling a company when the chief executive wanted to stay. Then there was the fact that this was an asset sale, including all of the challenges that go with it. Throw in all of the tax issues and the fact that this is a public company, and it is likely that the lawyers involved will have nightmares for years to come.

Continue reading the main story +

+
+ + + + + + + +
+ + +
+ + + +
+ + + + +
+ +
+ + + + + +
+ + + +
diff --git a/resources/tests/readability/nytimes-2/source.html b/resources/tests/readability/nytimes-2/source.html new file mode 100644 index 0000000..07dd748 --- /dev/null +++ b/resources/tests/readability/nytimes-2/source.html @@ -0,0 +1,5236 @@ + + + + + + + + + + Yahoo’s Sale to Verizon Leaves Shareholders With Little Say - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + + +
+ + + + + + + + + + + +
+ + +
+
+ + + +
+

+ + + +

+
+ + +
+
+ +

+

+

Yahoo’s Sale to Verizon Leaves Shareholders With Little Say

+
+
+

+ Deal Professor

+

By + STEVEN DAVIDOFF SOLOMON + + +

+
+ + +
+
+ + + + +
+ +
+
+ +
+ +
+ + + + +
+
+
+ +
+ +
+ +
+ + +
+
+ + Harry Campbell +
+
+

Yahoo’s $4.8 billion sale to Verizon is a complicated beast, showing how different acquisition structures can affect how shareholders are treated.

+

First, let’s say what the Yahoo sale is not. It is not a sale of the publicly traded company. Instead, it is a sale of the Yahoo subsidiary and some related assets to Verizon.

+

The sale is being done in two steps. The first step will be the transfer of any assets related to Yahoo business to a singular subsidiary. This includes the stock in the business subsidiaries that make up Yahoo that are not already in the single subsidiary, as well as the odd assets like benefit plan rights. This is what is being sold to Verizon. A license of Yahoo’s oldest patents is being held back in the so-called Excalibur portfolio. This will stay with Yahoo, as will Yahoo’s stakes in Alibaba Group and Yahoo Japan.

+

It is hard to overestimate how complex an asset sale like this is. Some of the assets are self-contained, but they must be gathered up and transferred. Employees need to be shuffled around and compensation arrangements redone. Many contracts, like the now-infamous one struck with the search engine Mozilla, which may result in a payment of up to a $1 billion, will contain change-of-control provisions that will be set off and have to be addressed. Tax issues always loom large.

+
+ + + +
+ +
+ +
+
+
+

In the second step, at the closing, Yahoo will sell the stock in the single subsidiary to Verizon. At that point, Yahoo will change its name to something without “Yahoo” in it. My favorite is simply Remain Co., the name Yahoo executives are using. Remain Co. will become a holding company for the Alibaba and Yahoo Japan stock. Included will also be $10 billion in cash, plus the Excalibur patent portfolio and a number of minority investments including Snapchat. Ahh, if only Yahoo had bought Snapchat instead of Tumblr (indeed, if only Yahoo had bought Google or Facebook when it had the chance).

+ +

Because it is a sale of a subsidiary, the $4.8 billion will be paid to Yahoo. Its shareholders will not receive any money unless Yahoo pays it out in a dividend (after paying taxes). Instead, Yahoo shareholders will be left holding shares in the renamed company.

+

Verizon’s Yahoo will then be run under the same umbrella as AOL. It is unclear whether there will be a further merger of the two businesses after the acquisition. Plans for Yahoo are still a bit in flux in part because of the abnormal sale process.

+

As for Remain Co., it will become a so-called investment company. This is a special designation for a company that holds securities for investment but does not operate a working business. Investment companies are subject to special regulation under the Investment Company Act of 1940. Remain Co. will probably just sit there, returning cash to shareholders and waiting for Alibaba to buy it in a tax-free transaction. (Alibaba says it has no plans to do this, but most people do not believe this).

+

The rights of Yahoo shareholders in this sale will be different from those in an ordinary sale, when an entire company is bought.

+

Ordinary sales are done in one of two ways: in a merger where the target is merged into a subsidiary of the buyer and the target shareholders receive the cash (or other consideration), or in a tender offer that gives the target shareholders a choice to tender into the offer or not. Then there will be a merger where the target is merged into the buyer’s subsidiary and the target shareholders are forcibly squeezed out, receiving the merger consideration. (if you want to know why you would choose one structure over another, I wrote a good primer in 2009.)

+

In either case, shareholders get a say. They either vote on the merger or decide whether to tender into the offer.

+

In both cases, there would be appraisal rights if the buyer pays cash. This means that shareholders can object to the deal by not voting for it or not tendering into the offer and instead asking a court to value their shares – this is what happened in Dell’s buyout in 2013.

+

The Yahoo deal, however, is not a sale of the public company. It is an asset sale, in which there is only a shareholder vote if there is a sale of “all” or “substantially all” of the assets of the company. Yahoo is not all of the assets or even “substantially all” – the Alibaba shares being left behind in Remain Co. are worth about $28 billion, or six times the value of the cash Verizon is paying for the Yahoo assets it is buying.

+ +

The courts have held that the definition of “substantially all” includes a change of business in a company because of an asset sale where the assets are “qualitatively vital.” And that certainly applies here. So there will be a vote – indeed, Yahoo has no problem with a vote – and shareholders are desperate to sell at this point.

+

There will be no appraisal rights, however. Again, in an asset sale, there are no appraisal rights. So anyone who votes against the deal and thinks this is a bum price is out of luck.

+

The different standards for voting and appraisal rights apply because the structure of the deal is a quirk of the law in Delaware, where Yahoo is incorporated, that allows lawyers to sometimes work around these issues simply by changing the way a deal is done.

+

In Yahoo’s case, this is not deliberate, though. It is simply the most expedient way to get rid of the assets.

+

Whether this is the most tax-efficient way is unclear to me as a nontax lawyer (email me if you know). Yahoo is likely to have a tax bill on the sale, possibly a substantial one. And I presume there were legal reasons for not using a Morris Trust structure, in which Yahoo would have been spun off and immediately sold to Verizon so that only Yahoo’s shareholders paid tax on the deal. In truth, the Yahoo assets being sold are only about 10 percent of the value of the company, so the time and logistics for such a sale when Yahoo is a melting ice cube may not have been worth it.

+

Finally, if another bidder still wants to acquire Yahoo, it has time. The agreement with Verizon allows Yahoo to terminate the deal and accept a superior offer by paying a $144 million breakup fee to Verizon. And if Yahoo shareholders change their minds and want to stick with Yahoo’s chief executive, Marissa Mayer, and vote down the deal, there is a so-called naked no-vote termination fee of $15 million payable to Verizon to reimburse expenses.

+

All in all, this was as hairy a deal as they come. There was the procedural and logistical complications of selling a company when the chief executive wanted to stay. Then there was the fact that this was an asset sale, including all of the challenges that go with it. Throw in all of the tax issues and the fact that this is a public company, and it is likely that the lawyers involved will have nightmares for years to come.

+
+ +
+
+ + +
+
+ + +
+
+ +
+ + + + + + + +
+ + + + + +
+ +
+ +
+
+

Go to Home Page »

+

+ + + + The New York Times +

+ +
+ + + + +
+ + + +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/tests/readability/nytimes-3/source.html b/resources/tests/readability/nytimes-3/source.html new file mode 100644 index 0000000..2936576 --- /dev/null +++ b/resources/tests/readability/nytimes-3/source.html @@ -0,0 +1,1638 @@ + + + + + + Manhole Fires and Burst Pipes: How Winter Wreaks Havoc on What’s Underneath N.Y.C. - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+
+
+
+
+
+
+
+ +
+
+ +
Skip to contentSkip to site index +
+
+ New York +
+
+ + +
+
+ +
+
+ +
+
+
+
+
+
+
+ Log In +
+
+ Today’s Paper +
+
+
+
+
+
+
+ + + + +
+
+
+
+
+ +
+

+ Manhole Fires and Burst Pipes: How Winter Wreaks Havoc on What’s Underneath N.Y.C. +

+
+

+ New York’s aging below-street infrastructure is tough to maintain, and the corrosive rock salt and “freeze-thaw” cycles of winter make it even worse. +

+
+
+
+
+
+ Image +
+
+ A Con Edison worker repairing underground cables this month in Flushing, Queens. The likely source of the problem was water and rock salt that had seeped underground.CreditCreditChang W. Lee/The New York Times +
+
+
+
+
+
+
+
+ Corey Kilgannon +
+
+

+ By Corey Kilgannon +

+
+
+
    +
  • + +
    • +
  • +
    + +
    +
    • +
+
+
+
+
+
+
+

+ [What you need to know to start the day: Get New York Today in your inbox.] +

+

+ A series of recent manhole fires in the heart of Manhattan forced the evacuation of several theaters and was a stark reminder that the subway is not the only creaky infrastructure beneath the streets of New York City. +

+

+ Underground lies a chaotic assemblage of utilities that, much like the subway, are lifelines for the city: a sprawling tangle of water mains, power cables, gas and steam lines, telecom wires and sewers. +

+

+ The city has one of the oldest and largest networks of subterranean infrastructure in the world, with some portions dating more than a century and prone to leaks and cracks. +

+

+ And winter — from the corrosive rock salt used on streets and sidewalks to “freeze-thaw” cycles that weaken pipes — makes infrastructure problems even worse. +

+
+ +
+
+
+

+ In the late 1800s, many of the city’s overhead utilities were buried to lessen the exposure to winter weather. “People think it’s all protected and safe, but it’s really not,” said Patrick McHugh, vice president of electrical engineering and planning for Con Edison, which maintains about 90,000 miles of underground cable in the city. +

+

+ “You have water, sewage, electricity and gas down there, and people don’t appreciate the effort that goes into keeping all that working,” he added. +

+
+ +
+
+
+
+
+
+ Image +
+
+ In the late 1800s, overhead utilities were buried to lessen the exposure to winter weather.CreditKirsten Luce for The New York Times +
+
+
+
+
+
+ +

+ When rock salt melts ice, and the water seeps down manholes and into electrical units, it can set off fires and explosions strong enough to pop a 300-pound manhole cover five stories into the air. +

+

+ For days after a storm, Con Edison officials say, they often deal with scores of electrical fires caused by the rock salt eating away at electrical cable insulation. The wet salt can create sparking that burns the insulation, producing both fire and gases that can combust and pop the manhole lids. +

+

+ To alleviate the threat, the officials said, the utility switched most of its manhole covers to vented ones that allow gases to escape, “so they cannot form a combustible amount,” Mr. McHugh said. +

+
+ +
+
+
+

+ “It also lets smoke escape, which can tip off the public to notify the authorities,” he added. +

+

+ Winter can also bring an increase in gas-line breakages. Con Edison, which maintains 4,300 miles of gas mains in and around New York City, records about 500 leaks — most of them nonemergencies — in a typical month, but many more in winter. +

+

+ Even this past January, which was unseasonably mild, there were 750 leaks, Con Edison officials said. +

+
+ +
+
+
+
+
+
+ Image +
+
+
+
+
+ There are typically between 400 and 600 water main breaks each year in New York City, an official said.CreditMichael Appleton for The New York Times +
+
+
+
+
+
+ +

+ The extreme temperature swings that many researchers link to climate change are adding to the challenges of winter. +

+

+ Officials monitor weather forecasts closely for freeze-thaw cycles, when they put extra repair crews on call. +

+

+ During a polar vortex in late January, for instance, single-digit temperatures in the city quickly ballooned into the 50s. The thaw, much welcomed by many New Yorkers, worried Tasos Georgelis, deputy commissioner for water and sewer operations at the Department of Environmental Protection, which operates the city’s water system. +

+

+ “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” Mr. Georgelis said. +

+

+ Along the city’s roughly 6,500 miles of water mains, there are typically between 400 and 600 breaks a year, he added. The majority occur in winter, when the cold can make older cast-iron mains brittle. +

+
+ +
+
+
+

+ Environmental Protection officials said the department repaired 75 water-main breaks in January, including one in Lower Manhattan that disrupted rush-hour subway service and another on the West Side that snarled traffic and left nearby buildings without water for hours. +

+

+ The city’s 7,500 miles of sewer lines are less affected by cold weather because they are generally buried deeper than other utilities, below the frost line, agency officials said. +

+
+ +
+
+
+
+
+
+ Image +
+
+
+
+
+ In 1978, a water main break caused severe flooding in Bushwick, Brooklyn.CreditFred R. Conrad/The New York Times +
+
+
+
+
+
+ +

+ Upgrading the city’s below-street utilities is a slow, painstaking process, “because you have such a fixed-in-place system,” said Rae Zimmerman, a research professor of planning and public administration at New York University. +

+

+ But there is progress. Con Edison officials said they had begun replacing the city’s nearly 1,600 miles of natural gas lines — which were made of either cast iron or unprotected steel — with plastic piping. The plastic is less susceptible to corrosion, cracks and leaks, said the officials, who added that they were swapping about 100 miles of line each year. +

+

+ The city is also replacing older, leak-prone water and sewer mains. +

+

+ Some pipes that are more than a century old hold up because they were built with a thicker grade of cast iron, according to Environmental Protection Department officials. For less healthy ones, the agency has invested more than $1 billion in the past five years — with an additional $1.4 billion budgeted over the next five years — for upgrades and replacements. New pipes will be made of a more durable, graphite-rich cast iron known as ductile iron. +

+
+ +
+
+
+
+
+
+ Image +
+
+
+
+
+ Matt Cruz snowboarded through Manhattan’s Lower East Side after a snowstorm in 2016 left the streets coated in slush and rock salt.CreditHiroko Masuike/The New York Times +
+
+
+
+
+
+ +

+ Of course, winter also poses problems aboveground. Most nonemergency repair and construction work involving concrete is halted because concrete and some types of dirt, used to fill in trenches, freeze in colder temperatures, said Ian Michaels, a spokesman for the city’s Department of Design and Construction. +

+

+ Digging by hand is also a challenge in frozen ground, so many excavations that are close to pipes and other utilities are put off, Mr. Michaels said. +

+
+ +
+
+
+

+ And asphalt is harder to obtain because it must be kept and transported at high temperatures, he added. +

+

+ In the extreme cold, city officials will not risk shutting down water mains for construction because spillage into the street could freeze, Mr. Michaels said. He added that stopping the water flow could freeze the private water-service connections that branch off the mains, he said. +

+

+ Even the basic task of locating utilities under the street can be complicated because infrastructure has been added piecemeal over the decades. +

+
+ +
+
+
+
+
+
+ Image +
+
+
+
+
+ A water main break in Manhattan in 2014. “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” said Tasos Georgelis of the city's Department of Environmental Protection.CreditÃngel Franco/The New York Times +
+
+
+
+
+
+ +

+ Street surfaces are affected by winter weather, too: Last year, the city filled 255,904 potholes. +

+

+ And should anyone forget that filling potholes, like snow removal, is a sacred staple of constituent services, transportation officials have compiled the number of potholes the city has filled — more than 1,786,300 — since Mayor Bill de Blasio took office in 2014. +

+

+ Potholes form when water and salt seep into cracks, freeze and expand, creating a larger crevice, said Joe Carbone, who works for the Transportation Department, where he is known as the pothole chief. +

+

+ Simply put, more freeze-thaw cycles result in more potholes, he said. Currently, the department has 25 crews repairing potholes. During peak pothole-repair season in early March, that number can expand to more than 60. +

+
+ +
+
+
+

+ Still, the department is continually resurfacing the city’s more than 6,000 miles of streets and 19,000 lane miles. Each year, agency officials said, it uses more than one million tons of asphalt to repave more than 1,300 lane-miles of street. +

+
+ +
+
+
+
+
+
+ Image +
+
+
+
+
+ Workers learning how to fix water main breaks at a training center in Queens.CreditChang W. Lee/The New York Times +
+
+
+
+
+
+ +

+ Of the 400 city laborers who work on water mains, many learn the finer points of leak repair at a training center in Queens, where underground pipes are made to spring leaks for repair drills. +

+

+ Workers from the Department of Environmental Protection recently gathered around a muddy hole as a co-worker, Nehemiah Dejesus, scrambled to apply a stainless-steel repair clamp around a cracked segment that was spewing water. +

+

+ “Don’t get nervous,” instructed Milton Velez, the agency’s district supervisor for Queens. +

+

+ “I’m not,” Mr. Dejesus said as he secured the clamp and stopped the leak. “It’s ‘Showtime at the Apollo.’” +

+
+ +
+
+
+
+ +
+
+ Why Are New York City’s Streets Always Under Construction? +
+
+
+ +
+
+
+
+
+
+ +
+
+
+
+
+
+
+

+ Corey Kilgannon is a Metro reporter covering news and human interest stories. His writes the Character Study column in the Sunday Metropolitan section. He was also part of the team that won the 2009 Pulitzer Prize for Breaking News. @coreykilgannon • Facebook +

+
+
+
+
+ A version of this article appears in print on , on Page A22 of the New York edition with the headline: Under the City’s Streets, A Battle Against Winter. Order Reprints | Today’s Paper | Subscribe +
+
+ +
+
+
+
+

+ Advertisement +

+
+
+
+
+
+
+
+
+ +
+
+
+
+
+ + +
+
+
+ + + + + + +
+ +
+ + diff --git a/resources/tests/readability/nytimes-4/source.html b/resources/tests/readability/nytimes-4/source.html new file mode 100644 index 0000000..81a2a68 --- /dev/null +++ b/resources/tests/readability/nytimes-4/source.html @@ -0,0 +1,3896 @@ + + + + + + As Debt Rises, the Government Will Soon Spend More on Interest Than on the Military - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+
+
+
+
+
+
+
+ +
+
+ +
Skip to contentSkip to site index +
+
+ Economy +
+
+ + +
+
+ +
+
+ +
+
+
+
+
+
+
+ Log In +
+
+ Today’s Paper +
+
+
+
+
+
+
+ + + + +
+
+
+
+
+ +
+

+ As Debt Rises, the Government Will Soon Spend More on Interest Than on the Military +

+
+

+ Tax cuts, spending increases and higher interest rates could make it harder to respond to future recessions and deal with other needs. +

+
+
+
+
+
+ Image +
+
+ Interest payments on the federal debt could surpass the Defense Department budget in 2023.CreditCreditJeon Heon-Kyun/EPA, via Shutterstock +
+
+
+
+
+
+
+
+

+ By Nelson D. Schwartz +

+
+
+
    +
  • + +
    • +
  • +
    + +
    +
    • +
+
+
+
+
+
+
+

+ The federal government could soon pay more in interest on its debt than it spends on the military, Medicaid or children’s programs. +

+

+ The run-up in borrowing costs is a one-two punch brought on by the need to finance a fast-growing budget deficit, worsened by tax cuts and steadily rising interest rates that will make the debt more expensive. +

+

+ With less money coming in and more going toward interest, political leaders will find it harder to address pressing needs like fixing crumbling roads and bridges or to make emergency moves like pulling the economy out of future recessions. +

+

+ Within a decade, more than $900 billion in interest payments will be due annually, easily outpacing spending on myriad other programs. Already the fastest-growing major government expense, the cost of interest is on track to hit $390 billion next year, nearly 50 percent more than in 2017, according to the Congressional Budget Office. +

+
+ +
+
+
+

+ “It’s very much something to worry about,” said C. Eugene Steuerle, a fellow at the Urban Institute and a co-founder of the Urban-Brookings Tax Policy Center in Washington. “Everything else is getting squeezed.” +

+

+ Gradually rising interest rates would have made borrowing more expensive even without additional debt. But the tax cuts passed late last year have created a deeper hole, with the deficit increasing faster than expected. A budget bill approved in February that raised spending by $300 billion over two years will add to the financial pressure. +

+

+ The deficit is expected to total nearly $1 trillion next year — the first time it has been that big since 2012, when the economy was still struggling to recover from the financial crisis and interest rates were near zero. +

+
+ +
+
+
+
+

+ Annual interest payments on the national debt are expected to triple over the next decade, according to the Congressional Budget Office. +

+
+
+ + + + +
+ +
+ +
+

+ $915 +

+

+ billion +

+
+
+

+ Interest payments on the national debt +

+
+
+

+ $900 +

+
+
+

+ billion +

+
+
+

+ 600 +

+
+
+

+ 300 +

+
+
+

+ $263 +

+

+ billion +

+
+
+

+ 0 +

+
+
+

+ ’17 +

+
+
+

+ ’18 +

+
+
+

+ ’19 +

+
+
+

+ ’20 +

+
+
+

+ ’21 +

+
+
+

+ ’22 +

+
+
+

+ ’23 +

+
+
+

+ ’24 +

+
+
+

+ ’25 +

+
+
+

+ ’26 +

+
+
+

+ ’27 +

+
+
+

+ ’28 +

+
+
+
+ +
+

+ Interest payments on the national debt +

+
+
+

+ $900 +

+
+
+

+ billion +

+
+
+

+ $915 +

+

+ billion +

+
+
+

+ 600 +

+
+
+

+ 300 +

+
+
+

+ $263 +

+

+ billion +

+
+
+

+ 0 +

+
+
+

+ ’17 +

+
+
+

+ ’18 +

+
+
+

+ ’19 +

+
+
+

+ ’20 +

+
+
+

+ ’21 +

+
+
+

+ ’22 +

+
+
+

+ ’23 +

+
+
+

+ ’24 +

+
+
+

+ ’25 +

+
+
+

+ ’26 +

+
+
+

+ ’27 +

+
+
+

+ ’28 +

+
+
+
+ + +
+
+

+ By The New York Times | Source: Congressional Budget Office +

+
+
+
+
+
+

+ Deficit hawks have gone silent, even proposing changes that would exacerbate the deficit. House Republicans introduced legislation this month that would make the tax cuts permanent. +

+

+ “The issue has just disappeared,” said Senator Mark Warner, a Virginia Democrat. “There’s collective amnesia.” +

+
+ +
+
+
+

+ The combination, say economists, marks a journey into mostly uncharted financial territory. +

+

+ In the past, government borrowing expanded during recessions and waned in recoveries. That countercyclical policy has been a part of the standard Keynesian toolbox to combat downturns since the Great Depression. +

+

+ The deficit is soaring now as the economy booms, meaning the stimulus is pro-cyclical. The risk is that the government would have less room to maneuver if the economy slows. +

+
+ +
+
+
+
+

+ Debt as a percentage of gross domestic product tends to increase during recessions and fall during recoveries. But the debt is increasing now, even as the economy is growing, because of tax cuts and spending increases. +

+
+
+ + + +
+ +
+ +
+

+ Public debt as a share of gross domestic product +

+
+
+

+ 100 +

+
+
+

+ % +

+
+
+

+ RECESSIONS +

+
+
+

+ PROJECTED +

+
+
+

+ 80 +

+
+
+

+ 60 +

+
+
+

+ 40 +

+
+
+

+ 20 +

+
+
+

+ 0 +

+
+
+

+ ’78 +

+
+
+

+ ’80 +

+
+
+

+ ’90 +

+
+
+

+ ’00 +

+
+
+

+ ’10 +

+
+
+

+ ’20 +

+
+
+

+ ’28 +

+
+
+
+ +
+

+ Public debt as a share of gross domestic product +

+
+
+

+ 100 +

+
+
+

+ % +

+
+
+

+ 80 +

+
+
+

+ 60 +

+
+
+

+ 40 +

+
+
+

+ 20 +

+
+
+

+ RECESSIONS +

+
+
+

+ PROJECTED +

+
+
+

+ 0 +

+
+
+

+ ’80 +

+
+
+

+ ’90 +

+
+
+

+ ’00 +

+
+
+

+ ’10 +

+
+
+

+ ’20 +

+
+
+

+ ’28 +

+
+
+
+ + +
+
+

+ By The New York Times | Source: Congressional Budget Office +

+
+
+
+
+
+

+ Aside from wartime or a deep downturn like the 1930s or 2008-9, “this sort of aggressive fiscal stimulus is unprecedented in U.S. history,” said Jeffrey Frankel, an economist at Harvard. +

+

+ Pouring gasoline on an already hot economy has resulted in faster growth — the economy expanded at an annualized rate of 4.2 percent in the second quarter. But Mr. Frankel warns that when the economy weakens, the government will find it more difficult to cut taxes or increase spending. +

+

+ Lawmakers might, in fact, feel compelled to cut spending as tax revenue falls, further depressing the economy. “There will eventually be another recession, and this increases the chances we will have to slam on the brakes when the car is already going too slowly,” Mr. Frankel said. +

+ +
+ +
+
+
+
+

+ Interest payments will make up 13 percent of the federal budget a decade from now, surpassing spending on Medicaid and defense. +

+
+
+ + + + +
+ +
+ +
+

+ Expenditures as a share of overall budget +

+
+
+

+ 15 +

+
+
+

+ % +

+
+
+

+ Net interest +

+
+
+

+ 13.0% +

+
+
+

+ Defense +

+
+
+

+ 10 +

+
+
+

+ Medicaid +

+
+
+

+ 6.6% +

+
+
+

+ 5 +

+
+
+

+ 0 +

+
+
+

+ ’17 +

+
+
+

+ ’18 +

+
+
+

+ ’19 +

+
+
+

+ ’20 +

+
+
+

+ ’21 +

+
+
+

+ ’22 +

+
+
+

+ ’23 +

+
+
+

+ ’24 +

+
+
+

+ ’25 +

+
+
+

+ ’26 +

+
+
+

+ ’27 +

+
+
+

+ ’28 +

+
+
+
+ +
+

+ Expenditures as a share of overall budget +

+
+
+

+ 15 +

+
+
+

+ % +

+
+
+

+ Net interest +

+
+
+

+ 13.0% +

+
+
+

+ Defense +

+
+
+

+ 10 +

+
+
+

+ Medicaid +

+
+
+

+ 6.6% +

+
+
+

+ 5 +

+
+
+

+ 0 +

+
+
+

+ ’17 +

+
+
+

+ ’18 +

+
+
+

+ ’19 +

+
+
+

+ ’20 +

+
+
+

+ ’21 +

+
+
+

+ ’22 +

+
+
+

+ ’23 +

+
+
+

+ ’24 +

+
+
+

+ ’25 +

+
+
+

+ ’26 +

+
+
+

+ ’27 +

+
+
+

+ ’28 +

+
+
+
+ + +
+
+

+ By The New York Times | Source: Congressional Budget Office +

+
+
+
+
+
+

+ Finding the money to pay investors who hold government debt will crimp other parts of the budget. In a decade, interest on the debt will eat up 13 percent of government spending, up from 6.6 percent in 2017. +

+

+ “By 2020, we will spend more on interest than we do on kids, including education, food stamps and aid to families,” said Marc Goldwein, senior policy director at the Committee for a Responsible Federal Budget, a research and advocacy organization. +

+
+ +
+
+
+

+ Interest costs already dwarf spending on many popular programs. For example, grants to students from low-income families for college total roughly $30 billion — about one-tenth of what the government will pay in interest this year. Interest payments will overtake Medicaid in 2020 and the Department of Defense budget in 2023. +

+

+ What’s more, the heavy burden of interest payments could make it harder for the government to repair aging infrastructure or take on other big new projects. +

+

+ Mr. Trump has called for spending $1 trillion on infrastructure, but Congress has not taken up that idea. +

+
+ +
+
+
+ More about the federal debt and the economy +
+
+
+ +
+
+ Trump Administration Mulls a Unilateral Tax Cut for the Rich +
+
+
+
+
+ +
+
+ How the Trump Tax Cut Is Helping to Push the Federal Deficit to $1 Trillion +
+
+
+
+
+
+
+
+ +

+ Until recently, ultralow interest rates, set by the Federal Reserve to support the economy, allowed lawmakers to borrow without fretting too much about the cost of that debt. +

+

+ But as the economy has strengthened, the Fed has gradually raised rates, starting in December 2015. The central bank is expected to push rates up again on Wednesday, and more increases are in store. +

+

+ “When rates went down to record lows, it allowed the government to take on more debt without paying more interest,” Mr. Goldwein said. “That party is ending.” +

+
+ +
+
+
+
+

+ After bottoming out below 2 percent, the yield on the 10-year Treasury note has climbed to over 3 percent recently. +

+
+
+ + + + +
+ +
+ +
+

+ Yield on 10-year Treasury note +

+
+
+

+ 8 +

+
+
+

+ % +

+
+
+

+ MONTHLY +

+
+
+

+ RECESSIONS +

+
+
+

+ 6 +

+
+
+

+ 4 +

+
+
+

+ 2 +

+
+
+

+ 0 +

+
+
+

+ ’98 +

+
+
+

+ ’00 +

+
+
+

+ ’02 +

+
+
+

+ ’04 +

+
+
+

+ ’06 +

+
+
+

+ ’08 +

+
+
+

+ ’10 +

+
+
+

+ ’12 +

+
+
+

+ ’14 +

+
+
+

+ ’16 +

+
+
+

+ ’18 +

+
+
+
+ +
+

+ Yield on 10-year Treasury note +

+
+
+

+ 8 +

+
+
+

+ % +

+
+
+

+ MONTHLY +

+
+
+

+ RECESSIONS +

+
+
+

+ 6 +

+
+
+

+ 4 +

+
+
+

+ 2 +

+
+
+

+ 0 +

+
+
+

+ ’98 +

+
+
+

+ ’02 +

+
+
+

+ ’06 +

+
+
+

+ ’10 +

+
+
+

+ ’14 +

+
+
+

+ ’18 +

+
+
+
+ + +
+
+

+ By The New York Times | Source: Reuters +

+
+
+
+
+
+

+ Since the beginning of the year, the yield on the 10-year Treasury note has risen by more than half a percentage point, to 3.1 percent. The Congressional Budget Office estimates that the yield will climb to 4.2 percent in 2021. Given that the total public debt of the United States stands at nearly $16 trillion, even a small uptick in rates can cost the government billions. +

+
+ +
+
+
+

+ There’s no guarantee that these forecasts will prove accurate. If the economy weakens, rates might fall or rise only slightly, reducing interest payments. But rates could also overshoot the budget office forecast. +

+

+ Some members of Congress want to set the stage for even more red ink. Republicans in the House want to make last year’s tax cuts permanent, instead of letting some of them expire at the end of 2025. That would reduce federal revenue by an additional $631 billion over 10 years, according to the Tax Policy Center. +

+ +
+ +
+
+
+
+

+ The dollar has strengthened even as government borrowing has increased. +

+
+
+ + + + +
+ +
+ +
+

+ U.S. Dollar index +

+
+
+

+ 140 +

+
+
+

+ MONTHLY +

+
+
+

+ RECESSIONS +

+
+
+

+ 120 +

+
+
+

+ 100 +

+
+
+

+ 80 +

+
+
+

+ 60 +

+
+
+

+ ’98 +

+
+
+

+ ’00 +

+
+
+

+ ’02 +

+
+
+

+ ’04 +

+
+
+

+ ’06 +

+
+
+

+ ’08 +

+
+
+

+ ’10 +

+
+
+

+ ’12 +

+
+
+

+ ’14 +

+
+
+

+ ’16 +

+
+
+

+ ’18 +

+
+
+
+ +
+

+ U.S. Dollar index +

+
+
+

+ 140 +

+
+
+

+ MONTHLY +

+
+
+

+ RECESSIONS +

+
+
+

+ 120 +

+
+
+

+ 100 +

+
+
+

+ 80 +

+
+
+

+ 60 +

+
+
+

+ ’98 +

+
+
+

+ ’02 +

+
+
+

+ ’06 +

+
+
+

+ ’10 +

+
+
+

+ ’14 +

+
+
+

+ ’18 +

+
+
+
+ + +
+
+

+ By The New York Times | Source: Reuters +

+
+
+
+
+
+

+ Deficit hawks have warned for years that a day of reckoning is coming, exposing the United States to the kind of economic crisis that overtook profligate borrowers in the past like Greece or Argentina. +

+

+ But most experts say that isn’t likely because the dollar is the world’s reserve currency. As a result, the United States still has plenty of borrowing capacity left because the Fed can print money with fewer consequences than other central banks. +

+

+ And interest rates plunged over the last decade, even as the government turned to the market for trillions each year after the recession. That’s because Treasury bonds are still the favored port of international investors in any economic storm. +

+

+ “We exported a financial crisis a decade ago, and the world responded by sending us money,” said William G. Gale, a senior fellow at the Brookings Institution. +

+

+ But that privileged position has allowed politicians in both parties to avoid politically painful steps like cutting spending or raising taxes. +

+
+ +
+
+
+

+ That doesn’t mean rapidly rising interest costs and a bigger deficit won’t eventually catch up with us. +

+

+ Charles Schultze, chairman of the Council of Economic Advisers in the Carter administration, once summed up the danger of deficits with a metaphor. “It’s not so much a question of the wolf at the door, but termites in the woodwork.” +

+ +

+ Rather than simply splitting along party lines, lawmakers’ attitudes toward the deficit also depend on which party is in power. Republicans pilloried the Obama administration for proposing a large stimulus in the depths of the recession in 2009 and complained about the deficit for years. +

+

+ In 2013, Senator Mitch McConnell of Kentucky called the debt and deficit “the transcendent issue of our era.” By 2017, as Senate majority leader, he quickly shepherded the tax cut through Congress. +

+

+ Senator James Lankford, an Oklahoma Republican who warned of the deficit’s dangers in the past, nevertheless played down that threat on the Senate floor as the tax billed neared passage. +

+

+ “I understand it’s a risk, but I think it’s an appropriate risk to be able to say let’s allow Americans to keep more of their own money to invest in this economy,” he said. +

+

+ He also claimed the tax cuts would pay for themselves even as the Congressional Budget Office estimated that they would add $250 billion to the deficit on average from 2019 to 2024. +

+
+ +
+
+
+

+ In an interview, Mr. Lankford insisted that the jury was still out on whether the tax cuts would generate additional revenue, citing the strong economic growth recently. +

+

+ While the Republican about-face has been much more striking, Democrats have adjusted their position, too. +

+

+ Mr. Warner, the Virginia Democrat, called last year’s tax bill “the worst piece of legislation we have passed since I arrived in the Senate.” In 2009, however, when Congress passed an $800 billion stimulus bill backed by the Obama administration, he called it “a responsible mix of tax cuts and investments that will create jobs.” +

+

+ The difference, Mr. Warner said, was that the economy was near the precipice then. +

+

+ “There was virtual unanimity among economists that we needed a stimulus,” he said. “But a $2 trillion tax cut at the end of a business cycle with borrowed money won’t end well.” +

+
+ +
+
+
+
+
+
+
+

+ Nelson D. Schwartz has covered economics since 2012. Previously, he wrote about Wall Street and banking, and also served as European economic correspondent in Paris. He joined The Times in 2007 as a feature writer for the Sunday Business section. @NelsonSchwartz +

+
+
+
+
+ A version of this article appears in print on , on Page A1 of the New York edition with the headline: What May Soon Exceed Cost of U.S. Military? Interest on U.S. Debt . Order Reprints | Today’s Paper | Subscribe +
+
+
+ +
+
+
+
+

+ Advertisement +

+
+
+
+
+
+
+
+
+ +
+
+
+
+
+ + +
+
+
+ + + + + + +
+ +
+ + diff --git a/resources/tests/readability/nytimes-5/source.html b/resources/tests/readability/nytimes-5/source.html new file mode 100644 index 0000000..91a431f --- /dev/null +++ b/resources/tests/readability/nytimes-5/source.html @@ -0,0 +1,1540 @@ + + + + + + The New York Times en Español - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+ +
+
+ +
Skip to contentSkip to site index +
+
+ + +
+
+ +
+
+
+ +
+
+
+ +
+
+ + +
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+

+ Highlights +

+
    +
  1. + +
    2. +
  2. +
    +
    + Photo +
    + CreditEllen Surrey +
    +
    +
    +
    + +
    +

    + Ensayo invitado +

    +

    + El día que renuncié a los Beatles +

    +

    + Estaba obsesionado con el cuarteto de Liverpool: celebraba sus cumpleaños, leía todo sobre la banda y memoricé todas sus canciones. Cuando me desintoxiqué de ellos descubrí un nuevo mundo musical. +

    +

    + Por Josh Max +

    +
    +
    +
    3. +
  3. +
      +
    1. +
      +
      + Photo +
      + CreditErik Carter +
      +
      +
      +

      + On Tech +

      +

      + ¿Qué tienen que ver los videojuegos con el metaverso? +

      +
      + +
      +

      + Las compañías tecnológicas creen que los videojuegos son el camino para avanzar más rápido hacia un internet inmersivo. La adquisición de Activision Blizzard por Microsoft es una muestra de esta tendencia. +

      +

      + Por Shira Ovide +

      +
      +
      +
      2. +
    2. +
      +
      + Photo +
      + CreditTed + Chelsea Cavanaugh para The New York Times +
      +
      +
      +

      + Skin Deep +

      +

      + Bebidas con beneficios: ¿de verdad funcionan? +

      +
      + +
      +

      + Hay un nuevo mercado de bebidas que prometen beneficios como la salud intestinal, una mente relajada y piel más brillante. El problema es que ninguno de esos efectos ha sido respaldado científicamente. +

      +

      + Por Rachel Strugatz +

      +
      +
      +
      3. +
    +
    4. +
+
+
+
+
    +
  1. +
    +

    + Charles M. Blow +

    +
    + Photo +
    + CreditNitashia Johnson para The New York Times +
    +
    +
    +

    + La muerte cambió mi vida +

    +

    + Por fuera parecía tener éxito, pero por dentro sentía que me ahogaba. Cuando murió mi hermano, tuve que replantearme todo. +

    +

    + By Charles M. Blow +

    +
    +
    +
    2. +
  2. +
    +

    + EDUCACIÓN FÍSICA +

    +
    + Photo +
    + CreditPiotr Redlinski for The New York Times +
    +
    +
    +

    + Así es como el ejercicio puede controlar nuestra ansiedad +

    +

    + Los corredores de esquí de fondo son menos propensos a desarrollar trastornos de ansiedad. La buena noticia es que las actividades aeróbicas menos intensas pueden proporcionar beneficios similares. +

    +

    + By Gretchen Reynolds +

    +
    +
    +
    3. +
  3. + +
    4. +
  4. +
    +

    + EL Desafío Come bien +

    +
    + Photo +
    + CreditFotoilustración de Andrew B. Myers para The New York Times +
    +
    +
    +

    + Siete opciones para alimentar a tu cerebro +

    +

    + Para el desafío de esta semana, prueba algunos alimentos que se han relacionado con la mejora del estado de ánimo. +

    +

    + By Tara Parker-Pope +

    +
    +
    +
    5. +
  5. + +
    6. +
+
+
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+
+
+
+
+

+ Opinión + +

Más en Opinión › +
+
    +
  1. + +
    2. +
  2. + +
    3. +
  3. + +
    4. +
  4. +
      +
    1. + +
      2. +
    +
    5. +
+
+
+
+

+ Especial + +

Más en Especial › +
+
    +
  1. +
    +
    + Photo  +
    + CreditPhoto Illustration by Andrew B. Myers for The New York Times +
    +
    +
    +

    + El desafío Come bien +

    +

    + Actualiza tus hábitos alimenticios este año, sin necesidad de hacer dieta. +

    +

    + By Tara Parker-Pope +

    +
    +
    +
    2. +
  2. + +
    3. +
  3. +
    +
    + Photo +
    + CreditFotoilustraciones de Andrew B. Myers para The New York Times +
    +
    +
    +

    + ¿Galletas? ¿Papitas? ¿Pizza? Aprende a dominar tus antojos +

    +

    + Para el desafío Come bien, las técnicas de atención plena, como “surfear los impulsos”, pueden ayudar a frenar el comer demasiado sin prohibir los alimentos favoritos. +

    +

    + By Tara Parker-Pope +

    +
    +
    +
    4. +
  4. +
    +
    + Photo +
    + CreditFotoilustración de Andrew B. Myers para The New York Times +
    +
    +
    +

    + Adiós a la dieta perpetua, hola a la alimentación intuitiva +

    +

    + El desafío Come bien de hoy te ayudará a evitar el efecto “al diablo” de las dietas y a sintonizar con tu cuerpo. +

    +

    + By Tara Parker-Pope +

    +
    +
    +
    5. +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+
+
+
+
+

+ El brote de Coronavirus + +

Más en El brote de Coronavirus › +
+
    +
  1. + +
    2. +
  2. +
    +
    + PhotoLargas filas para hacerse pruebas de coronavirus en Jonesboro, Georgia, este mes. La variante ómicron se identificó a finales de noviembre, por lo que es demasiado pronto para decir cuánto tiempo pueden persistir los síntomas. +
    + CreditDustin Chambers para The New York Times +
    +
    +
    +

    + ¿Ómicron puede causar covid prolongada? +

    +

    + Los científicos dicen que aún es muy pronto para saber si quienes se infectan con la nueva variante tendrán síntomas persistentes. Una infección leve no necesariamente es señal de que hay menos riesgo. +

    +

    + By Pam Belluck +

    +
    +
    +
    3. +
  3. +
    +
    + PhotoA 3-D plaster model of a coronavirus spike protein in the office of Dr. Barney Graham of the Vaccine Research Center of the National Institutes of Health. +
    + CreditJohnathon Kelso for The New York Times +
    +
    +
    +

    + Cómo se hicieron las vacunas ARNm: con lentitud y accidentes afortunados +

    +

    + Las sorprendentes vacunas para la covid fabricadas por Pfizer-BioNTech y Moderna utilizaron descubrimientos sepultados hace mucho con la esperanza de acabar con epidemias del pasado. +

    +

    + By Gina Kolata y Benjamin Mueller +

    +
    +
    +
    4. +
  4. +
    +
    + PhotoUn centro de pruebas de COVID-19 realizadas con saliva en la Universidad de Minnesota, en Mineápolis +
    + CreditJenn Ackerman para The New York Times +
    +
    +
    +

    + ¿Y si la saliva fuera mejor opción para detectar el coronavirus? +

    +

    + Al propagarse la variante ómicron algunos expertos defienden las pruebas que emplean muestras de saliva por su capacidad de detectar la infección antes que el hisopado nasal. +

    +

    + By Emily Anthes +

    +
    +
    +
    5. +
  5. + +
    6. +
+
+
+
+

+ Estados Unidos + +

Más en Estados Unidos › +
+
    +
  1. + +
    2. +
  2. + +
    3. +
  3. + +
    4. +
  4. +
    +
    + PhotoEl expresidente Donald Trump el año pasado. Liz Cheney, representante republicana por Wyoming, ha calificado su lenta respuesta al atentado del 6 de enero como una negligencia en el cumplimiento del deber. +
    + CreditCooper Neill para The New York Times +
    +
    +
    +

    + Todo lo que sabemos sobre la investigación del asalto al Capitolio de EE. UU. +

    +

    + La comisión de la Cámara de Representantes no está facultada para presentar cargos penales, pero puede darle al Departamento de Justicia las pruebas de cualquier infracción que descubra en su investigación. +

    +

    + By Luke Broadwater +

    +
    +
    +
    5. +
  5. +
    +
    + PhotoDurante una década, Holmes engañó a inversionistas inteligentes, a cientos de empleados inteligentes, a un comité de figuras ilustres y a los medios de comunicación que estaban ansiosos por ungir a una nueva estrella. +
    + CreditJenny Hueston +
    +
    +
    +

    + El auge y la caída de Elizabeth Holmes +

    +

    + El caso de la fundadora de Theranos podría cambiar el estatus de culto que tienen algunos emprendedores tecnológicos a los que no se les cuestionan sus ambiciosos proyectos ni se les exige que rindan cuentas. +

    +

    + By David Streitfeld +

    +
    +
    +
    6. +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+
+
+
+
+
+ +
+
+
    +
    +
  1. +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +

    + Elogio de la pereza +

    +

    + Consejo de un rabino, derrame petrolero en Perú, una infección de ómicron en gráficos y más para estar al día. +

    +
    +

    + Por Elda Cantú +

    +
    +
    +
    +
    +
    2. +
  2. +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +

    + Un ejecutivo de Nike busca el perdón de una familia por un asesinato que cometió en 1965 +

    +

    + Barbara Mack abrazó a Larry Miller después de que este se disculpara por haber matado a su hermano, Edward David White. Pero si fuera 30 años más joven, le dijo, “habría saltado por encima de la mesa para golpearte”. +

    +
    +

    + Por Jeré Longman y Kevin Draper +

    +
    +
    + Read in English +
    +
    +
    +
    +
    3. +
  3. +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +

    + EL CUADERNO DE LA CRÍTICA +

    +

    + Los Ángeles, gloriosa capital del sushi +

    +

    + Pocos lugares fuera de Japón pueden igualar la variedad, la maestría y la creatividad que se sirve en las innumerables barras de sushi de la ciudad.` +

    +
    +

    + Por Tejal Rao +

    +
    +
    + Read in English +
    +
    +
    +
    +
    4. +
  4. +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +

    + ¿Quién entregó a la familia de Ana Frank? Hay una nueva teoría. Es controversial +

    +

    + Un nuevo libro identifica a un sospechoso de delatar a la familia Frank a la administración nazi. Ahora que expertos e historiadores han tenido tiempo de revisar la investigación, muchos dudan de los métodos y la conclusión. +

    +
    +

    + Por Nina Siegal +

    +
    +
    + Read in English +
    +
    +
    +
    +
    5. +
  5. +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +

    + Cuaderno de la crítica +

    +

    + Cinco estupendas opciones de sushi en Los Ángeles +

    +

    + En la ciudad abundan los restaurantes donde se despliega estilo y destreza. Estos son algunos de los favoritos de la crítica Tejal Rao. +

    +
    +

    + Por Tejal Rao +

    +
    +
    + Read in English +
    +
    +
    +
    +
    6. +
  6. +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +

    + Modern Love +

    +

    + Tuvimos un divorcio amable y seguimos viviendo en la misma casa +

    +

    + Cuando el final de un matrimonio significa vivir en pisos separados de la misma residencia. +

    +
    +

    + Por Jordana Jacobs +

    +
    +
    + Read in English +
    +
    +
    +
    +
    7. +
  7. +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +

    + Defender Chernóbil en una invasión no vale la pena, dicen algunos ucranianos +

    +

    + Ucrania ha iniciado una estrategia defensiva para la Zona de Exclusión de Chernóbil, uno de los lugares más radiactivos de la Tierra, que se encuentra en el camino más corto entre Rusia y la capital de Ucrania, Kiev. +

    +
    +

    + Por Andrew E. Kramer y Tyler Hicks +

    +
    +
    + Read in English +
    +
    +
    +
    +
    8. +
  8. +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +

    + Tech Fix +

    +

    + ¿Por qué ha causado tanto alboroto el metaverso? +

    +

    + Microsoft dijo que pagó 68.700 millones de dólares por una empresa de videojuegos para fortalecer su posición en el metaverso. Veamos qué significa esto en realidad. +

    +
    +

    + Por Brian X. Chen +

    +
    +
    + Read in English +
    +
    +
    +
    +
    9. +
  9. +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +

    + Zara Rutherford, la mujer más joven en darle la vuelta al mundo +

    +

    + Cuando la adolescente aterrizó el jueves, se convirtió en la mujer más joven en circunnavegar el mundo en solitario. “Me alegra poder estar en el mismo lugar por unos meses”, dijo. +

    +
    +

    + Por Mike Ives +

    +
    +
    + Read in English +
    +
    +
    +
    +
    10. +
  10. +
    +
    + +
    +
    +
    + +
    +
    +
    +
    +

    + ‘El petróleo va a estar meses en el mar’: un derrame de crudo, provocado por el tsunami, arrasa la costa peruana +

    +

    + El derrame, que inicialmente se dijo que era de siete galones, resultó ser de 6000 barriles según indicó el Ministerio del Ambiente. El gobierno peruano ha comenzado a pedir ayuda internacional. +

    +
    +

    + Por Mitra Taj +

    +
    +
    + Read in English +
    +
    +
    +
    +
    11. +
+
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+ + + + + + + + + + diff --git a/src/full_text_parser/mod.rs b/src/full_text_parser/mod.rs index e8a112d..0e9f6c5 100644 --- a/src/full_text_parser/mod.rs +++ b/src/full_text_parser/mod.rs @@ -415,8 +415,16 @@ impl FullTextParser { } fn fix_lazy_images(context: &Context, doc: &Document) -> Result<(), FullTextParserError> { - let node_vec = Util::evaluate_xpath(context, "//img|picture|figure", false)?; - for mut node in node_vec { + let mut img_nodes = Util::evaluate_xpath(context, "//img", false)?; + let pic_nodes = Util::evaluate_xpath(context, "//picture", false)?; + let fig_nodes = Util::evaluate_xpath(context, "//figure", false)?; + + img_nodes.extend(pic_nodes); + img_nodes.extend(fig_nodes); + + for mut node in img_nodes { + let tag_name = node.get_name().to_uppercase(); + // In some sites (e.g. Kotaku), they put 1px square image as base64 data uri in the src attribute. // So, here we check if the data uri is too short, just might as well remove it. if let Some(src) = node.get_attribute("src") { @@ -481,8 +489,6 @@ impl FullTextParser { } if let Some(copy_to) = copy_to { - let tag_name = node.get_name().to_uppercase(); - //if this is an img or picture, set the attribute directly if tag_name == "IMG" || tag_name == "PICTURE" { _ = node.set_attribute(copy_to, &val); @@ -956,6 +962,7 @@ impl FullTextParser { } pub(crate) fn post_process_page(node: &mut Node) -> Result<(), FullTextParserError> { + Util::clean_headers(node); Util::clean_conditionally(node, "fieldset"); Util::clean_conditionally(node, "table"); Util::clean_conditionally(node, "ul"); diff --git a/src/full_text_parser/readability/tests.rs b/src/full_text_parser/readability/tests.rs index da9746a..4f9b2a5 100644 --- a/src/full_text_parser/readability/tests.rs +++ b/src/full_text_parser/readability/tests.rs @@ -372,6 +372,21 @@ async fn normalize_spaces() { run_test("normalize-spaces").await } +#[tokio::test] +async fn nytimes_1() { + run_test("nytimes-1").await +} + +#[tokio::test] +async fn nytimes_2() { + run_test("nytimes-2").await +} + +// #[tokio::test] +// async fn nytimes_3() { +// run_test("nytimes-3").await +// } + #[tokio::test] async fn webmd_1() { run_test("webmd-1").await diff --git a/src/util.rs b/src/util.rs index 2063998..b58e099 100644 --- a/src/util.rs +++ b/src/util.rs @@ -515,6 +515,18 @@ impl Util { } } + pub fn clean_headers(root: &mut Node) { + let mut nodes = Util::get_elements_by_tag_name(root, "h1"); + nodes.append(&mut Util::get_elements_by_tag_name(root, "h2")); + + for mut node in nodes.into_iter().rev() { + if Util::get_class_weight(&node) < 0 { + log::debug!("Removing header with low class weight: {} {}", node.get_name(), node.get_attribute("class").unwrap_or_default()); + node.unlink(); + } + } + } + // Clean an element of all tags of type "tag" if they look fishy. // "Fishy" is an algorithm based on content length, classnames, link density, number of images & embeds, etc. pub fn clean_conditionally(root: &mut Node, tag: &str) {