need reqwest streams

check for empty http response and parsed documents without root element

See merge request news-flash/article_scraper!11

.gitignore

@@ -1,4 +1,3 @@
 Cargo.lock
 .vscode/
-target/
-test_output/
+target/

.gitlab-ci.yml

@@ -1,31 +1,17 @@
-image: rust:latest
-
 stages:
   - build
-  - fmt
-  - lint
+  
 
 run-build:
   stage: build
-  image: rust:latest
-  script:
-  - rustc --version && cargo --version
-  - cargo build --release --jobs 1
-
-run-fmt:
-  stage: fmt
-  image: rust:latest
+  image: rust:1.86
   before_script:
   - rustup component add rustfmt
-  script:
-  - rustc --version && cargo --version
-  - cargo fmt -- --check
-
-run-lint:
-  stage: lint
-  image: rust:latest
-  before_script:
   - rustup component add clippy
+  - export LIBXML2=$(pkg-config libxml-2.0 --variable=libdir)/libxml2.so
   script:
   - rustc --version && cargo --version
+  - echo $LIBXML2
+  - cargo fmt -- --check
   - cargo clippy --all-targets --all-features -- -D warnings
+  - cargo build --release

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "article_scraper/ftr-site-config"]
+	path = article_scraper/ftr-site-config
+	url = https://github.com/fivefilters/ftr-site-config.git
+	branch = master

Cargo.toml

@@ -1,22 +1,10 @@
-[package]
-name = "article_scraper"
-version = "1.1.4"
-authors = ["Jan Lukas Gernert <jangernert@gmail.com>"]
-edition = "2018"
-license = "GPL-3.0-or-later"
-description = "Scrap article contents from the web. Powered by fivefilters full text feed configurations."
-repository = "https://gitlab.com/news-flash/article_scraper"
+[workspace]
+members = ["article_scraper", "article_scraper_cli"]
+resolver = "2"
 
-[dependencies]
-failure = "0.1"
-libxml = "0.2"
-reqwest = { version = "0.10", features = ["json", "native-tls"] }
-tokio = { version = "=0.2", features = ["macros"] }
-url = "2.1"
-regex = "1.3"
-encoding_rs = "0.8"
-chrono = "0.4"
-base64 = "0.12"
-image = "0.23"
-log = "0.4"
-parking_lot = "0.10"
+[workspace.package]
+version = "2.1.2"
+authors = ["Jan Lukas Gernert <jangernert@gmail.com>"]
+edition = "2021"
+license = "GPL-3.0-or-later"
+repository = "https://gitlab.com/news-flash/article_scraper"

Readme.md

@@ -0,0 +1,52 @@
+# article scraper
+
+The `article_scraper` crate provides a simple way to extract meaningful content from the web.
+It contains two ways of locating the desired content
+
+## 1. Rust implementation of [Full-Text RSS](https://www.fivefilters.org/full-text-rss/)
+
+This makes use of website specific extraction rules. Which has the advantage of fast & accurate results.
+The disadvantages however are: the config needs to be updated as the website changes and a new extraction rule is needed for every website.
+
+A central repository of extraction rules and information about writing your own rules can be found here: [ftr-site-config](https://github.com/fivefilters/ftr-site-config).
+Please consider contributing new rules or updates to it.
+
+`article_scraper` embeds all the rules in the ftr-site-config repository for convenience. Custom and updated rules can be loaded from a `user_configs` path.
+
+## 2. Mozilla Readability
+
+In case the ftr-config based extraction fails the [mozilla Readability](https://github.com/mozilla/readability) algorithm will be used as a fall-back.
+This re-implementation tries to mimic the original as closely as possible.
+
+# Example
+
+```
+use article_scraper::ArticleScraper;
+use url::Url;
+use reqwest::Client;
+
+let scraper = ArticleScraper::new(None);
+let url = Url::parse("https://www.nytimes.com/interactive/2023/04/21/science/parrots-video-chat-facetime.html");
+let client = Client::new();
+let article = scraper.parse(&url, false, &client, None).await.unwrap();
+```
+
+# CLI
+
+Various features of this crate can be used via [`article_scraper_cli`](./article_scraper_cli/).
+
+```
+Usage: article_scraper_cli [OPTIONS] <COMMAND>
+
+Commands:
+  all          Use the complete pipeline
+  readability  Only use the Readability parser
+  ftr          Only use (a subset of) the Ftr parser
+  help         Print this message or the help of the given subcommand(s)
+
+Options:
+  -d, --debug          Turn debug logging on
+  -o, --output <FILE>  Destination of resulting HTML file
+  -h, --help           Print help
+  -V, --version        Print version
+```

article_scraper/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "article_scraper"
+description = "Scrap article contents from the web. Powered by fivefilters full text feed configurations & mozilla readability."
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+readme = "../Readme.md"
+keywords = ["article", "scrape", "full-text", "readability"]
+exclude = ["resources/tests"]
+
+[dependencies]
+thiserror = "2.0"
+libxml = "0.3"
+reqwest = { version = "0.12", features = ["stream"] }
+tokio = { version = "1", features = ["macros", "fs", "io-util"] }
+url = "2.5"
+regex = "1.11"
+encoding_rs = "0.8"
+chrono = "0.4"
+base64 = "0.22"
+image = "0.25"
+log = "0.4"
+rust-embed="8.6"
+once_cell = "1.20"
+escaper = "0.1"
+futures = "0.3"
+unic-emoji-char = "0.9"
+
+[dev-dependencies]
+env_logger = "0.11"

article_scraper/ftr-site-config

@@ -0,0 +1 @@
+Subproject commit 69aa220193d99427d3822fabccdfaeede56cd532

article_scraper/resources/tests/ftr/golem/expected.html

@@ -0,0 +1,60 @@
+<article><article>
+
+<header>
+
+
+<p>
+Was als Aprilscherz begann, darf nach Meinung vieler Unterstützer nicht aus dem <a href="https://www.golem.de/specials/internet/" target="_blank">Internet</a> verschwinden: Der HTTP Statuscode 418 "I am a Teapot" bleibt - und könnte sogar zum <a href="https://www.golem.de/specials/ietf/" target="_blank">IETF</a>-Standard werden.
+</p>
+
+
+
+
+
+<figure>
+<img src="https://www.golem.de/1708/129460-144318-i_rc.jpg" title="HTTP Error 418: Fehlercode &quot;Ich bin eine Teekanne&quot; darf bleiben" alt="Teekannen sind Programmierern offenbar wichtig!"/>
+<figcaption>
+<span>Teekannen sind Programmierern offenbar wichtig!</span>
+<span>(Bild: <a href="https://upload.wikimedia.org/wikipedia/commons/a/a5/%28%28%28%D8%A7%D8%AA%D8%A7%D9%82%DA%A9_%D9%82%D9%88%D8%B1%DB%8C_%D9%BE%D8%A7%D8%B1%DA%A9_%D9%85%D9%84%D8%AA_%D8%B3%D9%84%D9%85%D8%A7%D8%B3%29%29%29_-_panoramio.jpg" rel="nofollow" target="_blank">auoob farabi</a>/<a href="http://creativecommons.org/licenses/by-sa/3.0/" rel="nofollow" target="_blank">CC-BY-SA 3.0</a>)</span>
+</figcaption>
+</figure>
+</header>
+
+
+
+
+
+<div>
+
+
+
+<p id="gpar1">Der im Jahr 1998 als Aprilscherz eingeführte HTTP-Stauscode 418 - <em>"Ich bin eine Teekanne"</em> wird vorerst nicht aus populären Projekten verschwinden. Der Entwurf für den Code wurde von Larry Masinter von der Internet Engineering Taskforce geschrieben, eine Initiative des australischen Programmierers Mark Nottingham zur Abschaffung wurde jetzt abgelehnt. Der Code gibt folgende Nachricht aus: <em>"418 Ich bin eine Teekanne. Jeder Versuch, mit mir Kaffee zu brauen, sollte mit dem Statuscode 418 'Ich bin eine Teekanne' beantwortet werden."</em></p>
+
+
+
+
+
+
+<p id="gpar2">Ursprünglich als Scherz über <em>"viele schlechte HTTP-Extensions"</em> gedacht, entwickelte der Statuscode ein Eigenleben und wurde in mehreren Projekten implementiert, etwa in Googles Programmiersprache Go, in Node.js und Asp.Net. Nottingham hatte in mehreren Repositories <a href="https://github.com/golang/go/issues/21326" target="_blank">vorgeschlagen</a>, den Code zu entfernen, weil die Nummer möglicherweise später für andere, wichtigere Zwecke gebraucht werden könnte. <em>"Bitte zieht in Erwägung, 418 aus Go HTTP zu entfernen, weil er kein richtiger HTTP-Statuscode ist (nicht einmal nach der eigenen Definition). Ich weiß, dass es lustig ist und dass einige Menschen aus Spaß eigene Implementierungen entwickelt haben, aber das Kernprotokoll sollte nicht verschmutzt werden."</em></p>
+
+<h3>418 muss bleiben</h3>
+
+<p id="gpar3">Doch Nottinghams Vorschlag wurde von der Community nicht besonders herzlich aufgenommen, <a href="https://www.gizmodo.com.au/2017/08/aussie-programmers-attempt-to-remove-joke-im-a-teapot-web-error-ends-up-making-it-official-instead/" target="_blank">wie Gizmodo berichtet</a>. Schnell wurde unter dem Motto <em>"Wir sind die Teekannen"</em> die Webseite <a href="http://save418.com/" target="_blank">Save418.com</a> ins Leben gerufen, um die erhebliche Notwendigkeit des Codes zu begründen. Denn der Code 418 sei <em>"eine Erinnerung daran, dass die dem Computer zugrundeliegenden Prozesse noch immer von Menschen gemacht werden."</em> Es wäre daher sehr schade, <em>"418 gehen zu sehen."</em></p>
+
+
+<p id="gpar4">Auch der Initiator des Löschantrages, Mark Nottingham, war von der Argumentation offenbar überzeugt. Denn Mittlerweile hat er <a href="https://tools.ietf.org/html/draft-nottingham-thanks-larry-00" target="_blank">bei der IETF einen Antrag eingereicht</a>, den Statuscode zu reservieren, anstatt ihn zu entfernen. Und so können Teekannen wohl auch in Zukunft ganz ohne Internet-of-Things-Technik ein wichtiger Teil des Internets bleiben.</p>
+
+
+
+
+</div>
+
+
+
+
+
+
+
+
+
+                </article></article>

article_scraper/resources/tests/ftr/hardwareluxx/expected.html

@@ -0,0 +1,24 @@
+<article><article itemscope="" itemtype="http://schema.org/Article">
+
+<div data-type="dpara" data-speed="8" id="dheader">
+<meta itemprop="datePublished" content="2023-04-25 16:14:21"/>
+<meta itemprop="dateModified" content="2023-04-25 16:14:23"/>
+<meta itemprop="mainEntityOfPage" content="https://www.hardwareluxx.de/index.php/news/software/spiele/60882-half-life-mit-ray-tracing-mod-gibt-dem-25-jahr-alten-shooter-neuen-glanz.html"/>
+ <div>
+<span>NEWS</span><br/>
+
+<p>Half-Life mit Raytracing</p>
+<p itemprop="name headline">
+Mod gibt dem 25 Jahr alten Shooter neuen Glanz </p>
+
+<div>
+<img src="https://www.hardwareluxx.de/images/avatare/HWL_avatar_default.jpg" alt="Portrait des Authors" width="87" height="87"/>
+</div>
+</div>
+<imageobject><a href="https://www.hardwareluxx.de/images/cdn02/uploads/2023/Apr/prompt_tech_bo/valve_half-life_ray-tracing001_1360px.jpg"><img width="1360" src="https://www.hardwareluxx.de/images/cdn02/uploads/2023/Apr/prompt_tech_bo/valve_half-life_ray-tracing001_1360px.jpg"/></a></imageobject></div>
+
+<div id="tocbar">Mod gibt dem 25 Jahr alten Shooter neuen Glanz</div>
+<div id="maincontent" itemprop="articleBody">
+<p>25 Jahre ist es nun her, dass Physiker Gordon Freeman und seine Brechstange zu den Helden eines bis heute legendären Ego-Shooters wurden. Mit Half-Life hat Valve eines der erfolgreichsten Spiele aller Zeiten produziert. Kein Wunder also, dass es bis heute eine aktive Fan-Base rund um den Titel gibt. </p> <p>Während immer wieder Gerüchte um einen eventuellen dritten Teil der Reihe aufflammen, haben Modder im Laufe der Jahre unzählige Mods und Total Conversions für das Ur-Spiel produziert. Unter dem Nickname sultim_t hat nun einer von ihnen Raytracing in Half-Life 1 integriert. Die Bilder zeigen eindrucksvoll, wie stark sich die neue Technik selbst in einem so alten Spiel auf das Erlebnis auswirkt. Neben Lampen und Displays haben auch die Waffen des Protagonisten neue Licht- und Schatteneffekte erhalten.</p><div data-nav="thumbs" data-width="100%" data-minwidth="320" data-maxheight="510" data-allowfullscreen="true" data-keyboard="true" data-arrows="true" data-click="false" data-swipe="true">     <imageobject><a href="https://www.hardwareluxx.de/images/cdn02/uploads/2023/Apr/grand_branch_i5/valve_half-life_ray-tracing004_300px.jpg"><img width="300" height="168" alt="Quelle: https://www.pcgamer.com/after-playing-half-life-with-this-new-ray-tracing-mod-i-need-a-cold-shower-in-a-dimly-lit-room/" title="Quelle: https://www.pcgamer.com/after-playing-half-life-with-this-new-ray-tracing-mod-i-need-a-cold-shower-in-a-dimly-lit-room/" src="https://www.hardwareluxx.de/images/cdn02/uploads/2023/Apr/grand_branch_i5/valve_half-life_ray-tracing004_300px.jpg"/></a></imageobject><imageobject><a href="https://www.hardwareluxx.de/images/cdn02/uploads/2023/Apr/roomy_segment_9v/valve_half-life_ray-tracing003_300px.jpg"><img width="300" height="168" alt="Quelle: https://www.pcgamer.com/after-playing-half-life-with-this-new-ray-tracing-mod-i-need-a-cold-shower-in-a-dimly-lit-room/" title="Quelle: https://www.pcgamer.com/after-playing-half-life-with-this-new-ray-tracing-mod-i-need-a-cold-shower-in-a-dimly-lit-room/" src="https://www.hardwareluxx.de/images/cdn02/uploads/2023/Apr/roomy_segment_9v/valve_half-life_ray-tracing003_300px.jpg"/></a></imageobject><imageobject><a href="https://www.hardwareluxx.de/images/cdn02/uploads/2023/Apr/noble_sample_tj/valve_half-life_ray-tracing002_300px.jpg"><img width="300" height="168" alt="Quelle: https://www.pcgamer.com/after-playing-half-life-with-this-new-ray-tracing-mod-i-need-a-cold-shower-in-a-dimly-lit-room/" title="Quelle: https://www.pcgamer.com/after-playing-half-life-with-this-new-ray-tracing-mod-i-need-a-cold-shower-in-a-dimly-lit-room/" src="https://www.hardwareluxx.de/images/cdn02/uploads/2023/Apr/noble_sample_tj/valve_half-life_ray-tracing002_300px.jpg"/></a></imageobject><imageobject><a href="https://www.hardwareluxx.de/images/cdn02/uploads/2023/Apr/prompt_tech_bo/valve_half-life_ray-tracing001_300px.jpg"><img width="300" height="168" alt="Quelle: https://www.pcgamer.com/after-playing-half-life-with-this-new-ray-tracing-mod-i-need-a-cold-shower-in-a-dimly-lit-room/" title="Quelle: https://www.pcgamer.com/after-playing-half-life-with-this-new-ray-tracing-mod-i-need-a-cold-shower-in-a-dimly-lit-room/" src="https://www.hardwareluxx.de/images/cdn02/uploads/2023/Apr/prompt_tech_bo/valve_half-life_ray-tracing001_300px.jpg"/></a></imageobject></div> <p>Wer die Mod selbst ausprobieren möchte, benötigt die Originalversion von Half-Life. Nach der Installation über Steam muss im Zielordner erst die Zip-Datei der Mod entpackt werden. Danach die Datei xash3d.exe starten und mit der X-Taste die neuen Render aktivieren. Half Life von 1998 gibt es auf <a href="https://store.steampowered.com/app/70/HalfLife/" target="_blank">Steam</a> aktuell für 8,19 Euro. Die Mod gibt es kostenlos auf der <a href="https://github.com/sultim-t/xash-rt/releases" target="_blank">GitHub-Seite von sultim_t</a>. Dort findet sich auch eine genaue Anleitung zur Installation. </p> <videoobject><h3>Related video</h3><a href="https://www.youtube.com/embed/LQCZTxzW6A0"><img src="http://img.youtube.com/vi/LQCZTxzW6A0/hqdefault.jpg"/></a></videoobject></div>
+
+</article></article>

article_scraper/resources/tests/ftr/heise-1/expected.html

@@ -0,0 +1,151 @@
+<article><article id="meldung" data-article-type="meldung">&#13;
+  &#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+  <div>&#13;
+        &#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+      &#13;
+&#13;
+      <p>Man kann sagen, dass Joshua Bergen ein sehr produktiver Mensch ist. Und sein Geheimnis dabei ist eine App namens <a href="https://www.heise.de/ratgeber/Wie-Sie-Webseiten-mit-Notion-veroeffentlichen-7164176.html" target="_blank">Notion</a>. Bergen, ein in Vancouver lebender Produktmanager, nutzt sie unter anderem, um Auslandsreisen bis ins kleinste Detail zu planen – mit Notizen und Zeitplänen. Er erstellt mit der Software Listen der Filme und Fernsehsendungen, die er gesehen hat, und hält fest, wie er sie findet. Notion sei auch praktisch, um seine 3D-Drucker-Projekte im Auge zu behalten, Snowboardtouren zu planen oder eine Liste der lustigsten Dinge, die sein Nachwuchs gesagt hat, auf dem aktuellen Stand zu halten.</p>&#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+<p>Interessanterweise ist Notion für das alles eigentlich gar nicht gedacht. Stattdessen ist das Werkzeug zum Wissensmanagement für die Arbeit in Unternehmen vorgesehen. Doch immer mehr User verwenden Notion, um ihr Privatleben zu organisieren. Sie nutzen die Software auf unterschiedlichste Weise: von der Dokumentation ihrer Meditationsgewohnheiten bis hin zur Protokollierung ihres Trinkverhaltens oder dem Austausch von Einkaufslisten.  </p>&#13;
+<p>"Ich habe festgestellt, dass meine Produktivität bei eigenen Projekten explodiert ist, seit ich es benutze", sagt Bergen. Er habe in den letzten zwei Jahren mehr Projekte erledigt als in den 10 Jahren davor. "Vielleicht ist das obsessiv, vielleicht ist das zu viel, aber es ist alles für mich und ich liebe es." Warum also hat eine Software, die laut Hersteller für ein "besseres, schnelleres Arbeiten" entwickelt wurde, auch im Privatleben so viele Fans? Schließlich gibt es zahlreiche Apps dieser Art.</p>&#13;
+<h3 id="nav_zahlreiche__0">Zahlreiche Anpassungen in Notion möglich</h3>&#13;
+<p>Einer der Gründe, warum Notion eine so treue Fangemeinde hat, ist seine Flexibilität. Im Kern ist die App so konzipiert, dass es die verschiedenen Systeme, die ein Unternehmen für Funktionen wie Personalwesen, Vertrieb und Produktplanung verwendet, zu einem "Hub" vereint. Es verwendet einfache Vorlagen (Templates), mit denen Benutzer Funktionen hinzufügen oder streichen können. Mitarbeiter unterschiedlicher Standorte können problemlos an Notizen, Datenbanken, Kalendern und Projekt-Boards zusammenarbeiten.</p>&#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+  &#13;
+    &#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+<p>Dieses hohe Maß an Anpassungsfähigkeit unterscheidet Notion von anderen Produktivitätsanwendungen. Es ist auch der Grund, warum Notion bei Menschen, die ihre Freizeit damit planen wollen, beliebt ist. Die App wurde 2018 insbesondere über eine Nische bei YouTube bekannt, in der sich immer mehr Menschen in Videos mit dem Thema Produktivität beschäftigen. Dort gibt es Tipps zum Zeitmanagement und Anleitungen zur Organisation des gesamten Lebens per Software. Regelmäßig werden Millionen Abrufe generiert.</p>&#13;
+<p>Die Fangemeinde hat sich in den letzten fünf Jahren vervielfacht. Mehr als 275.000 Menschen sind mittlerweile bei einem <a href="https://www.reddit.com/r/Notion/" rel="external noopener" target="_blank">speziellen Subreddit</a> dabei, Zehntausende von Nutzern tauschen kostenlose Notion-Templates in privaten Facebook-Gruppen aus. Und auf TikTok erreichen Videos, die zeigen, wie man Notion "schön macht", Millionen Abrufe.</p>&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+  &#13;
+    &#13;
+&#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+<h3 id="nav_notion_planung__1">Notion-Planung: vom Umzug bis zu den Mahlzeiten</h3>&#13;
+<p>"Sie müssen Ihre Gewohnheiten nicht ändern, weil die Software starr ist. Die Software ändert die Art und Weise, wie Sie denken", sagt Akshay Kothari, Mitbegründer und Chief Operating Officer (COO) von der Entwicklerfirma hinter Notion. "Ich glaube, das ist ein wichtiger Grund für ihre Beliebtheit in der Community: Die Leute haben das Gefühl, dass die Dinge, die sie damit machen, ihre eigenen sind."</p>&#13;
+<p>Die Möglichkeit der individuellen Anpassung ist durchaus beeindruckend. So kann Notion-Fan Bergen mit der App nicht nur die Seriennummern aller jemals von ihm gekaufter Produkte für den Fall speichern, dass sie gestohlen werden, sondern das detaillierte Inventar jeder einzelnen nummerierten Kiste erfassen, die er während eines Umzugs gepackt hat.</p>&#13;
+<p>Auch Wesley Anna Tiner, Produktdesignerin und Content Creator in New York, hält Notion für unverzichtbar. Sie plant damit ihren bevorstehenden Umzug genauso wie alle Mahlzeiten in der Woche. "Ich habe auch viele Notion-Pages, die nur zum Spaß dienen", erklärt sie. Sie habe zum Beispiel zu Weihnachten eine Parfümprobe bekommen und daraufhin eine Datenbank mit den verschiedenen weiteren Kosmetika angelegt, die sie künftig jeden Tag ausprobieren will – alle natürlich mit persönlicher Bewertung. Sie nutzt Notion mittlerweile auch als Stimmungstracker, Wunschliste und Self-Care-Werkzeug.</p>&#13;
+<h3 id="nav_f&#xFC;nf_jahre__2">Fünf Jahre Notion statt Notizbuch</h3>&#13;
+<p>Tommy Meyer, Webentwickler aus Phoenix, Arizona, begann schon 2018 mit Notion, nachdem er festgestellt hatte, dass er ständig drei verschiedene Notizbücher mit sich herumtrug, um alles geregelt zu bekommen. Er habe seit Jahren keine Einkaufsliste mehr auf Papier geschrieben, sagt er. Mittlerweile plottet er mit Notion auch Fantasy-Romane, die er schreiben möchte.</p>&#13;
+<p>Notion eignet sich gut für Notizen und zur Führung eines Tagesbuchs. Adam Warren, ein britischer Schauspieler und Synchronsprecher, hat es für sich als YouTube-Verwaltungswerkzeug entdeckt. "Ich verdiene jetzt mit meinem Kanal und Abos bei Patreon etwa so viel wie mit einem guten Vollzeitjob. Die gesamte Verwaltung dieses Geschäfts läuft über Notion", erklärt er. Alle seine Videoprojekte lägen in der Datenbank. "Ich verwende die Kanban-Ansicht, um ihren Status zu verfolgen. Und ich schreibe auch die Skripte für meine Videos direkt in diese Seiten hinein."</p>&#13;
+<p>Für Menschen, die das Gefühl mögen, gut organisiert zu sein, sind solche Plattformen offensichtlich sehr sinnhaft. Apps wie Notion können ihnen helfen, ihr Leben zu strukturieren und einfacher zu machen. Es fühle sich dann weniger überwältigend und chaotisch an, sagt Psychologin Elena Touroni, die Beratungen anbietet.</p>&#13;
+<h3 id="nav_wenn_die__3">Wenn die To-Do-Pflege wichtiger als die Arbeit wird</h3>&#13;
+<p>Wenn wir jedoch zu viel Zeit damit verbringen, unser Leben zu optimieren und zu organisieren, kann das kontraproduktiv sein. "Wenn wir dem Erstellen von To-Do-Listen mehr Bedeutung beimessen als der Erledigung der eigentlichen Aufgaben", sagt Gabriele Oettingen, Psychologieprofessorin an der New York University. Dafür gibt es bereits einen Fachbegriff: "Planning Fallacy", Planungsirrtum.</p>&#13;
+<p>Wenn man Notion benutzt, um zu tracken, ob man täglich genug Wasser trinkt oder wie vorgenommen joggen geht, bedeutet das nicht unbedingt, dass man diese Dinge auch tatsächlich tut. "In gewisser Weise kann Notion mir helfen, eine Struktur zu schaffen, aber es kann mich nicht dazu zwingen, ihr auch zu folgen", sagt sie.</p>&#13;
+<p>Und: Für Menschen wie Bergen, die dieselbe App verwenden, um sowohl ihr Privat- als auch ihr Arbeitsleben zu planen, könne das auch Nachteile haben, fügt Oettingens Kollegin Touroni hinzu.</p>&#13;
+<p>"Der offensichtliche Vorteil ist, dass sich Arbeits- und das Privatleben häufig überschneiden und die Verwendung derselben App dies dann für eine effizientere Zeitplanung berücksichtigt", so die Psychologin. "Der Nachteil ist, dass es schwieriger wird, Grenzen zwischen Arbeit und Privatleben zu ziehen, da man sich bei der Nutzung der App in beiden Lebensbereichen zurechtfinden muss."</p>&#13;
+<h3 id="nav_notion_es_ist__4">Notion – es ist eine Art Sucht</h3>&#13;
+<p>Trotz der Fülle an konkurrierenden Programmen, die ihnen zur Verfügung stehen, sagen die treuesten Fans von Notion, dass sie in absehbarer Zeit wohl kaum auf andere, womöglich bessere Plattformen ausweichen werden – es ist eine Art Sucht. Nutzerin Tiner aus New York meint, dass sie jetzt "95 Prozent des Lebens" über Notion abwickelt.</p>&#13;
+<p>Und die Zukunft scheint schon da zu sein: Die Macher der App haben vor Kurzem einen eigenen KI-Bot zur Automatisierung lästiger Aufgaben und zur Zusammenfassung umfangreicher Dokumente freigeschaltet. Man werde die Reaktion der Notion-Community in den sozialen Medien genau beobachten, hieß es. "Für ein Unternehmen, das mit Business-to-Business-Software sein Geld verdient, ist es einzigartig, diese Art von Nutzerliebe zu erfahren", sagt COO Kothari. "Das ist für uns ganz sicher keine Selbstverständlichkeit."</p>&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+  &#13;
+&#13;
+<figure>&#13;
+&#13;
+  <div>&#13;
+      <a href="https://www.instagram.com/technologyreview_de/" rel="external" target="_blank" title="MIT Technology Review auf Instagram">&#13;
+      &#13;
+&#13;
+&#13;
+&#13;
+<img alt="" height="190" src="https://heise.cloudimg.io/width/696/q50.png-lossy-50.webp-lossy-50.foil1/_www-heise-de_/imgs/18/4/1/5/0/0/2/4/tr-artikelbanner_insta_1830x500_11-2022-b1b5cf659568ea2b.png" srcset="https://heise.cloudimg.io/width/336/q30.png-lossy-30.webp-lossy-30.foil1/_www-heise-de_/imgs/18/4/1/5/0/0/2/4/tr-artikelbanner_insta_1830x500_11-2022-b1b5cf659568ea2b.png 336w, https://heise.cloudimg.io/width/1008/q30.png-lossy-30.webp-lossy-30.foil1/_www-heise-de_/imgs/18/4/1/5/0/0/2/4/tr-artikelbanner_insta_1830x500_11-2022-b1b5cf659568ea2b.png 1008w, https://heise.cloudimg.io/width/1392/q30.png-lossy-30.webp-lossy-30.foil1/_www-heise-de_/imgs/18/4/1/5/0/0/2/4/tr-artikelbanner_insta_1830x500_11-2022-b1b5cf659568ea2b.png 2x&#13;&#10;  " width="696" data-old-src="/imgs/18/4/1/5/0/0/2/4/tr-artikelbanner_insta_1830x500_11-2022-b1b5cf659568ea2b.png"/>&#13;
+&#13;
+&#13;
+&#13;
+      </a>&#13;
+    &#13;
+&#13;
+  </div>&#13;
+&#13;
+</figure>&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+&#13;
+      &#13;
+&#13;
+      &#13;
+&#13;
+      &#13;
+&#13;
+      &#13;
+&#13;
+      &#13;
+&#13;
+    </div>&#13;
+</article></article>

article_scraper/resources/tests/ftr/phoronix/expected.html

@@ -0,0 +1,16 @@
+<article><div>
+<div><img alt="GNOME" src="https://www.phoronix.com/assets/categories/gnome.webp" width="100" height="100"/></div>
+It's been one month already since the debut of <a href="https://www.phoronix.com/search/GNOME+44" target="_blank">GNOME 44</a> and out today is the first point release.
+<p>GNOME 44.1 brings many fixes to this updated Linux desktop, including many crash fixes and addressing newly uncovered memory leaks. Some of the GNOME 44.1 highlights include:
+</p><p>- Many fixes to GNOME Shell, including crash fixes, memory leak fixes, and other items addressed.
+</p><p>- GNOME's Mutter has also seen numerous fixes, including improved screencast support, fixing support for resizing windows via the keyboard, enabling modifiers by default for non-native backends, and various other fixes.
+</p><p>- The GNOME Settings Daemon will now connect to light sensors asynchronously.
+</p><p>- Crash fixes for GNOME Software and the Nautilus file manager.
+</p><p>- Nautilus now allows extraction of .tar.zst and .zstd archives.
+</p><p>- GNOME Control Center's display area now allows configuring all monitors and applying those settings at once.
+</p><p>- GNOME Calls will no longer crash on empty/null call ID.
+</p><p>- GNOME Web (Epiphany) has seen some crash fixes.
+</p><p>- GNOME Boxes for virtualization has a fix to always enable the boot menu option and fixing 3D acceleration not sticking at startup.
+</p><p>- GNOME Calendar has stability and performance improvements to its search.
+<br/></p><p><img src="https://www.phoronix.net/image.php?id=2023&amp;image=fedora38_beta_2_med" alt="Fedora 38 with GNOME 44"/></p>
+<br/>More details on the GNOME 44.1 changes via the <a href="https://discourse.gnome.org/t/gnome-44-1-released/15092" target="_blank">release announcement</a>.</div></article>

article_scraper/resources/tests/ftr/phoronix/source.html

@@ -0,0 +1,530 @@
+<!DOCTYPE html>
+<html xmlns="https://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+	<head>
+		<!-- Global site tag (gtag.js) - Google Analytics -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=UA-1978569-1"></script>
+<script>
+  window.dataLayer = window.dataLayer || [];
+  function gtag(){dataLayer.push(arguments);}
+  gtag('js', new Date());
+
+  gtag('config', 'UA-1978569-1');
+</script>
+
+<script data-cfasync="false" type="text/javascript">
+  var freestar = freestar || {};
+  freestar.hitTime = Date.now();
+  freestar.queue = freestar.queue || [];
+  freestar.config = freestar.config || {};
+  freestar.debug = window.location.search.indexOf('fsdebug') === -1 ? false : true;
+  freestar.config.enabled_slots = [];
+  !function(a,b){var c=b.getElementsByTagName("script")[0],d=b.createElement("script"),e="https://a.pub.network/phoronix-com";e+=freestar.debug?"/qa/pubfig.min.js":"/pubfig.min.js",d.async=!0,d.src=e,c.parentNode.insertBefore(d,c)}(window,document);
+  freestar.initCallback = function () { (freestar.config.enabled_slots.length === 0) ? freestar.initCallbackCalled = false : freestar.newAdSlots(freestar.config.enabled_slots) }
+</script>
+
+<script>
+freestar.config.disabledProducts = {
+    googleInterstitial: true
+};
+</script>
+		<title>GNOME 44.1 Released With Many Fixes - Phoronix</title>
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+		<link rel="stylesheet" href="/phxcms-css/phoronix-10.css?2030">
+				<meta name="viewport" content="width=device-width, initial-scale=1.0">
+		<meta name="keywords" content="Phoronix, Linux Hardware Reviews, Linux hardware benchmarks, Linux server benchmarks, Linux benchmarking, Desktop Linux, Linux performance, Open Source graphics, Linux How To, Ubuntu benchmarks, Ubuntu hardware, Phoronix Test Suite" />
+		<meta name="Description" content="It&#039;s been one month already since the debut of GNOME 44 and out today is the first point release." />
+		<link rel="alternate" type="application/rss+xml" title="Phoronix" href="rss.php" />
+		<link rel="shortcut icon" href="favicon.ico" />
+		<link rel="apple-touch-icon" sizes="57x57" href="/apple-touch-icon-57x57.png" />
+		<link rel="apple-touch-icon" sizes="60x60" href="/apple-touch-icon-60x60.png" />
+		<link rel="apple-touch-icon" sizes="72x72" href="/apple-touch-icon-72x72.png" />
+		<link rel="apple-touch-icon" sizes="76x76" href="/apple-touch-icon-76x76.png" />
+		<link rel="apple-touch-icon" sizes="114x114" href="/apple-touch-icon-114x114.png" />
+		<link rel="apple-touch-icon" sizes="120x120" href="/apple-touch-icon-120x120.png" />
+		<link rel="apple-touch-icon" sizes="144x144" href="/apple-touch-icon-144x144.png" />
+		<link rel="apple-touch-icon" sizes="152x152" href="/apple-touch-icon-152x152.png" />
+		<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon-180x180.png" />
+		<link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32" />
+		<link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192" />
+		<link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96" />
+		<link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16" />
+		<meta name="msapplication-TileImage" content="/mstile-144x144.png" />
+		<meta name="twitter:card" content="summary_large_image" />
+		<meta name="twitter:site" content="@phoronix" />
+		<meta name="twitter:creator" content="@michaellarabel" />
+		<meta name="twitter:title" content="GNOME 44.1 Released With Many Fixes" />
+		<meta name="og:title" content="GNOME 44.1 Released With Many Fixes" />
+		<meta name="twitter:image" content="https://www.phoronix.net/image.php?id=2023&image=fedora38_beta_2" />
+		<meta name="og:image" content="https://www.phoronix.net/image.php?id=2023&image=fedora38_beta_2" />
+		<meta name="twitter:description" content="It&#039;s been one month already since the debut of GNOME 44 and out today is the first point release." />
+		<!-- PHORONIX (phoronix.com) -->
+<!-- Quantcast Choice. Consent Manager Tag v2.0 (for TCF 2.0) -->
+<script type="text/javascript" async=true>
+(function() {
+  var host = 'phoronix.com';
+  var element = document.createElement('script');
+  var firstScript = document.getElementsByTagName('script')[0];
+  var url = 'https://cmp.quantcast.com'
+    .concat('/choice/', 'UEFjU_w924fg3', '/', host, '/choice.js');
+  var uspTries = 0;
+  var uspTriesLimit = 3;
+  element.async = true;
+  element.type = 'text/javascript';
+  element.src = url;
+
+  firstScript.parentNode.insertBefore(element, firstScript);
+
+  function makeStub() {
+    var TCF_LOCATOR_NAME = '__tcfapiLocator';
+    var queue = [];
+    var win = window;
+    var cmpFrame;
+
+    function addFrame() {
+      var doc = win.document;
+      var otherCMP = !!(win.frames[TCF_LOCATOR_NAME]);
+
+      if (!otherCMP) {
+        if (doc.body) {
+          var iframe = doc.createElement('iframe');
+
+          iframe.style.cssText = 'display:none';
+          iframe.name = TCF_LOCATOR_NAME;
+          doc.body.appendChild(iframe);
+        } else {
+          setTimeout(addFrame, 5);
+        }
+      }
+      return !otherCMP;
+    }
+
+    function tcfAPIHandler() {
+      var gdprApplies;
+      var args = arguments;
+
+      if (!args.length) {
+        return queue;
+      } else if (args[0] === 'setGdprApplies') {
+        if (
+          args.length > 3 &&
+          args[2] === 2 &&
+          typeof args[3] === 'boolean'
+        ) {
+          gdprApplies = args[3];
+          if (typeof args[2] === 'function') {
+            args[2]('set', true);
+          }
+        }
+      } else if (args[0] === 'ping') {
+        var retr = {
+          gdprApplies: gdprApplies,
+          cmpLoaded: false,
+          cmpStatus: 'stub'
+        };
+
+        if (typeof args[2] === 'function') {
+          args[2](retr);
+        }
+      } else {
+        if(args[0] === 'init' && typeof args[3] === 'object') {
+          args[3] = { ...args[3], tag_version: 'V2' };
+        }
+        queue.push(args);
+      }
+    }
+
+    function postMessageEventHandler(event) {
+      var msgIsString = typeof event.data === 'string';
+      var json = {};
+
+      try {
+        if (msgIsString) {
+          json = JSON.parse(event.data);
+        } else {
+          json = event.data;
+        }
+      } catch (ignore) {}
+
+      var payload = json.__tcfapiCall;
+
+      if (payload) {
+        window.__tcfapi(
+          payload.command,
+          payload.version,
+          function(retValue, success) {
+            var returnMsg = {
+              __tcfapiReturn: {
+                returnValue: retValue,
+                success: success,
+                callId: payload.callId
+              }
+            };
+            if (msgIsString) {
+              returnMsg = JSON.stringify(returnMsg);
+            }
+            if (event && event.source && event.source.postMessage) {
+              event.source.postMessage(returnMsg, '*');
+            }
+          },
+          payload.parameter
+        );
+      }
+    }
+
+    while (win) {
+      try {
+        if (win.frames[TCF_LOCATOR_NAME]) {
+          cmpFrame = win;
+          break;
+        }
+      } catch (ignore) {}
+
+      if (win === window.top) {
+        break;
+      }
+      win = win.parent;
+    }
+    if (!cmpFrame) {
+      addFrame();
+      win.__tcfapi = tcfAPIHandler;
+      win.addEventListener('message', postMessageEventHandler, false);
+    }
+  };
+
+  makeStub();
+
+  var uspStubFunction = function() {
+    var arg = arguments;
+    if (typeof window.__uspapi !== uspStubFunction) {
+      setTimeout(function() {
+        if (typeof window.__uspapi !== 'undefined') {
+          window.__uspapi.apply(window.__uspapi, arg);
+        }
+      }, 500);
+    }
+  };
+
+  var checkIfUspIsReady = function() {
+    uspTries++;
+    if (window.__uspapi === uspStubFunction && uspTries < uspTriesLimit) {
+      console.warn('USP is not accessible');
+    } else {
+      clearInterval(uspInterval);
+    }
+  };
+
+  if (typeof window.__uspapi === 'undefined') {
+    window.__uspapi = uspStubFunction;
+    var uspInterval = setInterval(checkIfUspIsReady, 6000);
+  }
+})();
+</script>
+<!-- End Quantcast Choice. Consent Manager Tag v2.0 (for TCF 2.0) -->
+    		
+	</head>
+	<body style="overflow: initial !important;">
+		<div style="text-align: center; overflow: hidden; background: #222;"><div align="center" id="phoronix_desktop_leaderboard_header">
+  <script data-cfasync="false" type="text/javascript">
+      freestar.config.enabled_slots.push({ placementName: "phoronix_desktop_leaderboard_header", slotId: "phoronix_desktop_leaderboard_header" });
+  </script>
+</div>
+</div>		<div id="headerwrap">
+			<div class="wcontainer defpadding">
+				<a href="/"><img id="logo" alt="Phoronix" src="/phxcms7-css/phoronix.png" style="height: 40px; margin-top: 5px;margin-left: 5px;" /></a>
+				<div id="headerright">
+					<div id="social">
+					    <a href="https://www.phoronix.com/rss.php" class="icon-rss"></a>
+					    <a href="https://twitter.com/Phoronix" rel="me" class="icon-twitter"></a>
+					    <a href="https://www.facebook.com/Phoronix" rel="me" class="icon-facebook"></a>
+					</div>
+				</div>
+			</div>
+		</div>
+		<div id="litemenu">
+		    <ul>
+		        <li><a href="/reviews">Articles &amp; Reviews</a></li>
+		        <li><a href="/news">News Archive</a></li>
+		        <li><a href="/forums/">Forums</a></li>
+		        <li><a href="/phoronix-premium">Premium</a></li>		        <li class="categ icon-angle-circled-down"><span> Categories</span></li>
+		         <li><a href="/reviews/Computers">Computers</a></li> <li><a href="/reviews/Display+Drivers">Display Drivers</a></li> <li><a href="/reviews/Graphics+Cards">Graphics Cards</a></li> <li><a href="/reviews/Linux+Gaming">Linux Gaming</a></li> <li><a href="/reviews/Memory">Memory</a></li> <li><a href="/reviews/Motherboards">Motherboards</a></li> <li><a href="/reviews/Processors">Processors</a></li> <li><a href="/reviews/Software">Software</a></li> <li><a href="/reviews/Storage">Storage</a></li> <li><a href="/reviews/Operating+Systems">Operating Systems</a></li> <li><a href="/reviews/Peripherals">Peripherals</a></li>				<li id="closelid">Close</li>
+		    </ul>		    
+		</div>
+		<div id="menuwrap">
+			<div class="wcontainer">
+				<ul id="linklist">
+					<li id="menubutton"><span class="icon-menu"></span>
+					</li>
+					<li class="menulink">
+						<a class="linknode" href="/reviews">Articles &amp; Reviews</a>
+					</li>
+					<li class="menulink">
+						<a class="linknode" href="/news">News Archive</a>
+					</li>
+					<li class="menulink">
+						<a class="linknode" href="/forums/">Forums</a>
+					</li>
+					<li class="menulink">
+						<a class="linknode" href="/phoronix-premium">Premium</a>
+					</li>					<li class="menulink">
+						<a class="linknode" href="/contact">Contact</a>
+					</li>
+					<li id="categories-link" class="menulink">
+						<span class="menunolink icon-angle-circled-down"> Categories</span>
+						<div id="categories-list">
+							<div id="categories-content">
+		         <a href="/reviews/Computers">Computers</a> <a href="/reviews/Display+Drivers">Display Drivers</a> <a href="/reviews/Graphics+Cards">Graphics Cards</a> <a href="/reviews/Linux+Gaming">Linux Gaming</a> <a href="/reviews/Memory">Memory</a> <a href="/reviews/Motherboards">Motherboards</a> <a href="/reviews/Processors">Processors</a> <a href="/reviews/Software">Software</a> <a href="/reviews/Storage">Storage</a> <a href="/reviews/Operating+Systems">Operating Systems</a> <a href="/reviews/Peripherals">Peripherals</a>							</div>
+						</div>
+					</li>
+					<li id="searchbox">
+						<div id="search">
+							<form action="https://www.google.com" id="cse-search-box" target="_blank">
+							  <div>
+							    <input type="hidden" name="cx" value="partner-pub-0253814508491313:1305299758" />
+							    <input type="hidden" name="ie" value="UTF-8" />
+							    <input type="text" name="q" />
+							    <input type="submit" name="sa" value="Search" />
+							  </div>
+							</form>
+
+							<script type="text/javascript" src="https://www.google.com/coop/cse/brand?form=cse-search-box&amp;lang=en"></script>
+						</div>
+					</li>
+				</ul>
+			</div>
+		</div>
+		<div id="content-bg">
+			<div id="content">
+				<div id="main-wrap">
+					<div id="main">
+						<div style="text-align: center; overflow: hidden; margin-bottom: 30px;">
+							<div align="center" id="phoronix_desktop_leaderboard_atf">
+  <script data-cfasync="false" type="text/javascript">
+      freestar.config.enabled_slots.push({ placementName: "phoronix_desktop_leaderboard_atf", slotId: "phoronix_desktop_leaderboard_atf" });
+  </script>
+</div>
+
+						</div>
+						<!-- google_ad_section_start -->
+<article class="full">
+<h1>GNOME 44.1 Released With Many Fixes</h1>
+<div class="author">Written by <a href="https://www.michaellarabel.com/">Michael Larabel</a> in <a href="/linux/GNOME">GNOME</a> on 26 April 2023 at 12:00 PM EDT. <a href="/forums/node/1384710">6 Comments</a></div>
+<div class="content">
+<div style="float: left; padding: 0 10px 10px;"><img alt="GNOME" src="/assets/categories/gnome.webp" width="100" height="100" /></div>
+It's been one month already since the debut of <a href="https://www.phoronix.com/search/GNOME+44">GNOME 44</a> and out today is the first point release.
+<br />
+<br />GNOME 44.1 brings many fixes to this updated Linux desktop, including many crash fixes and addressing newly uncovered memory leaks. Some of the GNOME 44.1 highlights include:
+<br />
+<br />- Many fixes to GNOME Shell, including crash fixes, memory leak fixes, and other items addressed.
+<br />
+<br />- GNOME's Mutter has also seen numerous fixes, including improved screencast support, fixing support for resizing windows via the keyboard, enabling modifiers by default for non-native backends, and various other fixes.
+<br />
+<br />- The GNOME Settings Daemon will now connect to light sensors asynchronously.
+<br />
+<br />- Crash fixes for GNOME Software and the Nautilus file manager.
+<br />
+<br />- Nautilus now allows extraction of .tar.zst and .zstd archives.
+<br />
+<br />- GNOME Control Center's display area now allows configuring all monitors and applying those settings at once.
+<br />
+<br />- GNOME Calls will no longer crash on empty/null call ID.
+<br />
+<br />- GNOME Web (Epiphany) has seen some crash fixes.
+<br />
+<br />- GNOME Boxes for virtualization has a fix to always enable the boot menu option and fixing 3D acceleration not sticking at startup.
+<br />
+<br />- GNOME Calendar has stability and performance improvements to its search.
+<br /><p align="center"><img src="//www.phoronix.net/image.php?id=2023&image=fedora38_beta_2_med" alt="Fedora 38 with GNOME 44" /></p>
+<br />More details on the GNOME 44.1 changes via the <a href="https://discourse.gnome.org/t/gnome-44-1-released/15092">release announcement</a>.</div>
+							<div style="text-align: center; overflow: hidden; margin-bottom: 30px;">
+								<div align="center" id="phoronix_leaderboard_btf">
+  <script data-cfasync="false" type="text/javascript">
+      freestar.config.enabled_slots.push({ placementName: "phoronix_leaderboard_btf", slotId: "phoronix_leaderboard_btf" });
+  </script>
+</div>
+							</div>
+							<div class="foot">
+								<div class="pagination">
+								</div>
+								<div class="comments-label">
+									<a href="/forums/node/1384710">6 Comments</a>								</div>
+							</div>
+						</article>
+<div id="sharebar">
+<div style="margin: 0 10px 10px; overflow: hidden;">
+
+<div style="float: left; overflow: hidden;"><a href="https://twitter.com/share" class="twitter-share-button" data-text="GNOME 44.1 Released With Many Fixes" data-url="https://www.phoronix.com/news/GNOME-44.1-Released" data-via="phoronix" data-related="michaellarabel">Tweet</a>
+<script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs');</script></div>
+
+<div style="float: left; overflow: hidden;"><iframe src="//www.facebook.com/plugins/like.php?href=https://www.phoronix.com/news/GNOME-44.1-Released&amp;width=200&amp;height=46&amp;colorscheme=light&amp;layout=button_count&amp;action=like&amp;show_faces=false&amp;send=true" scrolling="no" frameborder="0" style="border:none; overflow:hidden; width:200px; height:46px;" allowTransparency="true"></iframe></div>
+</div></div>
+<script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>
+<ins class="adsbygoogle"
+     style="display:block"
+     data-ad-format="autorelaxed"
+     data-ad-client="ca-pub-0253814508491313"
+     data-ad-slot="7279734770"></ins>
+<script>
+     (adsbygoogle = window.adsbygoogle || []).push({});
+</script>
+<div class="article-box">
+								<header>Related News</header>
+								<div class="node"><a href="/news/GTK-4.11.1">GTK 4.11.1 Released With Better Textures, Wayland Fractional Scaling</a></div><div class="node"><a href="/news/WebKitGTK-DMA-BUF-Rendering">WebKitGTK Working On Accelerated Composited Rendering With DMA-BUF</a></div><div class="node"><a href="/news/GNOME-44-Released">GNOME 44 Released With Many Desktop Enhancements</a></div><div class="node"><a href="/news/GNOME-44-Release-Candidate">GNOME 44 Release Candidate Arrives With Many Last Minute Changes</a></div><div class="node"><a href="/news/GNOME-Shell-Mutter-44-RC">GNOME Shell &amp; Mutter 44 Release Candidates Bring Last Minute Changes</a></div><div class="node"><a href="/news/GNOME-44-fractional_scale_v1">GNOME 44 Mutter Adds fractional_scale_v1 Wayland Support</a></div>
+	                             
+							</div>						<div class="after-article">
+							<div style="text-align: center; overflow: hidden; margin: 5px;">
+								<div align="center" id="phoronix_leaderboard_incontent_3">
+  <script data-cfasync="false" type="text/javascript">
+      freestar.config.enabled_slots.push({ placementName: "phoronix_leaderboard_incontent_3", slotId: "phoronix_leaderboard_incontent_3" });
+  </script>
+</div>
+</div>
+							<div id="about-author">
+								<header>About The Author</header>
+								<img class="pic" src="/assets/categories/michaellarabel.jpg" alt="Michael Larabel"/>
+								<div class="text"><p>Michael Larabel is the principal author of Phoronix.com and founded the site in 2004 with a focus on enriching the Linux hardware experience. Michael has written more than 20,000 articles covering the state of Linux hardware support, Linux performance, graphics drivers, and other topics. Michael is also the lead developer of the Phoronix Test Suite, Phoromatic, and OpenBenchmarking.org automated benchmarking software. He can be followed via <a href="https://twitter.com/MichaelLarabel">Twitter</a>, <a href="https://www.linkedin.com/in/michaellarabel/">LinkedIn</a>, or contacted via <a href="https://www.michaellarabel.com/">MichaelLarabel.com</a>.</p></div>
+							</div>
+<div class="article-box">
+								<header>Popular News This Week</header>
+								<div class="node"><a href="/news/Intel-i219-LM-Linux-60p-Fix">Intel i219-LM Had Only Been Running At ~60% Of Maximum Speed Due To Linux Driver Bug</a></div><div class="node"><a href="/news/Red-Hat-Layoffs">Red Hat Begins Cutting &quot;Hundreds Of Jobs&quot;</a></div><div class="node"><a href="/news/GIMP-GTK3-Finished">GTK3 Port Of GIMP Is &quot;Officially Finished&quot;</a></div><div class="node"><a href="/news/Linux-Kernel-Rust-1.68">The Linux Kernel Preparing For An Upgrade To Its Rust Toolchain</a></div><div class="node"><a href="/news/AMD-Linux-Dynamic-Boost-Control">AMD Posts New Linux Patches Enabling Dynamic Boost Control</a></div><div class="node"><a href="/news/Linux-6.3-Released">Linux 6.3 Released With More Meteor Lake Enablement, Zen 4 Auto IBRS &amp; Much More</a></div><div class="node"><a href="/news/Linux-6.3-Feature-Recap">Linux 6.3 Features Have A Lot For AMD &amp; Intel, Steam Deck, ASUS Motherboards &amp; More</a></div><div class="node"><a href="/news/QEMU-8.0-Released">QEMU 8.0 Released With 32-bit x86 Host Support Deprecated</a></div>
+							</div>		</div>
+<!-- google_ad_section_end -->
+					</div>
+				</div>
+				<div id="sidebar-wrap">
+					<div id="sidebar">
+					    <aside>
+						<div style="text-align: center;">
+                           				<div align="center" id="phoronix_right_rail_1">
+  <script data-cfasync="false" type="text/javascript">
+      freestar.config.enabled_slots.push({ placementName: "phoronix_right_rail_1", slotId: "phoronix_right_rail_1" });
+  </script>
+</div>
+                        </div>
+                        </aside>
+                        <aside class="widget">
+			<header>Latest Linux News</header><div class="popular-list">
+                                <a href="/news/Dragora-3.0-Beta-2">Dragora 3.0 Beta 2 OS Released: 10+ Years In Development, FSF Backed &amp; Using SysV Init</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/news/Linux-6.4-x86-Platform-Drivers">Linux 6.4 Brings Improved MSI Laptop Support, Apple GMUX Support For T2 Macs</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/news/GNOME-44.1-Released">GNOME 44.1 Released With Many Fixes</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/news/Wayland-Weston-12.0-Alpha">Wayland&#039;s Weston 12 Alpha Brings Multi-GPU Support, PipeWire Backend, Tearing Control</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/news/Linux-6.4-Hardware-Monitoring">100+ More ASUS Motherboards Enabled For Sensor Monitoring With Linux 6.4</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/news/Linux-AMD-Spectral-Chicken">Linux Kernel Drama: AMD&#039;s Spectral Chicken</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/news/GCC-13.1-Released">GCC 13.1 Released With Modula-2 Language Support, More C23/C++23 Features</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/news/xf86-video-ati-22.0">xf86-video-ati 22.0 Released For Older ATI/AMD GPUs</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/news/Btrfs-F2FS-Linux-6.4">F2FS &amp; Btrfs Enjoy Some Nice Improvements With Linux 6.4</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/news/System76-Scheduler-2.0">System76-Scheduler 2.0 Released With PipeWire Integration, Performance Optimizations</a>
+                            </div>
+                        </aside>
+			<aside>
+			<div style="text-align: center;">
+                          <div align="center" id="phoronix_right_rail_2">
+  <script data-cfasync="false" type="text/javascript">
+      freestar.config.enabled_slots.push({ placementName: "phoronix_right_rail_2", slotId: "phoronix_right_rail_2" });
+  </script>
+</div>
+
+                        </div>
+                        </aside>
+			                        <aside class="widget">
+                       <header>Show Your Support, Go Premium</header>
+                       <p><a href="/phoronix-premium">Phoronix Premium</a> allows ad-free access to the site, multi-page articles on a single page, and other features while supporting this site's continued operations.</p>
+                        </aside>
+			                        <aside class="widget">
+			<header>Latest Featured Articles</header><div class="popular-list">
+                                <a href="/review/windows11-wsl2-zen4">Windows 11 WSL2 Performance vs. Ubuntu Linux With The AMD Ryzen 7 7800X3D</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/review/corsair-48gb-ddr5-7000">Corsair 2 x 24GB DDR5-7000 Memory Linux Performance</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/review/ubuntu-2304-laptops">Ubuntu 23.04 Laptop Performance Mixed Against Ubuntu 22.10</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/review/nvidia-windows11-ubuntu2304">NVIDIA GeForce RTX 4080/4090: Windows 11 vs. Ubuntu 23.04 Performance</a>
+                            </div>
+<div class="popular-list">
+                                <a href="/review/amd-7800x3d-windows11-ubuntu">AMD Ryzen 7 7800X3D: Windows 11 vs. Ubuntu 23.04 Linux Performance</a>
+                            </div>
+                        </aside>
+			<aside>
+						<div style="text-align: center;">
+                           				<div style="position: sticky; top: 0; position: -webkit-sticky;"><div align="center" id="phoronix_right_rail_3">
+  <script data-cfasync="false" type="text/javascript">
+      freestar.config.enabled_slots.push({ placementName: "phoronix_right_rail_3", slotId: "phoronix_right_rail_3" });
+  </script>
+</div>
+
+</div>                       				 </div>
+                        </aside>
+                        <aside class="widget">
+                            <header>Support Phoronix</header>
+                            <p>The mission at Phoronix since 2004 has centered around enriching the Linux hardware experience. In addition to supporting our site through advertisements, you can help by <a href="/phoronix-premium">subscribing to Phoronix Premium</a>. You can also contribute to Phoronix through a <a href="https://www.paypal.com/donate/?hosted_button_id=EA79CCDLNFJNW">PayPal tip</a> or <a href="https://buy.stripe.com/28o02d1yG1Lp8H67ss">tip via Stripe</a>.</p>
+                        </aside>
+					</div>
+				</div>
+			</div>
+		</div>
+		<div id="footer">
+		    <div id="footer-content">
+			<p align="center"></p>
+		        <div id="footer-left">
+		        <div id="footer-left-content">
+		            <div class="extra-info">
+		                <header>Phoronix Media</header>
+		                <hr/>
+		                <ul>
+		                    <li><a href="/contact">Contact</a></li>
+		                    <li><a href="/michaellarabel">Michael Larabel</a></li>
+		                    <li><a href="https://openbenchmarking.org/">OpenBenchmarking.org</a></li>
+		                </ul>
+		            </div>
+		            <div class="extra-info">
+		                <header>Phoronix Premium</header>
+		                <hr/>
+		                <a href="/phoronix-premium"><ul>
+                            <li>Support Phoronix</li>
+		                    <li>While Having Ad-Free Browsing,</li>
+                            <li>Single-Page Article Viewing</li>
+                        </ul></a>
+		            </div>
+		            <div class="extra-info">
+		                <header>Share</header>
+		                <hr/>
+		                <ul>
+		                    <li><a href="https://facebook.com/Phoronix" rel="me">Facebook</a></li>
+		                    <li><a href="https://twitter.com/Phoronix" rel="me">Twitter</a></li>
+		                </ul>
+		            </div>
+		       </div>
+		       </div>
+		       <div id="footer-right">
+		            <div class="extra-info">
+		                <ul>
+		                <li><p><a href="/legal">Legal Disclaimer, Privacy Policy, Cookies</a> | <a href="https://www.phoronix-media.com/?k=contact">Contact</a></p></li>
+                        <li><p>Copyright &copy; 2004 - 2023 by <a href="https://www.phoronix-media.com/">Phoronix Media</a>.</p></li>
+                        <li><p>All trademarks used are properties of their respective owners. All rights reserved.</p></li>
+				</ul>
+		            </div>
+		        </div>
+		     </div>		   
+		</div>
+	</body>
+</html>

article_scraper/resources/tests/ftr/spiegel-1/expected.html

@@ -0,0 +1,52 @@
+<article><section data-article-el="body">
+<div data-area="top_element&gt;image">
+<figure>
+<div data-sara-component="{&quot;id&quot;:&quot;51d712c3-9d59-4234-b369-6dc5de953038&quot;,&quot;name&quot;:&quot;image&quot;,&quot;title&quot;:&quot;\u003cp\u003e&#xBB;Barbie&#xAB; soll im Libanon nicht gezeigt werden\u003c/p\u003e&quot;,&quot;type&quot;:&quot;media&quot;}">
+<picture>
+<source srcset="https://cdn.prod.www.spiegel.de/images/51d712c3-9d59-4234-b369-6dc5de953038_w948_r2.1283783783783785_fpx55_fpy28.webp 948w, https://cdn.prod.www.spiegel.de/images/51d712c3-9d59-4234-b369-6dc5de953038_w520_r2.1283783783783785_fpx55_fpy28.webp 520w" sizes="(max-width: 519px) 100vw, (min-width: 520px) and (max-width: 719px) 520px, (min-width: 720px) and (max-width: 919px) 100vw, (min-width: 920px) and (max-width: 1011px) 920px, (min-width: 1012px) 948px" type="image/webp">
+<img data-image-el="img" src="https://cdn.prod.www.spiegel.de/images/51d712c3-9d59-4234-b369-6dc5de953038_w948_r2.1283783783783785_fpx55_fpy28.jpg" width="948" height="445" title="&#xBB;Barbie&#xAB; soll im Libanon nicht gezeigt werden" alt="&#xBB;Barbie&#xAB; soll im Libanon nicht gezeigt werden" data-image-animation-origin="0df59b8e-c667-4bf5-b686-3266795cecf1"/>
+</source></picture>
+</div>
+<figcaption>
+<p>»Barbie« soll im Libanon nicht gezeigt werden</p>
+<span>
+Foto: <p>- / dpa</p>
+</span>
+</figcaption>
+</figure>
+</div>
+<div data-area="body">
+<div data-sara-click-el="body_element">
+<p>Im <a href="https://www.spiegel.de/thema/libanon/" data-link-flag="spon" target="_blank">Libanon</a> soll der erfolgreiche <a href="https://www.spiegel.de/thema/hollywood/" data-link-flag="spon" target="_blank">Hollywood</a>-Streifen »Barbie« verboten werden, weil dieser der Regierung zufolge »Werbung für Homosexualität und Geschlechtsumwandlung« macht. Der Film verstoße gegen die »moralischen und religiösen Werte« des Landes, erklärte der libanesische Kulturminister Mohammed Mourtada. Ursprünglich sollte der Blockbuster, der weltweit <a href="https://www.spiegel.de/kultur/kino/barbie-hat-eine-milliarde-us-dollar-weltweit-eingespielt-a-61b0c5f7-e354-4ce9-a021-8c3305a29ff7" data-link-flag="spon" target="_blank">bereits mehr als eine Milliarde Dollar </a>(rund 910 Millionen Euro) eingespielt hat, ab dem 31. August im Libanon gezeigt werden.</p>
+</div>
+
+<div data-sara-click-el="body_element">
+<p>Mourtada erklärte weiter, »Barbie« unterstütze die »Ablehnung der Vormundschaft des Vaters«, ziehe die Rolle der Mutter ins Lächerliche und stelle die Ehe und die Gründung einer Familie infrage.</p>
+</div>
+<div data-sara-click-el="body_element">
+
+
+<div data-settings="{&quot;consentType&quot;:&quot;thirdparty&quot;}">
+
+An dieser Stelle finden Sie einen externen Inhalt von Twitter,
+der den Artikel ergänzt und von der Redaktion empfohlen wird. Sie können ihn sich mit einem Klick anzeigen lassen
+und wieder ausblenden.
+</div>
+
+<div data-sara-click-el="body_element">
+<p>In dem Film von US-Regisseurin Greta Gerwig verlassen <a href="https://www.spiegel.de/thema/barbie/" data-link-flag="spon" target="_blank">Barbie</a> und Ken, gespielt von den Superstars <a href="https://www.spiegel.de/thema/margot-robbie/" data-link-flag="spon" target="_blank">Margot Robbie</a> und <a href="https://www.spiegel.de/thema/ryan_gosling/" data-link-flag="spon" target="_blank">Ryan Gosling</a>, die pinkfarbene Plastikwelt Barbieland und lernen in <a href="https://www.spiegel.de/thema/los_angeles/" data-link-flag="spon" target="_blank">Los Angeles</a> das echte Leben kennen.</p>
+</div>
+
+<div data-sara-click-el="body_element">
+<h3>Community unter Druck</h3><p>Der Libanon gilt in Bezug auf Homosexualität als toleranter als andere arabische Staaten. Allerdings haben religiöse Organisationen wie die radikalislamische <a href="https://www.spiegel.de/thema/hisbollah/" data-link-flag="spon" target="_blank">Hisbollah</a> einen großen Einfluss auf soziale und kulturelle Einrichtungen.</p><p>Immer wieder wurden in den vergangenen Jahren Veranstaltungen der <a href="https://www.spiegel.de/kultur/musik/mashrou-leila-regenbogen-auf-halbmast-a-1e6bdb86-e0e4-4ffa-8f35-5aac4cb5d1e1" data-link-flag="spon" target="_blank">libanesischen LGBTQ+-Community </a>abgesagt. Zudem steht Homosexualität im Libanon nach wie vor unter Strafe.</p>
+</div>
+
+
+<div data-sara-click-el="body_element">
+<p>Die englische Abkürzung LGBTQ+ steht für lesbisch, schwul, bisexuell, transgender, queer und andere Geschlechtsidentitäten.</p>
+
+</div>
+</div>
+
+</div>
+</section></article>

article_scraper/resources/tests/ftr/youtube/expected.html

@@ -0,0 +1 @@
+<article><iframe id="video" width="480" height="360" aspect-ratio="auto" src="https://www.youtube.com/embed/8KjaIumu-jI?feature=oembed" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen="" title="RIGGED! Arena Shuffler is BROKEN | 13 Land Mono Red Burn"><empty></empty></iframe></article>

article_scraper/resources/tests/ftr/youtube/source.html

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<oembed><title>RIGGED! Arena Shuffler is BROKEN | 13 Land Mono Red Burn</title><author_name>CovertGoBlue</author_name><author_url>https://www.youtube.com/@covertgoblue</author_url><type>video</type><height>113</height><width>200</width><version>1.0</version><provider_name>YouTube</provider_name><provider_url>https://www.youtube.com/</provider_url><thumbnail_height>360</thumbnail_height><thumbnail_width>480</thumbnail_width><thumbnail_url>https://i.ytimg.com/vi/8KjaIumu-jI/hqdefault.jpg</thumbnail_url><html>&lt;iframe width=&quot;200&quot; height=&quot;113&quot; src=&quot;https://www.youtube.com/embed/8KjaIumu-jI?feature=oembed&quot; frameborder=&quot;0&quot; allow=&quot;accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share&quot; allowfullscreen title=&quot;RIGGED! Arena Shuffler is BROKEN | 13 Land Mono Red Burn&quot;&gt;&lt;/iframe&gt;</html></oembed>

article_scraper/resources/tests/images/planet_gnome/source.html

@@ -41,3 +41,4 @@ A lot of work has gone into improving the general performance of GNOME 3.34. The
 </p><div class="wp-caption alignnone" id="attachment_7948" style="width: 1010px;"><a href="https://blogs.gnome.org/uraeus/files/2019/09/sonic-boom.jpg"><img alt="Sonic Boom" class="size-full wp-image-7948" height="334" src="https://blogs.gnome.org/uraeus/files/2019/09/sonic-boom.jpg" width="500"/></a><p class="wp-caption-text" id="caption-attachment-7948">Performance is better than ever</p></div> <p/>
 <p><b>Summary</b><br/>
 So this has been a roundup of some of the core items you should look forward to in Fedora Workstation 31. There are other items coming too in this release, like the Miracast GNOME Network Display application that Benjamin Berg has written, more Fedora Flatpaks available than ever before and more. We also have a lot of interesting items coming up in Fedora Workstation 32 like Bastien Noceras work improving low memory handling. So stay tuned.</p></div>
+

article_scraper/resources/tests/readability/001/expected.html

@@ -0,0 +1,132 @@
+<article><section id="readability-page-1">
+                        <p><strong>So finally you're <a href="http://fakehost/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/" target="_blank">testing your frontend JavaScript code</a>? Great! The more you
+write tests, the more confident you are with your code… but how much precisely?
+That's where <a href="http://en.wikipedia.org/wiki/Code_coverage" target="_blank">code coverage</a> might
+help.</strong>
+                        </p>
+                        <p>The idea behind code coverage is to record which parts of your code (functions,
+                            statements, conditionals and so on) have been executed by your test suite,
+                            to compute metrics out of these data and usually to provide tools for navigating
+                            and inspecting them.</p>
+                        <p>Not a lot of frontend developers I know actually test their frontend code,
+                            and I can barely imagine how many of them have ever setup code coverage…
+                            Mostly because there are not many frontend-oriented tools in this area
+                            I guess.</p>
+                        <p>Actually I've only found one which provides an adapter for <a href="http://visionmedia.github.io/mocha/" target="_blank">Mocha</a> and
+                            actually works…</p>
+                        <blockquote>
+                            <p>Drinking game for web devs:
+                                <br/>(1) Think of a noun
+                                <br/>(2) Google "&lt;noun&gt;.js"
+                                <br/>(3) If a library with that name exists - drink</p>— Shay Friedman (@ironshay)
+                            <a href="https://twitter.com/ironshay/statuses/370525864523743232" target="_blank">August 22, 2013</a>
+                        </blockquote>
+                        <p><strong><a href="http://blanketjs.org/" target="_blank">Blanket.js</a></strong> is an <em>easy to install, easy to configure,
+and easy to use JavaScript code coverage library that works both in-browser and
+with nodejs.</em>
+                        </p>
+                        <p>Its use is dead easy, adding Blanket support to your Mocha test suite
+                            is just matter of adding this simple line to your HTML test file:</p>
+<pre><code>&lt;script src="vendor/blanket.js"
+        data-cover-adapter="vendor/mocha-blanket.js"&gt;&lt;/script&gt;
+</code></pre>
+
+                        <p>Source files: <a href="https://raw.github.com/alex-seville/blanket/master/dist/qunit/blanket.min.js" target="_blank">blanket.js</a>,
+                            <a href="https://raw.github.com/alex-seville/blanket/master/src/adapters/mocha-blanket.js" target="_blank">mocha-blanket.js</a>
+                        </p>
+                        <p>As an example, let's reuse the silly <code>Cow</code> example we used
+                            <a href="http://fakehost/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/" target="_blank">in a previous episode</a>:</p>
+<pre><code>// cow.js
+(function(exports) {
+  "use strict";
+
+  function Cow(name) {
+    this.name = name || "Anon cow";
+  }
+  exports.Cow = Cow;
+
+  Cow.prototype = {
+    greets: function(target) {
+      if (!target)
+        throw new Error("missing target");
+      return this.name + " greets " + target;
+    }
+  };
+})(this);
+</code></pre>
+
+                        <p>And its test suite, powered by Mocha and <a href="http://chaijs.com/" target="_blank">Chai</a>:</p>
+<pre><code>var expect = chai.expect;
+
+describe("Cow", function() {
+  describe("constructor", function() {
+    it("should have a default name", function() {
+      var cow = new Cow();
+      expect(cow.name).to.equal("Anon cow");
+    });
+
+    it("should set cow's name if provided", function() {
+      var cow = new Cow("Kate");
+      expect(cow.name).to.equal("Kate");
+    });
+  });
+
+  describe("#greets", function() {
+    it("should greet passed target", function() {
+      var greetings = (new Cow("Kate")).greets("Baby");
+      expect(greetings).to.equal("Kate greets Baby");
+    });
+  });
+});
+</code></pre>
+
+                        <p>Let's create the HTML test file for it, featuring Blanket and its adapter
+                            for Mocha:</p>
+<pre><code>&lt;!DOCTYPE html&gt;
+&lt;html&gt;
+&lt;head&gt;
+  &lt;meta charset="utf-8"&gt;
+  &lt;title&gt;Test&lt;/title&gt;
+  &lt;link rel="stylesheet" media="all" href="vendor/mocha.css"&gt;
+&lt;/head&gt;
+&lt;body&gt;
+  &lt;div id="mocha"&gt;&lt;/div&gt;
+  &lt;div id="messages"&gt;&lt;/div&gt;
+  &lt;div id="fixtures"&gt;&lt;/div&gt;
+  &lt;script src="vendor/mocha.js"&gt;&lt;/script&gt;
+  &lt;script src="vendor/chai.js"&gt;&lt;/script&gt;
+  &lt;script src="vendor/blanket.js"
+          data-cover-adapter="vendor/mocha-blanket.js"&gt;&lt;/script&gt;
+  &lt;script&gt;mocha.setup('bdd');&lt;/script&gt;
+  &lt;script src="cow.js" data-cover&gt;&lt;/script&gt;
+  &lt;script src="cow_test.js"&gt;&lt;/script&gt;
+  &lt;script&gt;mocha.run();&lt;/script&gt;
+&lt;/body&gt;
+&lt;/html&gt;
+</code></pre>
+
+                        <p><strong>Notes</strong>:</p>
+                        <ul>
+                            <li>Notice the <code>data-cover</code> attribute we added to the script tag
+                                loading the source of our library;</li>
+                            <li>The HTML test file <em>must</em> be served over HTTP for the adapter to
+                                be loaded.</li>
+                        </ul>
+                        <p>Running the tests now gives us something like this:</p>
+                        <p>
+                            <img alt="screenshot" src="http://fakehost/static/code/2013/blanket-coverage.png"/>
+                        </p>
+                        <p>As you can see, the report at the bottom highlights that we haven't actually
+                            tested the case where an error is raised in case a target name is missing.
+                            We've been informed of that, nothing more, nothing less. We simply know
+                            we're missing a test here. Isn't this cool? I think so!</p>
+                        <p>Just remember that code coverage will only <a href="http://codebetter.com/karlseguin/2008/12/09/code-coverage-use-it-wisely/" target="_blank">bring you numbers</a> and
+                            raw information, not actual proofs that the whole of your <em>code logic</em> has
+                            been actually covered. If you ask me, the best inputs you can get about
+                            your code logic and implementation ever are the ones issued out of <a href="http://www.extremeprogramming.org/rules/pair.html" target="_blank">pair programming</a>
+sessions
+                            and <a href="http://alexgaynor.net/2013/sep/26/effective-code-review/" target="_blank">code reviews</a> —
+                            but that's another story.</p>
+                        <p><strong>So is code coverage silver bullet? No. Is it useful? Definitely. Happy testing!</strong>
+                        </p>
+                    </section></article>

article_scraper/resources/tests/readability/001/source.html

@@ -0,0 +1,234 @@
+<!DOCTYPE html>
+<html class="no-js" lang="en">
+    
+    <head>
+        <meta charset="utf-8"/>
+        <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
+        <title>Get your Frontend JavaScript Code Covered | Code | Nicolas Perriault</title>
+        <meta
+        name="description" content="Nicolas Perriault's homepage."/>
+            <meta name="viewport" content="width=device-width"/>
+            <link href="//fonts.googleapis.com/css?family=Asap:400,400italic,700,700italic&amp;subset=latin,latin-ext"
+            rel="stylesheet" type="text/css"/>
+            <link rel="stylesheet" type="text/css" href="/static/packed.css?1412806084"/>
+            <link rel="alternate" type="application/rss+xml" href="/code/feed/" title="Code (RSS)"/>
+            <link rel="alternate" type="application/rss+xml" href="/photography/feed/"
+            title="Photography (RSS)"/>
+            <link rel="alternate" type="application/rss+xml" href="/talks/feed/" title="Talks (RSS)"/>
+            <link rel="alternate" type="application/rss+xml" href="/carnet/feed/"
+            title="Carnet (RSS)"/>
+            <link rel="alternate" type="application/rss+xml" href="/feed/" title="Everything (RSS)"/>
+            <!--[if lt IE 9]>
+                <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
+            <![endif]-->
+    </head>
+    
+    <body class="code " onload="prettyPrint()">
+        <!--[if lt IE 7]>
+            <p class="chromeframe">Your browser is <em>ancient!</em> Please <a href="http://www.quirksmode.org/upgrade.html">upgrade</a>.</p>
+        <![endif]-->
+        <div class="container">
+            <header class="main-title">
+                 <h1><a href="/">Hi, I'm <strong>Nicolas.</strong></a></h1>
+ <small>I code stuff. I take photos. I write rants.</small>
+
+            </header>
+            <main class="contents" role="main">
+                <article lang="en" class="code" itemscope="" itemtype="http://schema.org/BlogPosting">
+                    <link itemprop="url" href="/code/2013/get-your-frontend-javascript-code-covered/"/>
+                    <header>
+                         <h2><a itemprop="name" href="/code/2013/get-your-frontend-javascript-code-covered/">Get your Frontend JavaScript Code Covered</a></h2>
+                    </header>
+                    <section>
+                        <p><strong>So finally you're <a href="/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">testing your frontend JavaScript code</a>? Great! The more you
+write tests, the more confident you are with your code… but how much precisely?
+That's where <a href="http://en.wikipedia.org/wiki/Code_coverage">code coverage</a> might
+help.</strong>
+                        </p>
+                        <p>The idea behind code coverage is to record which parts of your code (functions,
+                            statements, conditionals and so on) have been executed by your test suite,
+                            to compute metrics out of these data and usually to provide tools for navigating
+                            and inspecting them.</p>
+                        <p>Not a lot of frontend developers I know actually test their frontend code,
+                            and I can barely imagine how many of them have ever setup code coverage…
+                            Mostly because there are not many frontend-oriented tools in this area
+                            I guess.</p>
+                        <p>Actually I've only found one which provides an adapter for <a href="http://visionmedia.github.io/mocha/">Mocha</a> and
+                            actually works…</p>
+                        <blockquote class="twitter-tweet tw-align-center">
+                            <p>Drinking game for web devs:
+                                <br />(1) Think of a noun
+                                <br />(2) Google "&lt;noun&gt;.js"
+                                <br />(3) If a library with that name exists - drink</p>— Shay Friedman (@ironshay)
+                            <a
+                            href="https://twitter.com/ironshay/statuses/370525864523743232">August 22, 2013</a>
+                        </blockquote>
+                        <p><strong><a href="http://blanketjs.org/">Blanket.js</a></strong> is an <em>easy to install, easy to configure,
+and easy to use JavaScript code coverage library that works both in-browser and
+with nodejs.</em>
+                        </p>
+                        <p>Its use is dead easy, adding Blanket support to your Mocha test suite
+                            is just matter of adding this simple line to your HTML test file:</p>
+<pre><code>&lt;script src="vendor/blanket.js"
+        data-cover-adapter="vendor/mocha-blanket.js"&gt;&lt;/script&gt;
+</code></pre>
+
+                        <p>Source files: <a href="https://raw.github.com/alex-seville/blanket/master/dist/qunit/blanket.min.js">blanket.js</a>,
+                            <a
+                            href="https://raw.github.com/alex-seville/blanket/master/src/adapters/mocha-blanket.js">mocha-blanket.js</a>
+                        </p>
+                        <p>As an example, let's reuse the silly <code>Cow</code> example we used
+                            <a
+                            href="/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">in a previous episode</a>:</p>
+<pre><code>// cow.js
+(function(exports) {
+  "use strict";
+
+  function Cow(name) {
+    this.name = name || "Anon cow";
+  }
+  exports.Cow = Cow;
+
+  Cow.prototype = {
+    greets: function(target) {
+      if (!target)
+        throw new Error("missing target");
+      return this.name + " greets " + target;
+    }
+  };
+})(this);
+</code></pre>
+
+                        <p>And its test suite, powered by Mocha and <a href="http://chaijs.com/">Chai</a>:</p>
+<pre><code>var expect = chai.expect;
+
+describe("Cow", function() {
+  describe("constructor", function() {
+    it("should have a default name", function() {
+      var cow = new Cow();
+      expect(cow.name).to.equal("Anon cow");
+    });
+
+    it("should set cow's name if provided", function() {
+      var cow = new Cow("Kate");
+      expect(cow.name).to.equal("Kate");
+    });
+  });
+
+  describe("#greets", function() {
+    it("should greet passed target", function() {
+      var greetings = (new Cow("Kate")).greets("Baby");
+      expect(greetings).to.equal("Kate greets Baby");
+    });
+  });
+});
+</code></pre>
+
+                        <p>Let's create the HTML test file for it, featuring Blanket and its adapter
+                            for Mocha:</p>
+<pre><code>&lt;!DOCTYPE html&gt;
+&lt;html&gt;
+&lt;head&gt;
+  &lt;meta charset="utf-8"&gt;
+  &lt;title&gt;Test&lt;/title&gt;
+  &lt;link rel="stylesheet" media="all" href="vendor/mocha.css"&gt;
+&lt;/head&gt;
+&lt;body&gt;
+  &lt;div id="mocha"&gt;&lt;/div&gt;
+  &lt;div id="messages"&gt;&lt;/div&gt;
+  &lt;div id="fixtures"&gt;&lt;/div&gt;
+  &lt;script src="vendor/mocha.js"&gt;&lt;/script&gt;
+  &lt;script src="vendor/chai.js"&gt;&lt;/script&gt;
+  &lt;script src="vendor/blanket.js"
+          data-cover-adapter="vendor/mocha-blanket.js"&gt;&lt;/script&gt;
+  &lt;script&gt;mocha.setup('bdd');&lt;/script&gt;
+  &lt;script src="cow.js" data-cover&gt;&lt;/script&gt;
+  &lt;script src="cow_test.js"&gt;&lt;/script&gt;
+  &lt;script&gt;mocha.run();&lt;/script&gt;
+&lt;/body&gt;
+&lt;/html&gt;
+</code></pre>
+
+                        <p><strong>Notes</strong>:</p>
+                        <ul>
+                            <li>Notice the <code>data-cover</code> attribute we added to the script tag
+                                loading the source of our library;</li>
+                            <li>The HTML test file <em>must</em> be served over HTTP for the adapter to
+                                be loaded.</li>
+                        </ul>
+                        <p>Running the tests now gives us something like this:</p>
+                        <p>
+                            <img alt="screenshot" src="/static/code/2013/blanket-coverage.png"/>
+                        </p>
+                        <p>As you can see, the report at the bottom highlights that we haven't actually
+                            tested the case where an error is raised in case a target name is missing.
+                            We've been informed of that, nothing more, nothing less. We simply know
+                            we're missing a test here. Isn't this cool? I think so!</p>
+                        <p>Just remember that code coverage will only <a href="http://codebetter.com/karlseguin/2008/12/09/code-coverage-use-it-wisely/">bring you numbers</a> and
+                            raw information, not actual proofs that the whole of your <em>code logic</em> has
+                            been actually covered. If you ask me, the best inputs you can get about
+                            your code logic and implementation ever are the ones issued out of <a href="http://www.extremeprogramming.org/rules/pair.html">pair programming</a>
+sessions
+                            and <a href="http://alexgaynor.net/2013/sep/26/effective-code-review/">code reviews</a> —
+                            but that's another story.</p>
+                        <p><strong>So is code coverage silver bullet? No. Is it useful? Definitely. Happy testing!</strong>
+                        </p>
+                    </section>
+                    <aside>
+                        <p> <span class="article-author" itemprop="author" itemscope="" itemtype="http://schema.org/Person">
+                <span itemprop="name">Nicolas Perriault</span> —</span>
+                            <time
+                            datetime="2013-09-29" itemprop="datePublished">2013-09-29</time>— in <a href="/code/" itemprop="genre">Code</a>
+— <a href="/code/2013/get-your-frontend-javascript-code-covered/">Permalink</a>
+—
+                                <a
+                                rel="license" href="http://creativecommons.org/licenses/by-sa/3.0/">License</a>— <a href="http://flattr.com/submit/auto?url=https://nicolas.perriault.net/code/2013/get-your-frontend-javascript-code-covered/&amp;title=Get your Frontend JavaScript Code Covered&amp;user_id=n1k0&amp;category=software&amp;language=en">flattr this</a>
+
+                        </p>
+                    </aside>
+                    <hr/>
+                    <nav> <a class="prev" href="/code/2013/functional-javascript-for-crawling-the-web/">Functional JavaScript for crawling the Web</a>
+|
+                        <a
+                        class="next" href="/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">Testing your frontend JavaScript code using mocha, chai, and sinon</a>
+                    </nav>
+                </article>
+            </main>
+            <nav class="sidebar">
+                <ul>
+                    <li class="home"><a href="/" hreflang="en">Home</a>
+                    </li>
+                    <li class="code"><a href="/code/" hreflang="en">Code</a>
+                    </li>
+                    <li class="photography"><a href="/photography/" hreflang="en">Photography</a>
+                    </li>
+                    <li class="talks"><a href="/talks/" hreflang="en">Talks</a>
+                    </li>
+                    <li class="carnet"><a href="/carnet/" hreflang="fr">Carnet <span>fr</span></a>
+                    </li>
+                    <li class="contact"><a href="/contact/" hreflang="en">Contact</a>
+                    </li>
+                </ul>
+            </nav>
+            <footer class="site-footer">
+                <p>© 2012 Nicolas Perriault — <a href="https://twitter.com/n1k0">Tweet at me</a>
+—
+                    <a
+                    href="https://github.com/n1k0">Get my code</a>— <a href="http://500px.com/n1k0">Enjoy my pics</a>
+— <a href="/contact/">Contact me</a>
+
+                </p>
+            </footer>
+        </div>
+        <!-- /container -->
+        <script src="//ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
+        <script>
+            window.jQuery || document.write('&lt;script src="js/libs/jquery-1.7.1.min.js">&lt;\/script>')
+        </script>
+        <script type="text/javascript" src="/static/js/libs/prettify/prettify.js"></script>
+        <script type="text/javascript" src="/static/js/app.js"></script>
+        <script src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
+    </body>
+
+</html>
+

article_scraper/resources/tests/readability/002/expected.html

@@ -0,0 +1,418 @@
+<article><DIV id="readability-page-1">
+                    <article role="article">
+                        <p>For more than a decade the Web has used XMLHttpRequest (XHR) to achieve
+                            asynchronous requests in JavaScript. While very useful, XHR is not a very
+                            nice API. It suffers from lack of separation of concerns. The input, output
+                            and state are all managed by interacting with one object, and state is
+                            tracked using events. Also, the event-based model doesn’t play well with
+                            JavaScript’s recent focus on Promise- and generator-based asynchronous
+                            programming.</p>
+                        <p>The <a href="https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API" target="_blank">Fetch API</a> intends
+                            to fix most of these problems. It does this by introducing the same primitives
+                            to JS that are used in the HTTP protocol. In addition, it introduces a
+                            utility function <code>fetch()</code> that succinctly captures the intention
+                            of retrieving a resource from the network.</p>
+                        <p>The <a href="https://fetch.spec.whatwg.org/" target="_blank">Fetch specification</a>, which
+                            defines the API, nails down the semantics of a user agent fetching a resource.
+                            This, combined with ServiceWorkers, is an attempt to:</p>
+                        <ol>
+                            <li>Improve the offline experience.</li>
+                            <li>Expose the building blocks of the Web to the platform as part of the
+                                <a href="https://extensiblewebmanifesto.org/" target="_blank">extensible web movement</a>.</li>
+                        </ol>
+                        <p>As of this writing, the Fetch API is available in Firefox 39 (currently
+                            Nightly) and Chrome 42 (currently dev). Github has a <a href="https://github.com/github/fetch" target="_blank">Fetch polyfill</a>.</p>
+                        
+<h2>Feature detection</h2>
+
+                        <p>Fetch API support can be detected by checking for <code>Headers</code>,<code>Request</code>, <code>Response</code> or <code>fetch</code> on
+                            the <code>window</code> or <code>worker</code> scope.</p>
+                        
+<h2>Simple fetching</h2>
+
+                        <p>The most useful, high-level part of the Fetch API is the <code>fetch()</code> function.
+                            In its simplest form it takes a URL and returns a promise that resolves
+                            to the response. The response is captured as a <code>Response</code> object.</p>
+                        <DIV><pre>fetch<span>(</span><span>"/data.json"</span><span>)</span>.<span>then</span><span>(</span><span>function</span><span>(</span>res<span>)</span> <span>{</span>
+  <span>// res instanceof Response == true.</span>
+  <span>if</span> <span>(</span>res.<span>ok</span><span>)</span> <span>{</span>
+    res.<span>json</span><span>(</span><span>)</span>.<span>then</span><span>(</span><span>function</span><span>(</span>data<span>)</span> <span>{</span>
+      console.<span>log</span><span>(</span>data.<span>entries</span><span>)</span><span>;</span>
+    <span>}</span><span>)</span><span>;</span>
+  <span>}</span> <span>else</span> <span>{</span>
+    console.<span>log</span><span>(</span><span>"Looks like the response wasn't perfect, got status"</span><span>,</span> res.<span>status</span><span>)</span><span>;</span>
+  <span>}</span>
+<span>}</span><span>,</span> <span>function</span><span>(</span>e<span>)</span> <span>{</span>
+  console.<span>log</span><span>(</span><span>"Fetch failed!"</span><span>,</span> e<span>)</span><span>;</span>
+<span>}</span><span>)</span><span>;</span></pre>
+                                        </DIV>
+            <p>Submitting some parameters, it would look like this:</p>
+            <DIV><pre>fetch<span>(</span><span>"http://www.example.org/submit.php"</span><span>,</span> <span>{</span>
+  method<span>:</span> <span>"POST"</span><span>,</span>
+  headers<span>:</span> <span>{</span>
+    <span>"Content-Type"</span><span>:</span> <span>"application/x-www-form-urlencoded"</span>
+  <span>}</span><span>,</span>
+  body<span>:</span> <span>"firstName=Nikhil&amp;favColor=blue&amp;password=easytoguess"</span>
+<span>}</span><span>)</span>.<span>then</span><span>(</span><span>function</span><span>(</span>res<span>)</span> <span>{</span>
+  <span>if</span> <span>(</span>res.<span>ok</span><span>)</span> <span>{</span>
+    alert<span>(</span><span>"Perfect! Your settings are saved."</span><span>)</span><span>;</span>
+  <span>}</span> <span>else</span> <span>if</span> <span>(</span>res.<span>status</span> <span>==</span> <span>401</span><span>)</span> <span>{</span>
+    alert<span>(</span><span>"Oops! You are not authorized."</span><span>)</span><span>;</span>
+  <span>}</span>
+<span>}</span><span>,</span> <span>function</span><span>(</span>e<span>)</span> <span>{</span>
+  alert<span>(</span><span>"Error submitting form!"</span><span>)</span><span>;</span>
+<span>}</span><span>)</span><span>;</span></pre>
+                            </DIV>
+            <p>The <code>fetch()</code> function’s arguments are the same as those passed
+                to the
+                <br/>
+<code>Request()</code> constructor, so you may directly pass arbitrarily
+                complex requests to <code>fetch()</code> as discussed below.</p>
+            
+<h2>Headers</h2>
+
+            <p>Fetch introduces 3 interfaces. These are <code>Headers</code>, <code>Request</code> and
+            <br/>
+<code>Response</code>. They map directly to the underlying HTTP concepts,
+                but have
+                <br/>certain visibility filters in place for privacy and security reasons,
+                such as
+                <br/>supporting CORS rules and ensuring cookies aren’t readable by third parties.</p>
+            <p>The <a href="https://fetch.spec.whatwg.org/#headers-class" target="_blank">Headers interface</a> is
+                a simple multi-map of names to values:</p>
+            <DIV><pre><span>var</span> content <span>=</span> <span>"Hello World"</span><span>;</span>
+<span>var</span> reqHeaders <span>=</span> <span>new</span> Headers<span>(</span><span>)</span><span>;</span>
+reqHeaders.<span>append</span><span>(</span><span>"Content-Type"</span><span>,</span> <span>"text/plain"</span>
+reqHeaders.<span>append</span><span>(</span><span>"Content-Length"</span><span>,</span> content.<span>length</span>.<span>toString</span><span>(</span><span>)</span><span>)</span><span>;</span>
+reqHeaders.<span>append</span><span>(</span><span>"X-Custom-Header"</span><span>,</span> <span>"ProcessThisImmediately"</span><span>)</span><span>;</span></pre>
+                            </DIV>
+            <p>The same can be achieved by passing an array of arrays or a JS object
+                literal
+                <br/>to the constructor:</p>
+            <DIV><pre>reqHeaders <span>=</span> <span>new</span> Headers<span>(</span><span>{</span>
+  <span>"Content-Type"</span><span>:</span> <span>"text/plain"</span><span>,</span>
+  <span>"Content-Length"</span><span>:</span> content.<span>length</span>.<span>toString</span><span>(</span><span>)</span><span>,</span>
+  <span>"X-Custom-Header"</span><span>:</span> <span>"ProcessThisImmediately"</span><span>,</span>
+<span>}</span><span>)</span><span>;</span></pre>
+                            </DIV>
+            <p>The contents can be queried and retrieved:</p>
+            <DIV><pre>console.<span>log</span><span>(</span>reqHeaders.<span>has</span><span>(</span><span>"Content-Type"</span><span>)</span><span>)</span><span>;</span> <span>// true</span>
+console.<span>log</span><span>(</span>reqHeaders.<span>has</span><span>(</span><span>"Set-Cookie"</span><span>)</span><span>)</span><span>;</span> <span>// false</span>
+reqHeaders.<span>set</span><span>(</span><span>"Content-Type"</span><span>,</span> <span>"text/html"</span><span>)</span><span>;</span>
+reqHeaders.<span>append</span><span>(</span><span>"X-Custom-Header"</span><span>,</span> <span>"AnotherValue"</span><span>)</span><span>;</span>
+ 
+console.<span>log</span><span>(</span>reqHeaders.<span>get</span><span>(</span><span>"Content-Length"</span><span>)</span><span>)</span><span>;</span> <span>// 11</span>
+console.<span>log</span><span>(</span>reqHeaders.<span>getAll</span><span>(</span><span>"X-Custom-Header"</span><span>)</span><span>)</span><span>;</span> <span>// ["ProcessThisImmediately", "AnotherValue"]</span>
+ 
+reqHeaders.<span>delete</span><span>(</span><span>"X-Custom-Header"</span><span>)</span><span>;</span>
+console.<span>log</span><span>(</span>reqHeaders.<span>getAll</span><span>(</span><span>"X-Custom-Header"</span><span>)</span><span>)</span><span>;</span> <span>// []</span></pre>
+                            </DIV>
+            <p>Some of these operations are only useful in ServiceWorkers, but they provide
+            <br/>a much nicer API to Headers.</p>
+            <p>Since Headers can be sent in requests, or received in responses, and have
+                various limitations about what information can and should be mutable, <code>Headers</code> objects
+                have a <strong>guard</strong> property. This is not exposed to the Web, but
+                it affects which mutation operations are allowed on the Headers object.
+                <br/>Possible values are:</p>
+            <ul>
+                <li>“none”: default.</li>
+                <li>“request”: guard for a Headers object obtained from a Request (<code>Request.headers</code>).</li>
+                <li>“request-no-cors”: guard for a Headers object obtained from a Request
+                    created
+                    <br/>with mode “no-cors”.</li>
+                <li>“response”: naturally, for Headers obtained from Response (<code>Response.headers</code>).</li>
+                <li>“immutable”: Mostly used for ServiceWorkers, renders a Headers object
+                <br/>read-only.</li>
+            </ul>
+            <p>The details of how each guard affects the behaviors of the Headers object
+                are
+                <br/>in the <a href="https://fetch.spec.whatwg.org/" target="_blank">specification</a>. For example,
+                you may not append or set a “request” guarded Headers’ “Content-Length”
+                header. Similarly, inserting “Set-Cookie” into a Response header is not
+                allowed so that ServiceWorkers may not set cookies via synthesized Responses.</p>
+            <p>All of the Headers methods throw TypeError if <code>name</code> is not a
+                <a href="https://fetch.spec.whatwg.org/#concept-header-name" target="_blank">valid HTTP Header name</a>. The mutation operations will throw TypeError
+                    if there is an immutable guard. Otherwise they fail silently. For example:</p>
+            <DIV><pre><span>var</span> res <span>=</span> Response.<span>error</span><span>(</span><span>)</span><span>;</span>
+<span>try</span> <span>{</span>
+  res.<span>headers</span>.<span>set</span><span>(</span><span>"Origin"</span><span>,</span> <span>"http://mybank.com"</span><span>)</span><span>;</span>
+<span>}</span> <span>catch</span><span>(</span>e<span>)</span> <span>{</span>
+  console.<span>log</span><span>(</span><span>"Cannot pretend to be a bank!"</span><span>)</span><span>;</span>
+<span>}</span></pre>
+                            </DIV>
+        
+<h2>Request</h2>
+
+        <p>The Request interface defines a request to fetch a resource over HTTP.
+            URL, method and headers are expected, but the Request also allows specifying
+            a body, a request mode, credentials and cache hints.</p>
+        <p>The simplest Request is of course, just a URL, as you may do to GET a
+            resource.</p>
+        <DIV><pre><span>var</span> req <span>=</span> <span>new</span> Request<span>(</span><span>"/index.html"</span><span>)</span><span>;</span>
+console.<span>log</span><span>(</span>req.<span>method</span><span>)</span><span>;</span> <span>// "GET"</span>
+console.<span>log</span><span>(</span>req.<span>url</span><span>)</span><span>;</span> <span>// "http://example.com/index.html"</span></pre>
+                        </DIV>
+        <p>You may also pass a Request to the <code>Request()</code> constructor to
+            create a copy.
+            <br/>(This is not the same as calling the <code>clone()</code> method, which
+            is covered in
+            <br/>the “Reading bodies” section.).</p>
+        <DIV><pre><span>var</span> copy <span>=</span> <span>new</span> Request<span>(</span>req<span>)</span><span>;</span>
+console.<span>log</span><span>(</span>copy.<span>method</span><span>)</span><span>;</span> <span>// "GET"</span>
+console.<span>log</span><span>(</span>copy.<span>url</span><span>)</span><span>;</span> <span>// "http://example.com/index.html"</span></pre>
+                        </DIV>
+        <p>Again, this form is probably only useful in ServiceWorkers.</p>
+        <p>The non-URL attributes of the <code>Request</code> can only be set by passing
+            initial
+            <br/>values as a second argument to the constructor. This argument is a dictionary.</p>
+        <DIV><pre><span>var</span> uploadReq <span>=</span> <span>new</span> Request<span>(</span><span>"/uploadImage"</span><span>,</span> <span>{</span>
+  method<span>:</span> <span>"POST"</span><span>,</span>
+  headers<span>:</span> <span>{</span>
+    <span>"Content-Type"</span><span>:</span> <span>"image/png"</span><span>,</span>
+  <span>}</span><span>,</span>
+  body<span>:</span> <span>"image data"</span>
+<span>}</span><span>)</span><span>;</span></pre>
+                        </DIV>
+            <p>The Request’s mode is used to determine if cross-origin requests lead
+                to valid responses, and which properties on the response are readable.
+                Legal mode values are <code>"same-origin"</code>, <code>"no-cors"</code> (default)
+                and <code>"cors"</code>.</p>
+            <p>The <code>"same-origin"</code> mode is simple, if a request is made to another
+                origin with this mode set, the result is simply an error. You could use
+                this to ensure that
+                <br/>a request is always being made to your origin.</p>
+            <DIV><pre><span>var</span> arbitraryUrl <span>=</span> document.<span>getElementById</span><span>(</span><span>"url-input"</span><span>)</span>.<span>value</span><span>;</span>
+fetch<span>(</span>arbitraryUrl<span>,</span> <span>{</span> mode<span>:</span> <span>"same-origin"</span> <span>}</span><span>)</span>.<span>then</span><span>(</span><span>function</span><span>(</span>res<span>)</span> <span>{</span>
+  console.<span>log</span><span>(</span><span>"Response succeeded?"</span><span>,</span> res.<span>ok</span><span>)</span><span>;</span>
+<span>}</span><span>,</span> <span>function</span><span>(</span>e<span>)</span> <span>{</span>
+  console.<span>log</span><span>(</span><span>"Please enter a same-origin URL!"</span><span>)</span><span>;</span>
+<span>}</span><span>)</span><span>;</span></pre>
+                            </DIV>
+            <p>The <code>"no-cors"</code> mode captures what the web platform does by default
+                for scripts you import from CDNs, images hosted on other domains, and so
+                on. First, it prevents the method from being anything other than “HEAD”,
+                “GET” or “POST”. Second, if any ServiceWorkers intercept these requests,
+                they may not add or override any headers except for <a href="https://fetch.spec.whatwg.org/#simple-header" target="_blank">these</a>.
+                Third, JavaScript may not access any properties of the resulting Response.
+                This ensures that ServiceWorkers do not affect the semantics of the Web
+                and prevents security and privacy issues that could arise from leaking
+                data across domains.</p>
+            <p><code>"cors"</code> mode is what you’ll usually use to make known cross-origin
+                requests to access various APIs offered by other vendors. These are expected
+                to adhere to
+                <br/>the <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS" target="_blank">CORS protocol</a>.
+                Only a <a href="https://fetch.spec.whatwg.org/#concept-filtered-response-cors" target="_blank">limited set</a> of
+                headers is exposed in the Response, but the body is readable. For example,
+                you could get a list of Flickr’s <a href="https://www.flickr.com/services/api/flickr.interestingness.getList.html" target="_blank">most interesting</a> photos
+                today like this:</p>
+            <DIV><pre><span>var</span> u <span>=</span> <span>new</span> URLSearchParams<span>(</span><span>)</span><span>;</span>
+u.<span>append</span><span>(</span><span>'method'</span><span>,</span> <span>'flickr.interestingness.getList'</span><span>)</span><span>;</span>
+u.<span>append</span><span>(</span><span>'api_key'</span><span>,</span> <span>'&lt;insert api key here&gt;'</span><span>)</span><span>;</span>
+u.<span>append</span><span>(</span><span>'format'</span><span>,</span> <span>'json'</span><span>)</span><span>;</span>
+u.<span>append</span><span>(</span><span>'nojsoncallback'</span><span>,</span> <span>'1'</span><span>)</span><span>;</span>
+ 
+<span>var</span> apiCall <span>=</span> fetch<span>(</span><span>'https://api.flickr.com/services/rest?'</span> <span>+</span> u<span>)</span><span>;</span>
+ 
+apiCall.<span>then</span><span>(</span><span>function</span><span>(</span>response<span>)</span> <span>{</span>
+  <span>return</span> response.<span>json</span><span>(</span><span>)</span>.<span>then</span><span>(</span><span>function</span><span>(</span>json<span>)</span> <span>{</span>
+    <span>// photo is a list of photos.</span>
+    <span>return</span> json.<span>photos</span>.<span>photo</span><span>;</span>
+  <span>}</span><span>)</span><span>;</span>
+<span>}</span><span>)</span>.<span>then</span><span>(</span><span>function</span><span>(</span>photos<span>)</span> <span>{</span>
+  photos.<span>forEach</span><span>(</span><span>function</span><span>(</span>photo<span>)</span> <span>{</span>
+    console.<span>log</span><span>(</span>photo.<span>title</span><span>)</span><span>;</span>
+  <span>}</span><span>)</span><span>;</span>
+<span>}</span><span>)</span><span>;</span></pre>
+                            </DIV>
+            <p>You may not read out the “Date” header since Flickr does not allow it
+                via
+                <br/>
+<code>Access-Control-Expose-Headers</code>.</p>
+            <DIV><pre>response.<span>headers</span>.<span>get</span><span>(</span><span>"Date"</span><span>)</span><span>;</span> <span>// null</span></pre>
+                            </DIV>
+            <p>The <code>credentials</code> enumeration determines if cookies for the other
+                domain are
+                <br/>sent to cross-origin requests. This is similar to XHR’s <code>withCredentials</code>
+                <br/>flag, but tri-valued as <code>"omit"</code> (default), <code>"same-origin"</code> and <code>"include"</code>.</p>
+            <p>The Request object will also give the ability to offer caching hints to
+                the user-agent. This is currently undergoing some <a href="https://github.com/slightlyoff/ServiceWorker/issues/585" target="_blank">security review</a>.
+                Firefox exposes the attribute, but it has no effect.</p>
+            <p>Requests have two read-only attributes that are relevant to ServiceWorkers
+            <br/>intercepting them. There is the string <code>referrer</code>, which is
+                set by the UA to be
+                <br/>the referrer of the Request. This may be an empty string. The other is
+                <br/>
+<code>context</code> which is a rather <a href="https://fetch.spec.whatwg.org/#requestcredentials" target="_blank">large enumeration</a> defining
+                what sort of resource is being fetched. This could be “image” if the request
+                is from an
+                &lt;img&gt;tag in the controlled document, “worker” if it is an attempt to load a
+                worker script, and so on. When used with the <code>fetch()</code> function,
+                it is “fetch”.</p>
+            
+<h2>Response</h2>
+
+            <p><code>Response</code> instances are returned by calls to <code>fetch()</code>.
+                They can also be created by JS, but this is only useful in ServiceWorkers.</p>
+            <p>We have already seen some attributes of Response when we looked at <code>fetch()</code>.
+                The most obvious candidates are <code>status</code>, an integer (default
+                value 200) and <code>statusText</code> (default value “OK”), which correspond
+                to the HTTP status code and reason. The <code>ok</code> attribute is just
+                a shorthand for checking that <code>status</code> is in the range 200-299
+                inclusive.</p>
+            <p><code>headers</code> is the Response’s Headers object, with guard “response”.
+                The <code>url</code> attribute reflects the URL of the corresponding request.</p>
+            <p>Response also has a <code>type</code>, which is “basic”, “cors”, “default”,
+                “error” or
+                <br/>“opaque”.</p>
+            <ul>
+                <li><code>"basic"</code>: normal, same origin response, with all headers exposed
+                    except
+                    <br/>“Set-Cookie” and “Set-Cookie2″.</li>
+                <li><code>"cors"</code>: response was received from a valid cross-origin request.
+                    <a href="https://fetch.spec.whatwg.org/#concept-filtered-response-cors" target="_blank">Certain headers and the body</a>may be accessed.</li>
+                <li><code>"error"</code>: network error. No useful information describing
+                    the error is available. The Response’s status is 0, headers are empty and
+                    immutable. This is the type for a Response obtained from <code>Response.error()</code>.</li>
+                <li><code>"opaque"</code>: response for “no-cors” request to cross-origin
+                resource. <a href="https://fetch.spec.whatwg.org/#concept-filtered-response-opaque" target="_blank">Severely<br/>
+  restricted</a>
+                </li>
+            </ul>
+            <p>The “error” type results in the <code>fetch()</code> Promise rejecting with
+                TypeError.</p>
+            <p>There are certain attributes that are useful only in a ServiceWorker scope.
+                The
+                <br/>idiomatic way to return a Response to an intercepted request in ServiceWorkers
+                is:</p>
+            <DIV><pre>addEventListener<span>(</span><span>'fetch'</span><span>,</span> <span>function</span><span>(</span>event<span>)</span> <span>{</span>
+  event.<span>respondWith</span><span>(</span><span>new</span> Response<span>(</span><span>"Response body"</span><span>,</span> <span>{</span>
+    headers<span>:</span> <span>{</span> <span>"Content-Type"</span> <span>:</span> <span>"text/plain"</span> <span>}</span>
+  <span>}</span><span>)</span><span>;</span>
+<span>}</span><span>)</span><span>;</span></pre>
+                            </DIV>
+            <p>As you can see, Response has a two argument constructor, where both arguments
+                are optional. The first argument is a body initializer, and the second
+                is a dictionary to set the <code>status</code>, <code>statusText</code> and <code>headers</code>.</p>
+            <p>The static method <code>Response.error()</code> simply returns an error
+                response. Similarly, <code>Response.redirect(url, status)</code> returns
+                a Response resulting in
+                <br/>a redirect to <code>url</code>.</p>
+            
+<h2>Dealing with bodies</h2>
+
+            <p>Both Requests and Responses may contain body data. We’ve been glossing
+                over it because of the various data types body may contain, but we will
+                cover it in detail now.</p>
+            <p>A body is an instance of any of the following types.</p>
+            <ul>
+                <li><a href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer" target="_blank">ArrayBuffer</a>
+                </li>
+                <li><a href="https://developer.mozilla.org/en-US/docs/Web/API/ArrayBufferView" target="_blank">ArrayBufferView</a> (Uint8Array
+                    and friends)</li>
+                <li><a href="https://developer.mozilla.org/en-US/docs/Web/API/Blob" target="_blank">Blob</a>/
+                    <a href="https://developer.mozilla.org/en-US/docs/Web/API/File" target="_blank">File</a>
+                </li>
+                <li>string</li>
+                <li><a href="https://url.spec.whatwg.org/#interface-urlsearchparams" target="_blank">URLSearchParams</a>
+                </li>
+                <li><a href="https://developer.mozilla.org/en-US/docs/Web/API/FormData" target="_blank">FormData</a> –
+                    currently not supported by either Gecko or Blink. Firefox expects to ship
+                    this in version 39 along with the rest of Fetch.</li>
+            </ul>
+            <p>In addition, Request and Response both offer the following methods to
+                extract their body. These all return a Promise that is eventually resolved
+                with the actual content.</p>
+            <ul>
+                <li><code>arrayBuffer()</code>
+                </li>
+                <li><code>blob()</code>
+                </li>
+                <li><code>json()</code>
+                </li>
+                <li><code>text()</code>
+                </li>
+                <li><code>formData()</code>
+                </li>
+            </ul>
+            <p>This is a significant improvement over XHR in terms of ease of use of
+                non-text data!</p>
+            <p>Request bodies can be set by passing <code>body</code> parameters:</p>
+            <DIV><pre><span>var</span> form <span>=</span> <span>new</span> FormData<span>(</span>document.<span>getElementById</span><span>(</span><span>'login-form'</span><span>)</span><span>)</span><span>;</span>
+fetch<span>(</span><span>"/login"</span><span>,</span> <span>{</span>
+  method<span>:</span> <span>"POST"</span><span>,</span>
+  body<span>:</span> form
+<span>}</span><span>)</span></pre>
+                            </DIV>
+                <p>Responses take the first argument as the body.</p>
+                <DIV><pre><span>var</span> res <span>=</span> <span>new</span> Response<span>(</span><span>new</span> File<span>(</span><span>[</span><span>"chunk"</span><span>,</span> <span>"chunk"</span><span>]</span><span>,</span> <span>"archive.zip"</span><span>,</span>
+                       <span>{</span> type<span>:</span> <span>"application/zip"</span> <span>}</span><span>)</span><span>)</span><span>;</span></pre>
+                                </DIV>
+                <p>Both Request and Response (and by extension the <code>fetch()</code> function),
+                    will try to intelligently <a href="https://fetch.spec.whatwg.org/#concept-bodyinit-extract" target="_blank">determine the content type</a>.
+                    Request will also automatically set a “Content-Type” header if none is
+                    set in the dictionary.</p>
+                
+<h3>Streams and cloning</h3>
+
+                <p>It is important to realise that Request and Response bodies can only be
+                    read once! Both interfaces have a boolean attribute <code>bodyUsed</code> to
+                    determine if it is safe to read or not.</p>
+                <DIV><pre><span>var</span> res <span>=</span> <span>new</span> Response<span>(</span><span>"one time use"</span><span>)</span><span>;</span>
+console.<span>log</span><span>(</span>res.<span>bodyUsed</span><span>)</span><span>;</span> <span>// false</span>
+res.<span>text</span><span>(</span><span>)</span>.<span>then</span><span>(</span><span>function</span><span>(</span>v<span>)</span> <span>{</span>
+  console.<span>log</span><span>(</span>res.<span>bodyUsed</span><span>)</span><span>;</span> <span>// true</span>
+<span>}</span><span>)</span><span>;</span>
+console.<span>log</span><span>(</span>res.<span>bodyUsed</span><span>)</span><span>;</span> <span>// true</span>
+ 
+res.<span>text</span><span>(</span><span>)</span>.<span>catch</span><span>(</span><span>function</span><span>(</span>e<span>)</span> <span>{</span>
+  console.<span>log</span><span>(</span><span>"Tried to read already consumed Response"</span><span>)</span><span>;</span>
+<span>}</span><span>)</span><span>;</span></pre>
+                                </DIV>
+                <p>This decision allows easing the transition to an eventual <a href="https://streams.spec.whatwg.org/" target="_blank">stream-based</a> Fetch
+                    API. The intention is to let applications consume data as it arrives, allowing
+                    for JavaScript to deal with larger files like videos, and perform things
+                    like compression and editing on the fly.</p>
+                <p>Often, you’ll want access to the body multiple times. For example, you
+                    can use the upcoming <a href="http://slightlyoff.github.io/ServiceWorker/spec/service_worker/index.html#cache-objects" target="_blank">Cache API</a> to
+                    store Requests and Responses for offline use, and Cache requires bodies
+                    to be available for reading.</p>
+                <p>So how do you read out the body multiple times within such constraints?
+                    The API provides a <code>clone()</code> method on the two interfaces. This
+                    will return a clone of the object, with a ‘new’ body. <code>clone()</code> MUST
+                    be called before the body of the corresponding object has been used. That
+                    is, <code>clone()</code> first, read later.</p>
+                <DIV><pre>addEventListener<span>(</span><span>'fetch'</span><span>,</span> <span>function</span><span>(</span>evt<span>)</span> <span>{</span>
+  <span>var</span> sheep <span>=</span> <span>new</span> Response<span>(</span><span>"Dolly"</span><span>)</span><span>;</span>
+  console.<span>log</span><span>(</span>sheep.<span>bodyUsed</span><span>)</span><span>;</span> <span>// false</span>
+  <span>var</span> clone <span>=</span> sheep.<span>clone</span><span>(</span><span>)</span><span>;</span>
+  console.<span>log</span><span>(</span>clone.<span>bodyUsed</span><span>)</span><span>;</span> <span>// false</span>
+ 
+  clone.<span>text</span><span>(</span><span>)</span><span>;</span>
+  console.<span>log</span><span>(</span>sheep.<span>bodyUsed</span><span>)</span><span>;</span> <span>// false</span>
+  console.<span>log</span><span>(</span>clone.<span>bodyUsed</span><span>)</span><span>;</span> <span>// true</span>
+ 
+  evt.<span>respondWith</span><span>(</span>cache.<span>add</span><span>(</span>sheep.<span>clone</span><span>(</span><span>)</span><span>)</span>.<span>then</span><span>(</span><span>function</span><span>(</span>e<span>)</span> <span>{</span>
+    <span>return</span> sheep<span>;</span>
+  <span>}</span><span>)</span><span>;</span>
+<span>}</span><span>)</span><span>;</span></pre>
+                                </DIV>
+                
+<h2>Future improvements</h2>
+
+                <p>Along with the transition to streams, Fetch will eventually have the ability
+                    to abort running <code>fetch()</code>es and some way to report the progress
+                    of a fetch. These are provided by XHR, but are a little tricky to fit in
+                    the Promise-based nature of the Fetch API.</p>
+                <p>You can contribute to the evolution of this API by participating in discussions
+                    on the <a href="https://whatwg.org/mailing-list" target="_blank">WHATWG mailing list</a> and
+                    in the issues in the <a href="https://www.w3.org/Bugs/Public/buglist.cgi?product=WHATWG&amp;component=Fetch&amp;resolution=---" target="_blank">Fetch</a> and
+                    <a href="https://github.com/slightlyoff/ServiceWorker/issues" target="_blank">ServiceWorker</a>specifications.</p>
+                <p>For a better web!</p>
+                <p><em>The author would like to thank Andrea Marchesini, Anne van Kesteren and Ben<br/>
+Kelly for helping with the specification and implementation.</em>
+                </p>
+                
+                </article>
+                
+                
+                </DIV></article>

article_scraper/resources/tests/readability/003/expected.html

@@ -0,0 +1,19 @@
+<article><DIV id="readability-page-1"><article>
+      <h2>Test document title</h2>
+        <p>
+          Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+          tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+          quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+          consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+          cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+          proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+        </p>
+	<p>
+          Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+          tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+          quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+          consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+          cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+          proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+        </p>
+      </article></DIV></article>

article_scraper/resources/tests/readability/003/source.html

@@ -0,0 +1,45 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8"/>
+    <title>Title Element</title>
+    <meta name="title" content="Meta name title"/>
+    <meta name="og:title" content="Open Graph name title"/>
+    <meta name="twitter:title" content="Twitter name title"/>
+    <meta name="DC.title" content="Dublin Core name title"/>
+    <meta property="dc:title" content="Dublin Core property title"/>
+    <meta property="twitter:title" content="Twitter property title"/>
+    <meta property="og:title" content="Open Graph property title"/>
+    <meta name="author" content="Meta name author"/>
+    <meta name="DC.creator" content="Dublin Core name author"/>
+    <meta property="dc:creator" content="Dublin Core property author"/>
+     <meta name="description" content="Meta name description"/>
+    <meta name="og:description" content="Open Graph name description"/>
+    <meta name="twitter:description" content="Twitter name description"/>
+    <meta name="DC.description" content="Dublin Core name description"/>
+    <meta property="dc:description" content="Dublin Core property description"/>
+    <meta property="twitter:description" content="Twitter property description"/>
+    <meta property="og:description" content="Open Graph property description"/>
+  </head>
+  <body>
+    <article>
+      <h1>Test document title</h1>
+        <p>
+          Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+          tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+          quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+          consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+          cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+          proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+        </p>
+	<p>
+          Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+          tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+          quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+          consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+          cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+          proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+        </p>
+      </article>
+  </body>
+</html>

article_scraper/resources/tests/readability/aclu/expected.html

@@ -0,0 +1,124 @@
+<article><div id="readability-page-1">
+                                        <p>
+                                            I don't use Facebook. I'm not technophobic — I'm a geek. I've been using email since the early 1990s, I have accounts on hundreds of services around the net, and I do software development and internet protocol design both for work and for fun. I believe that a globe-spanning communications network like the internet can be a positive social force, and I publish much of my own work on the open web.
+                                        </p>
+                                        <p>
+                                            But Facebook and other massive web companies represent a strong push toward unaccountable centralized social control, which I think makes our society more unequal and more unjust. The Cambridge Analytica scandal is one instance of this long-running problem with what I call the "surveillance economy." I don't want to submit to these power structures, and I don’t want my presence on such platforms to serve as bait that lures other people into the digital panopticon.
+                                        </p>
+                                        <p>
+                                            But while I've never "opted in" to Facebook or any of the other big social networks, Facebook still has a detailed profile that can be used to target me. I've never consented to having Facebook collect my data, which can be used to draw very detailed inferences about my life, my habits, and my relationships. As we aim to take Facebook to task for its breach of user trust, we need to think about what its capabilities imply for society overall. After all, if you do #deleteFacebook, you'll find yourself in my shoes: non-consenting, but still subject to Facebook’s globe-spanning surveillance and targeting network.
+                                        </p>
+                                        <p>
+                                            There are at least two major categories of information available to Facebook about non-participants like me: information from other Facebook users, and information from sites on the open web.
+                                        </p>
+                                        <h3>
+                                            <strong>Information from other Facebook users</strong>
+                                        </h3>
+                                        <p>
+                                            When you sign up for Facebook, it encourages you to upload your list of contacts so that the site can "find your friends." Facebook uses this contact information to learn about people, even if those people don't agree to participate. It also links people together based on who they know, even if the shared contact hasn't agreed to this use.
+                                        </p>
+                                        <p>
+                                            For example, I received an email from Facebook that lists the people who have all invited me to join Facebook: my aunt, an old co-worker, a friend from elementary school, etc. This email includes names and email addresses — including my own name — and at least one <a href="https://en.wikipedia.org/wiki/Web_bug" target="_blank">web bug</a> designed to identify me to Facebook’s web servers when I open the email. Facebook records this group of people as my contacts, even though I've never agreed to this kind of data collection.
+                                        </p>
+                                        <p>
+                                            Similarly, I'm sure that I'm in some photographs that someone has uploaded to Facebook — and I'm probably tagged in some of them. I've never agreed to this, but Facebook could still be keeping track.
+                                        </p>
+                                        <p>
+                                            So even if you decide you need to join Facebook, remember that you might be giving the company information about someone else who didn't agree to be part of its surveillance platform.
+                                        </p>
+                                        <h3>
+                                            <strong>Information from sites on the open Web</strong>
+                                        </h3>
+                                        <p>
+                                            Nearly every website that you visit that has a "Like" button is actually encouraging your browser to tell Facebook about your browsing habits. Even if you don't click on the "Like" button, displaying it requires your browser to send a request to Facebook's servers for the "Like" button itself. That request includes <a href="https://en.wikipedia.org/wiki/HTTP_referer" target="_blank">information</a> mentioning the name of the page you are visiting and any Facebook-specific <a href="https://en.wikipedia.org/wiki/HTTP_cookie" target="_blank">cookies</a> your browser might have collected. (See <a href="https://www.facebook.com/help/186325668085084" target="_blank">Facebook's own description of this process</a>.) This is called a "third-party request."
+                                        </p>
+                                        <p>
+                                            This makes it possible for Facebook to create a detailed picture of your browsing history — even if you've never even visited Facebook directly, let alone signed up for a Facebook account.
+                                        </p>
+                                        <p>
+                                            Think about most of the web pages you've visited — how many of them <em>don't</em> have a "Like" button? If you administer a website and you include a "Like" button on every page, you're helping Facebook to build profiles of your visitors, even those who have opted out of the social network. Facebook’s <a href="https://developers.facebook.com/docs/plugins/" target="_blank">“Share” buttons</a> on other sites — along with <a href="https://www.facebook.com/business/learn/facebook-ads-pixel" target="_blank">other tools</a> — work a bit differently from the “Like” button, but do effectively the same thing.
+                                        </p>
+                                        <p>
+                                            The profiles that Facebook builds on non-users don't necessarily include so-called "personally identifiable information" (PII) like names or email addresses. But they do include fairly unique patterns. Using <a href="https://dev.chromium.org/for-testers/providing-network-details" target="_blank">Chromium's NetLog dumping</a>, I performed a simple five-minute browsing test last week that included visits to various sites — but not Facebook. In that test, the PII-free data that was sent to Facebook included information about which news articles I was reading, my dietary preferences, and my hobbies.
+                                        </p>
+                                        <p>
+                                            Given the precision of this kind of mapping and targeting, "PII" isn’t necessary to reveal my identity. How many vegans examine specifications for computer hardware from the ACLU's offices while reading about Cambridge Analytica? Anyway, if Facebook combined that information with the "web bug" from the email mentioned above — which <em>is</em> clearly linked to my name and e-mail address — no guesswork would be required.
+                                        </p>
+                                        <p>
+                                            I'd be shocked if Facebook were not connecting those dots given the goals <a href="https://www.facebook.com/about/privacy/cookies" target="_blank">they claim for data collection</a>:
+                                        </p>
+                                        <blockquote>
+                                            <p>
+                                                We use the information we have to improve our advertising and measurement systems so we can show you relevant ads on and off our Services and measure the effectiveness and reach of ads and services.
+                                            </p>
+                                        </blockquote>
+                                        <p>
+                                            This is, in essence, exactly what Cambridge Analytica did.
+                                        </p>
+                                        <h3>
+                                            <strong>Consent</strong>
+                                        </h3>
+                                        <p>
+                                            Facebook and other tech companies often deflect accusations against excessive data collection by arguing "consent" — that they harvest and use data with the consent of the users involved.
+                                        </p>
+                                        <p>
+                                            But even if we accept that clicking through a "Terms of Service" that <a href="https://tosdr.org/" target="_blank">no one reads</a> can actually constitute true consent, even if we ignore the fact that these terms are overwhelmingly one-sided and non-negotiable, and even if we accept that it's meaningful for people to give consent when sharing data about other people who may have also opted in — what is the recourse for someone who has not opted into these systems at all?
+                                        </p>
+                                        <p>
+                                            Are those of us who have explicitly avoided agreeing to the Facebook terms of service simply fair game for an industry-wide surveillance and targeting network?
+                                        </p>
+                                        <h3>
+                                            <strong>Privilege</strong>
+                                        </h3>
+                                        <p>
+                                            I don’t mean to critique people who have created a Facebook profile or suggest they deserve whatever they get.
+                                        </p>
+                                        <p>
+                                            My ability to avoid Facebook comes from privilege — I have existing social contacts with whom I know how to stay in touch without using Facebook's network. My job does not require that I use Facebook. I can afford the time and expense to communicate with my electoral representatives and political allies via other channels.
+                                        </p>
+                                        <p>
+                                            Many people do not have these privileges and are compelled to "opt in" on Facebook's non-negotiable terms.
+                                        </p>
+                                        <p>
+                                            Many journalists, organizers, schools, politicians, and others who have good reasons to oppose Facebook's centralized social control feel compelled by Facebook's reach and scale to participate in their practices, even those we know to be harmful. That includes the ACLU.
+                                        </p>
+                                        <p>
+                                            Privacy should not be a luxury good, and while I'm happy to encourage people to opt out of these subtle and socially fraught arrangements, I do not argue that anyone who has signed up has somehow relinquished concerns about their privacy. We need to evaluate privacy concerns in their full social contexts. These are not problems that can be resolved on an individual level, because of the interpersonal nature of much of this data and the complexities of the tradeoffs involved.
+                                        </p>
+                                        <h3>
+                                            <strong>Technical countermeasures</strong>
+                                        </h3>
+                                        <p>
+                                            While they may not solve the problem, there are some technical steps people can take to limit the scope of these surveillance practices. For example, some web browsers do not send "third-party cookies" by default, or <a href="https://wiki.mozilla.org/Thirdparty" target="_blank">they scope cookies</a> so that centralized surveillance doesn't get a single view of one user. The most privacy-preserving modern browser is <a href="https://www.torproject.org/" target="_blank">the Tor Browser</a>, which everyone should have installed and available, even if it's not the browser they choose to use every day. It limits the surveillance ability of systems that you have not signed up for to track you as you move around the web.
+                                        </p>
+                                        <p>
+                                            You can also modify some browsers — for example, with plug-ins for <a href="https://requestpolicycontinued.github.io/" target="_blank">Firefox</a> and <a href="https://chrome.google.com/webstore/detail/umatrix/ogfcmafjalglgifnmanfmnieipoejdcf" target="_blank">Chrome</a> — so that they <a href="https://addons.mozilla.org/en-US/firefox/addon/umatrix/" target="_blank">do not send third-party</a> <a href="https://requestpolicycontinued.github.io/" target="_blank">requests at all</a>. Firefox is also exploring even more <a href="https://addons.mozilla.org/en-US/firefox/addon/multi-account-containers/" target="_blank">privacy-preserving techniques</a><a href="https://addons.mozilla.org/en-US/firefox/addon/multi-account-containers/" target="_blank">.</a>
+                                        </p>
+                                        <p>
+                                            It can’t be denied, though, that these tools are harder to use than the web browsers most people are accustomed to, and they create barriers to some online activities. (For example, logging in to <a href="https://offcampushousing.uconn.edu/login" target="_blank">some sites</a> and accessing some <a href="https://filestore.community.support.microsoft.com/api/images/0253d8fb-b050-401a-834d-9d80a99c0b12" target="_blank">web applications</a> is impossible without third-party cookies.)
+                                        </p>
+                                        <p>
+                                            Some website operators take their visitors' privacy more seriously than others, by reducing the amount of third-party requests. For example, it's possible to display "share on Facebook" or "Like" buttons without sending user requests to Facebook in the first place. The ACLU's own website does this because we believe that the right to read with privacy is a fundamental protection for civic discourse.
+                                        </p>
+                                        <p>
+                                            If you are responsible for running a website, try browsing it with a third-party-blocking extension turned on. Think about how much information you're requiring your users to send to third parties as a condition for using your site. If you care about being a good steward of your visitors' data, you can re-design your website to reduce this kind of leakage.
+                                        </p>
+                                        <h3>
+                                            <strong>Opting out?</strong>
+                                        </h3>
+                                        <p>
+                                            Some advertisers claim that you can "opt out" of their targeted advertising, and even offer <a href="http://optout.aboutads.info/" target="_blank">a centralized place meant to help you do so</a>. However, my experience with these tools isn't a positive one. They don't appear to work all of the time. (In a recent experiment I conducted, two advertisers’ opt-out mechanisms failed to take effect.) And while advertisers claim to allow the user to opt out of "interest-based ads," it's not clear that the opt-outs govern data collection itself, rather than just the use of the collected data for displaying ads. Moreover, opting out on their terms requires the use of third-party cookies, thereby enabling another mechanism that other advertisers can then exploit.
+                                        </p>
+                                        <p>
+                                            It's also not clear how they function over time: How frequently do I need to take these steps? Do they expire? How often should I check back to make sure I’m still opted out? I'd much prefer an approach requiring me to opt <em>in</em> to surveillance and targeting.
+                                        </p>
+                                        <h3>
+                                            <strong>Fix the surveillance economy, not just Facebook</strong>
+                                        </h3>
+                                        <p>
+                                            These are just a few of the mechanisms that enable online tracking. Facebook is just one culprit in this online "surveillance economy," albeit a massive one — the company owns <a href="https://www.instagram.com/" target="_blank">Instagram</a>, <a href="https://atlassolutions.com/" target="_blank">Atlas</a>, <a href="https://www.whatsapp.com/" target="_blank">WhatsApp</a>, and dozens of other internet and technology companies and services. But it’s not the only player in this space. Google’s business model also relies on this kind of surveillance, and there are dozens of smaller players as well.
+                                        </p>
+                                        <p>
+                                            As we work to address the fallout from the current storm around Facebook and Cambridge Analytica, we can't afford to lose sight of these larger mechanisms at play. Cambridge Analytica's failures and mistakes are inherent to Facebook's business model. We need to seriously challenge the social structures that encourage people to opt in to this kind of surveillance. At the same time, we also need to protect those of us who manage to opt out.
+                                        </p>
+                                    </div></article>

article_scraper/resources/tests/readability/aktualne/expected.html

@@ -0,0 +1,74 @@
+<article><DIV id="readability-page-1"><p>
+                                        Zázrak jedné sezony? West Ham dává pochybovačům stále pádnější odpovědi a fotbalový svět si začíná uvědomovat, že se absolutní anglická fotbalová elita rozrůstá o nového člena. Tým manažera Davida Moyese prohání giganty i v aktuálním ročníku Premier League.
+                                    </p><div id="article-content">
+                                        <p>
+                                            Pět vítězných soutěžních duelů v řadě, během nich jediný inkasovaný gól. Čtvrté místo v lize, stejný bodový zisk jako loňský šampion Manchester City a nadšené ohlasy z tábora těch nejrenomovanějších komentátorů ostrovního fotbalu.
+                                        </p>
+                                        
+                                        <p>
+                                            West Ham je opět v kurzu, nadšené ohlasy po nedělní jasné výhře 4:1 na hřišti Aston Villy zaplnily anglický mediální prostor.
+                                        </p>
+                                        <p>
+                                            "Stali se excelentním týmem. Jsou skvělí ve všech částech hřiště a David Moyes si zaslouží obrovský kredit za to, do jaké pozice je dostal," píše na Twitter Gary Lineker.
+                                        </p>
+                                        <p>
+                                            "Nenapadá mě jediný důvod, proč by letos nemohli skončit v elitní čtyřce," přidává se Emile Heskey, někdejší útočník Liverpoolu. "Je fér říct, že vypadají fantasticky. Moyes je neskutečně oživil."
+                                        </p>
+                                        <p>
+                                            I Heskey si všiml, že se Kladiváři skvěle vyrovnávají s náročným programem a pro ně novou rolí: účastí ve více soutěžích najednou. Moyes zůstává konzervativní v určování základní sestavy, chytře ale rozšířil kádr a v Evropské lize či ligovém poháru nechává některé opory odpočívat. Výjimkou potvrzující pravidlo je přitom Tomáš Souček, o jehož nezbytnosti bude řeč níže.
+                                        </p>
+                                        <p>
+                                            "Klíčová věc je ta, že když udělá změny, pořád jim zůstává stejná struktura. To je něco, co pravidelně říkáme třeba o Manchesteru City. Ve hře neustále zůstává nějaká fundamentální filosofie. West Ham to má podobně a už kvůli tomu je třeba před Moyesem smeknout," přirovnává Heskey.
+                                        </p>
+                                        <p>
+                                            <em>Podívejte se na důležité momenty zápasu Aston Villa - West Ham:</em>
+                                        </p>
+                                        
+                                        
+                                        <p>
+                                            V Evropské lize má West Ham po třech zápasech plný bodový zisk. V anglickém ligovém poháru dobyl čtvrtfinále, když vyřadil oba bohaté velkokluby z Manchesteru.
+                                        </p>
+                                        <p>
+                                            Čeští fotbalisté nicméně momentálně nejsou ve světlech těch nejjasnějších reflektorů.
+                                        </p>
+                                        <p>
+                                            Vladimír Coufal už sice uzdravil poraněné tříslo, v sestavě ale před ním dostal přednost rozjetý Ben Johnson. Anglický mladík další působivé představení okořenil parádním gólem a potvrdil, že se stává tvrdou konkurencí pro českého reprezentačního beka.
+                                        </p>
+                                        <p>
+                                            Tomáš Souček zůstává nepostradatelným členem základní sestavy, navzdory tomu, že jeho poslední výkony působí nenápadně.
+                                        </p>
+                                        <p>
+                                            "Pořád toho odvádí strašnou spoustu mimo hlavní pozornost. Jsou to důležité věci, které je snadné přehlédnout," píše ve svém hodnocení server Claret and Hugh. 
+                                        </p>
+                                        
+                                        <p>
+                                            "S Declanem Ricem vytvořil silné partnerství a udělal spoustu těžké práce. Má dobrou rozehrávku. Jediné, na co si lze stěžovat, jsou jeho občasná špatná rozhodnutí ve finální třetině hřiště," hodnotí českého středopolaře londýnský večerník Evening Standard.
+                                        </p>
+                                        <p>
+                                            Web Football.London to vidí podobně. "Opět byl silný ve vzduchu, na obou koncích hřiště. Ve finální fázi se ale nerozhodoval dobře, příliš často volil špatnou variantu."
+                                        </p>
+                                        <p>
+                                            Moyes nicméně nenechává Součka oddechnout. V pěti posledních utkáních, které West Ham odehrál během pouhých čtrnácti dnů, chyběl Čech jen pár minut v závěru na Evertonu, když utrpěl zranění v obličeji.
+                                        </p>
+                                        <p>
+                                            Fanoušci pravidelně spekulují o únavě, skotský manažer ale - jak se zdá - bude mít v sestavě raději unaveného Součka než kohokoli jiného. Zvlášť, když Alex Král, plánovaný back-up do středu zálohy, stále není k dispozici.
+                                        </p>
+                                        <p>
+                                            Zatímco v minulé sezoně Souček častokrát zastínil svého kolegu Rice, letos je to právě anglický reprezentant, kdo si užívá zasloužené ódy na svou adresu.
+                                        </p>
+                                        <p>
+                                            "Hraje prostě velkolepě a připomínám, že je mu stále jen dvaadvacet let," kroutí hlavou Lineker. Není sám. Ještě před pár měsíci se většina odborníků pozastavovala nad údajnou cenovkou kolem 100 milionů liber. Nyní už zaznívají hlasy o tom, jak může být i tato hranice při případném přestupu Declana Rice výrazně překročena.
+                                        </p>
+                                        
+                                        <p>
+                                            S blížícím se zimním přestupním termínem budou spekulace nabývat na síle, fanoušci Hammers ale věří, že Rice zůstane nejméně do léta. Jeho spokojenost je do očí bijící, stejně jako ochota nechat na hřišti všechno ve prospěch Clarets and Blues.
+                                        </p>
+                                        <p>
+                                            "Náš kolektiv je teď opravdu speciální. Působíme ve výjimečném prostředí. Každé ráno se probouzíme s obrovskou touhou po dalším tréninku. Jsme nadšení," tvrdí mladá anglická superstar.
+                                        </p>
+                                        <p>
+                                            "Jsme na děleném třetím místě. Lidé se před sezonou hodně ptali, zda to můžeme dokázat znovu. Ukázali jsme, že ano. Ale musíme pokračovat. Tohle musí být náš standard. Nesmíme polevit, pokud chceme být velkým týmem," zdůrazňuje Rice.
+                                        </p>
+                                    </div><p>
+                                            Pokud jste v článku zaznamenali chybu nebo překlep, dejte nám, prosím, vědět prostřednictvím <a href="https://ankety.aktualne.cz/s3/00310d93156a?utm_source=aktualne.cz&amp;utm_medium=upozorneni&amp;from=https%3A%2F%2Fsport.aktualne.cz%2Ffotbal%2Fzahranici%2Fwest-ham-hrozi-gigantum-okouzlil-i-linekera-souckovu-praci-j%2Fr~8fa032ba3add11ec8a900cc47ab5f122%2F" target="_blank">kontaktního formuláře</a>. Děkujeme!
+                                        </p></DIV></article>

article_scraper/resources/tests/readability/archive-of-our-own/expected.html

@@ -0,0 +1,317 @@
+<article><div role="article" id="readability-page-1">
+                                        <h3 id="work">
+                                            Chapter Text
+                                        </h3>
+                                        <p>
+                                            Izuku was struggling to understand how he had even managed to get here, seated before the archvillain of Japan with only a sense of dread to keep him company. All Might sat concealed in an observation room, of the firm opinion that he could only aggravate the prisoner and he sent Izuku off with a strained smile. A vague haze hovered over Izuku’s memory. It started with a simple conversation gone astray on a long drive home.
+                                        </p>
+                                        
+                                        <p>
+                                            “So, who is All For One? Do we know anything about him beyond what you told me before? He’s been imprisoned for months now.” Izuku remembered asking All Might from the backseat of the car as Detective Tsukauchi leisurely drove along a sprawling highway.
+                                        </p>
+                                        <p>
+                                            Playing on the car radio was an aftermath report of a villain attack in downtown Tokyo. Izuku caught the phrase “liquid body” from the female reporter before Detective Tsukauchi changed the channel.
+                                        </p>
+                                        <p>
+                                            “Nope. Still nothing. No one really wants to speak to him,” All Might had replied brightly. “He gives off polite airs, but he’s a piece of work.” All Might’s mostly obstructed shoulders in the front seat shrugged. “Not much you can do with someone like him. Everything that comes out is a threat or taunt.” All Might carefully waved his hand in a circular motion towards the side of his head.
+                                        </p>
+                                        <p>
+                                            “No one’s even made it through a full interview with him, from what I’ve heard,” Detective Tsukauchi added from behind the wheel. “He plays mind games with them. The prison also has a “no recent events” policy on any discussions with him as well. Just in case he ends up with ideas or has some means of communicating. Given that people only want to ask him about current events, it doesn’t leave much to talk about.”
+                                        </p>
+                                        <p>
+                                            “Wait, they still don’t know what Quirks he has?” Izuku asked exasperatedly. “They can’t if there’s still an information block on visits.”
+                                        </p>
+                                        <p>
+                                            “Nope. We have no idea what he can do. They can run DNA tests, but it’s not like anyone apart from him even knows how his Quirk works. They could get matches with any number of people, but if they’re not in a database then we can’t cross-reference them anyway. Even if they run an analysis, the data doesn’t mean anything without the ability to interpret it,” All Might gestured with a skeletal finger. “It’s a waste of time after the initial tests were conducted. They weren’t game to MRI him either, given he’s definitely got a Quirk that creates metal components.”
+                                        </p>
+                                        <p>
+                                            “No one’s bothered to ask him anything about… anything?” Izuku asked, dumbfounded. “He must be around two-hundred years old and people can’t think of a single non-current affairs thing to ask him?”
+                                        </p>
+                                        <p>
+                                            In some ways it was unfathomable that they’d let a potential resource go to waste. On the other hand, said potential resource had blown up a city, murdered numerous people and terrorised Japan for over a century. At the very least.
+                                        </p>
+                                        <p>
+                                            “Well, I tried to ask him about Shigaraki, but he didn’t say much of anything really. Some garbage about you being too dependent on me and him letting Shigaraki run wild and how he just wanted to be the ultimate evil,” All Might shrugged again. “He spends too much time talking about nothing.”
+                                        </p>
+                                        <p>
+                                            Izuku shifted his head onto his arm. “But, that’s not really nothing, is it?”
+                                        </p>
+                                        <p>
+                                            “What do you mean?” Izuku had the feeling that All Might would have been looking at him with the <i>you’re about to do something stupid aren’t you</i> expression that was thankfully becoming less common.
+                                        </p>
+                                        <p>
+                                            “Well, he clearly doesn’t know anything about us, All Might, if he thinks that you’re just going to let go of me after not even two years of being taught. Maybe Shigaraki was dependent on adult figures, but I don’t even remember my dad and mum’s been busy working and keeping the house together. I’ve never had a lot of adult supervision before,” Izuku laughed nervously. “I had to find ways to keep myself entertained. If anything, I’m on the disobedient side of the scale.” All Might outright giggled.
+                                        </p>
+                                        <p>
+                                            “I’ll say, especially after what happened with Overhaul. I’m surprised your mother let you leave the dorms again after that.”
+                                        </p>
+                                        <p>
+                                            “I’m surprised she didn’t withdraw and ground me until I was thirty.”
+                                        </p>
+                                        <p>
+                                            “Oh? That strict?” Tsukauchi asked.
+                                        </p>
+                                        <p>
+                                            “She has her moments,” Izuku smiled fondly. “Do you think she’d agree to me asking the archvillain of Japan about his Quirk?” Izuku asked, only partially joking. There was an itch at the back of his head, a feeling of something missing that poked and prodded at his senses.
+                                        </p>
+                                        <p>
+                                            All Might coughed and sprayed the dash with a fine red mist. “Absolutely not! I forbid it!”
+                                        </p>
+                                        <p>
+                                            “That’s exactly why I’m asking her and not you,” Izuku grinned from the backseat.
+                                        </p>
+                                        <p>
+                                            “He’s evil!”
+                                        </p>
+                                        <p>
+                                            “He’s ancient. You honestly don’t wonder about the sort of things someone with that life experience and Quirk would have run across to end up the way he did?”
+                                        </p>
+                                        <p>
+                                            “Nope, he made it perfectly clear that he always wanted to be the supreme evil,” All Might snipped through folded arms.
+                                        </p>
+                                        <p>
+                                            “Yeah, and I’ll just take his word for that, won’t I?” Izuku grinned. “If he does nothing but lie, then that’s probably one too, but there’s a grain of truth in there somewhere.”
+                                        </p>
+                                        <p>
+                                            “What would you even do? Harass him into telling you his life story?” All Might sighed.
+                                        </p>
+                                        <p>
+                                            “Not when I can kill him with kindness. Who knows, it might even be poisonous for him.”
+                                        </p>
+                                        <p>
+                                            “You’re explaining this to your mother. Teacher or not, I’m not being on the receiving end of this one.”
+                                        </p>
+                                        <p>
+                                            Izuku blinked for a moment. “You’ll let me?”
+                                        </p>
+                                        <p>
+                                            “I’m not entirely for it, but any prospective information on what influenced Shigaraki can only be a good thing. If anything goes south we can pull you out pretty easily. Just be aware of who and what you’re dealing with.” Struggling, All Might turned a serious look to Izuku around the side of the seat. “<i>Only</i> if your mother gives the okay.”
+                                        </p>
+                                        <p>
+                                            The conversation turned to school for the rest of the way.
+                                        </p>
+                                        
+                                        <p>
+                                            It might have been curiosity or it might have been the nagging sensation that chewed at his brain for the three weeks that he researched the subject of the conversation. All For One was a cryptid. Mystical in more ways than one, he was only a rumour on a network that was two-hundred years old. There were whispers of a shadowy figure who once ruled Japan, intermingled with a string of conspiracies and fragmented events.
+                                        </p>
+                                        <p>
+                                            Izuku had even braved the dark web, poking and prodding at some of the seedier elements of the world wide web. The internet had rumours, but the dark web had stories.<br/>
+                                        </p>
+                                        <p>
+                                            An implied yakuza wrote about his grandfather who lost a fire manipulation Quirk and his sanity without any reason. His grandfather had been institutionalised, crying and repeating “he took it, he took it” until his dying days. No one could console him.
+                                        </p>
+                                        <p>
+                                            Another user spoke of a nursing home where a room full of dementia residents inexplicably became docile and no longer used their Quirks on the increasingly disturbed staff. The nursing home erupted into flames just before a court case against them commenced.
+                                        </p>
+                                        <p>
+                                            A user with neon pink text spoke of how their great-great-great-great grandmother with a longevity Quirk had simply aged rapidly one day and passed away in her sleep, her face a mask of terror. No cause had ever been found.
+                                        </p>
+                                        <p>
+                                            A hacker provided a grainy CCTV recording of a heist and a scanned collection of documents from over a century ago, where there was a flash of light and entire bank vault had been emptied. What separated it from the usual robbery was that it contained a list containing confidential information on the Quirks of the First Generation. Izuku had greedily snavelled up and saved the video and documents to an external hard drive.
+                                        </p>
+                                        <p>
+                                            Paging through, Izuku saw someone recount how their Quirkless uncle had developed a warp Quirk and gone from rags to riches under a mysterious benefactor. A decade ago, the uncle had simply disappeared.
+                                        </p>
+                                        <p>
+                                            Numerous and terrifying, the stories were scattered nuggets of gold hidden across the web. They’d never last long, vanishing within hours of posting. Izuku bounced from proxy to proxy, fleeing from a series of deletions that seemed to follow Izuku’s aliased postings across snitch.ru, rabbit.az, aconspiracy.xfiles and their compatriots.
+                                        </p>
+                                        <p>
+                                            After thirty-two identity changes (all carefully logged in a separate notebook), a large amount of feigning communal interest in a lucky tabloid article on All For One which had been released at the start of the first of the three weeks, Izuku hung up his tinfoil hat and called it a month. He haphazardly tossed a bulging notebook into his bookshelf and lodged his hard drive in a gap containing seven others and went to dinner.
+                                        </p>
+                                        <p>
+                                            It took another week to present his research to All Might and Tsukauchi, whose jaws reached the proverbial floor.
+                                        </p>
+                                        <p>
+                                            “We never found any of this,” the Detective Tsukauchi exclaimed. “How did you find all of it?”
+                                        </p>
+                                        <p>
+                                            “I asked the right people. Turns out criminals have very long and very unforgiving memories,” Izuku explained through sunken eyes. “There’s more than this that could be linked to him, but these ones seem to be the most obvious.”
+                                        </p>
+                                        <p>
+                                            “They would do, you can’t be head of the underworld without making an army of enemies,” All Might agreed. “You know, if you can get any more information about these events, I think you’ll give people a lot of peace of mind.”
+                                        </p>
+                                        <p>
+                                            “Provided mum agrees to it.”
+                                        </p>
+                                        <p>
+                                            “Only if she agrees to it.”
+                                        </p>
+                                        <p>
+                                            It took another month to convince his mother, who eventually gave in once All Might provided an extremely comprehensive schedule of how the visitations and any resulting research would be carefully balanced against Izuku’s schoolwork and internship.
+                                        </p>
+                                        
+                                        <p>
+                                            The day of the visit finally arrived, four months after the initial conversation, much to Izuku’s dismay.
+                                        </p>
+                                        <p>
+                                            Izuku remembered how he had arrived, with the Detective and All Might escorting him through its sterile, white innards. A list of rules rattled off at the gate, “no current affairs” was chief among them and an assertion that he’d be dragged from the room if need be if Izuku was to breach any of them. No smuggling of communication devices, no weapons, no Quirks, nothing that could compromise the prisoner’s secure status.
+                                        </p>
+                                        <p>
+                                            Heavily armoured and drilled guards leading him underground into the deepest bowels of the Tartarus complex.
+                                        </p>
+                                        <p>
+                                            Izuku understood the rules, dressed casually in a cotton t-shirt with “Shirt” printed across it in haphazard English and clutching at a carefully screened and utterly blank notebook.
+                                        </p>
+                                        <p>
+                                            Across from him, behind reinforced glass, the archvillain of Japan was bound and unmoving.
+                                        </p>
+                                        <p>
+                                            “Hello,” Izuku initiated uncertainly. His skin had been crawling the moment he crossed the threshold, a memory of the encounter and escape at the Kamino Ward months ago.
+                                        </p>
+                                        <p>
+                                            “Ah, All Might’s disciple,” drawled All For One, “is he too cowardly to come himself? Yet I don’t hear the garments of a hero.” With hardly a word out, All For One had already lunged for the figurative jugular.
+                                        </p>
+                                        <p>
+                                            A stray thought of <i>how does he know who I am if he’s blind and isn’t familiar with me?</i> whispered its way through Izuku’s head.
+                                        </p>
+                                        <p>
+                                            “Oh, no,” Izuku corrected hastily, almost relieved at the lack of any pretence, “I asked if I could talk to you. This isn’t exactly hero related.”
+                                        </p>
+                                        <p>
+                                            “I’m surprised he said yes.” While there was little by way of expression, Izuku could just about sense the contempt dripping from the prisoner’s tone. It wasn’t anything he wasn’t expecting. Kacchan had already said worse to him in earlier years. Water off a duck’s back.
+                                        </p>
+                                        <p>
+                                            “Well, he’s not my legal guardian, so I think you should be more surprised that mum said yes. She’s stricter with these things than All Might,” Izuku corrected again. “Mum gave the okay, but that was a stressful discussion.” And there it was, a miniscule twitch from the man opposite. A spasm more than anything else. <i>Interesting.</i> Pinned down as he was, the prisoner oozed irritation.
+                                        </p>
+                                        <p>
+                                            “At least your mother is a wise person. I wonder why the student doesn’t heed all of the advice of the teacher.” All For One’s tone didn’t indicate a question, so much as an implicit statement that All Might wasn’t worth listening to in any capacity. Kacchan would have hated the comparison, but the hostility had an almost comfortable familiarity. “He no doubt warned you off speaking to me, overprotective as he is, but here you are.”
+                                        </p>
+                                        <p>
+                                            Izuku found himself smiling at the thought of Kacchan’s outrage if he ever found out about the mental comparison as he replied. “I don’t think it’s normal for anyone my age to listen completely to their teachers. We pick and choose and run with what works best for us. He warned me, but I’m still here. Mum warned me as well, but I think she cared more about the time management aspect of it."
+                                        </p>
+                                        <p>
+                                            “Is that a recent development?” All For One probed.
+                                        </p>
+                                        <p>
+                                            “Not really. My old homeroom teacher told me not to bother applying to U.A.” His mother’s beaming face had carried Izuku through the cheerful and resolute signing of that application form.
+                                        </p>
+                                        <p>
+                                            “I see you followed their advice to the letter,” came the snide, dismissive reply.
+                                        </p>
+                                        <p>
+                                            Izuku hoisted up his legs and sat cross-legged in his seat. Leaning slightly forward as he did so as to better prop up his notebook.
+                                        </p>
+                                        <p>
+                                            “You’re a walking contrarian, aren’t you? All Might told me about his run ins with you. What someone does or doesn’t do really doesn’t matter to you, you’ll just find a way to rationalise it as a negative and go on the attack anyway. What you’re currently doing is drawing attention away from yourself and focusing it on me so you can withhold information.” Izuku flipped open his notebook and put pen to paper. “You’ve got something fairly big to hide and you diverting attention exposes that motivation as existing anyway. The only real questions here are what and why?” Izuku paused in mortification as the man opposite’s lips parted. “I just said that aloud, didn’t I?”
+                                        </p>
+                                        <p>
+                                            Of the responses Izuku had expected, it wasn’t laughter. Unrestrained, Izuku would have expected a violent outburst. In this situation, he would have expected another scathing comment. Instead, All For One laughed breathily, leaning into his bonds. Wheezingly he spoke, “I’ll have to change tactics, if that one’s too transparent for you. How refreshing.”
+                                        </p>
+                                        <p>
+                                            Doing his best not to glow a blinding red and simultaneously pale at the interest, Izuku carried on. “I add it to the list when you do. I’m not emotionally involved enough to really be impacted by what you’re saying. I know about you in theory, but that’s it. Maybe All Might has a history with you, but I don’t really know enough about you personally to…”
+                                        </p>
+                                        <p>
+                                            “Care,” All For One supplied, somewhat subdued as he struggled to breathe. “You’re only here to satisfy your curiosity as to whether or not the stories were true.”
+                                        </p>
+                                        <p>
+                                            Izuku nodded, scratching at his notebook with his left hand. “Yes and no, I’m actually here to ask you about how your Quirk works.” <i>For now.</i>
+                                        </p>
+                                        <p>
+                                            Another chortle, more restrained that the last.
+                                        </p>
+                                        <p>
+                                            "What makes you think others haven’t already asked?” Had All For One been unrestrained, Izuku could imagine the stereotypical scene of the villain confidently leaning back in some overblown chair in a secret lair, drink of choice in hand, if the tone of voice was any indication. Deflections aside, the man easily rose to each comment.
+                                        </p>
+                                        <p>
+                                            “Whether or not they asked it’s irrelevant if they can’t read the answers.” Answers didn’t matter if the people involved were too attached to read into the answers. If none of the interviewers had managed a full interview, then it seemed unlikely that any sort of effort was put into understanding the villain.
+                                        </p>
+                                        <p>
+                                            “And you think you can? What expertise do you hold above theirs?” Doubt and reprimand weighted the words. Oddly enough, had Izuku been any younger he could have mistaken the man for a disapproving parent rebuking an overly ambitious child. Albeit an extremely evil one.
+                                        </p>
+                                        <p>
+                                            Izuku inhaled shortly and went for it. “If there’s something I know, it’s Quirks and how they work. Maybe I don’t know you, but I don’t really need to. Quirks fall under broad categories of function. You can take and give, consent doesn’t seem to be a factor. You either can’t “see” certain types of Quirks or you need to have prior knowledge of it before you take it with what I know about your brother. Despite your <i>nom de guerre</i>, because we both know it’s not your real name, you have a history of giving multiple Quirks and causing brain damage to the receiver. You clearly aren’t impacted by those same restrictions, so it must either alter your brain mapping or adjust functions to allow for simultaneous use and storage. It also must isolate or categories the Quirks you stock, because from the few people who do remember you, you creating certain Quirks is always in the context of giving them to someone else meaning there’s probably an inherent immunity to stop it from tainting your own Quirk with a mutation,” Izuku mumbled, almost to himself. “The only thing really in question about your Quirk is the finer details and whether or not you need to maintain those features or if they’re inherent and your hard limit for holding Quirks.”
+                                        </p>
+                                        <p>
+                                            There was silence, for only a moment. “If only my hands were free, I would clap for such a thoughtful assessment. Clearly you’re not all brawn,” All For One positively purred. “Speculate away.” A wide and slightly unhinged smile was directed at Izuku.
+                                        </p>
+                                        <p>
+                                            It was all Izuku could do not to wince at the eagerness. An image of a nervous All Might, hidden in the observation room above with the grim-faced prison staff, came to mind.
+                                        </p>
+                                        <p>
+                                            “I note that you said thoughtful and not correct,” and Izuku breathed and unsteadily jotted it down in his notebook. “You don’t seem bothered by the guess.”
+                                        </p>
+                                        <p>
+                                            “Few people live long enough to question my Quirk, let alone have the talent to guess so thoughtfully at its functions. It seems we share a hobby.” There was something terribly keen in that voice that hadn’t been there before, twisting itself through the compliment.
+                                        </p>
+                                        <p>
+                                            “I suppose it helps that you’re playing along out of boredom,” Izuku verbally dodged, unease uncoiling itself from the back of his mind.
+                                        </p>
+                                        <p>
+                                            “I <i>was</i> playing along out of boredom,” All For One corrected smoothly. “Now, I’m curious. Admittedly, my prior assumptions of you weren’t generous, but I’ve been too hasty in my assessments before.”
+                                        </p>
+                                        <p>
+                                            “I’ll pack up and leave now if that’s the case,” Izuku replied with only half an ear on the conversation as the words on his page began to drastically expand to distract himself from the building anxiety.
+                                        </p>
+                                        <p>
+                                            “Sarcasm, so you do have characteristics of a normal teenager. Your willingness to maim yourself has often left me wondering…”
+                                        </p>
+                                        <p>
+                                            “You’re deflecting again,” Izuku observed. “I’m not sure if that’s a nervous habit for you or if you’re doing it because I’m close to being right about your Quirk. That being said, I don’t think you know what a normal teenager is if Shigaraki is any indication. He’s about seven years too late for his rebellious phase.”
+                                        </p>
+                                        <p>
+                                            “I’m hurt and offended,” came the amused reply.
+                                        </p>
+                                        <p>
+                                            “By how Shigaraki ended up or your parenting? You only have yourself to blame for both of them.”
+                                        </p>
+                                        <p>
+                                            “How harsh. Shigaraki is a product of society that birthed him. I can’t take credit for all of the hard work,” All For One laid out invitingly. Perhaps someone else would have risen to the bait, but Izuku was already packing his mental bags and heading for the door.
+                                        </p>
+                                        <p>
+                                            Clearly the prisoner’s anticipation had registered poorly with someone in the observation room, because a voice rang through the air. “Time’s up Midoriya-kun.”
+                                        </p>
+                                        <p>
+                                            “Okay!” Izuku called back and etched out his last thoughtful of words, untangled his legs and rose to his feet.
+                                        </p>
+                                        <p>
+                                            “What a shame, my visitations are always so short,” All For One spoke mournfully.
+                                        </p>
+                                        <p>
+                                            “Well, you did blow up half a city. They could have just let you suffocate instead. Same time next week, then?” Izuku offered brightly, notebook stuffed into a pocket and was followed out the door by wheezing laughter.
+                                        </p>
+                                        <p>
+                                            It was only after he had made it safely back to the communal room where All Might waited did he allow the spring to fade from his step and discard his nervous smile. Shuddering, he turned to All Might whose face was set in a grimace.
+                                        </p>
+                                        <p>
+                                            “I won’t say I told you so,” All Might offered, perched on the edge of his couch like a misshapen vulture.
+                                        </p>
+                                        <p>
+                                            “He’s… not really what I was expecting. I was expecting someone, more openly evil.” Izuku allowed himself to collapse into the leather of the seat. He shakily reached for the warm tea that had been clearly been prepared the moment Izuku left the cell. “I suppose he does it to lull people into a false sense of security. I didn’t understand how someone with only half a set of expressions could have “villain” written all over them until I met him.”
+                                        </p>
+                                        <p>
+                                            “He’s always been like that. He feigns concern and sympathy to lure in society’s outcasts. They’re easy targets,” All Might said through a mouthful of biscuit.
+                                        </p>
+                                        <p>
+                                            “Has he ever tried it on any of the One For All successors?”
+                                        </p>
+                                        <p>
+                                            “Not really, but you might have accidentally given him the incentive for it. He never had access to any of the One For All wielders while they were young.” All Might snorted, “not that it’ll make a difference with you”.
+                                        </p>
+                                        <p>
+                                            “I think he was trying to gauge me for a world view before the wardens ended it. I need more time to work out his response to the stuff on his Quirk.”
+                                        </p>
+                                        <p>
+                                            “He’s conversation starved since it’s solitary confinement. If what the people monitoring his brain activity said was true, you’re the most exciting thing to have happened to him in months. He replied after you left, said he was looking forward to it.”
+                                        </p>
+                                        <p>
+                                            “That’s pretty sad."
+                                        </p>
+                                        <p>
+                                            “It’s even sadder that we’re the only two members of the public who have had anything to do with him. Stain gets a pile of mail from his “fans”, but All For One has nothing,” All Might waved a tea spoon. “That’s what he gets.”
+                                        </p>
+                                        <p>
+                                            “Let’s get out of here and tell Detective Tsukauchi how it went.” Izuku gulped down his tea and headed for the exit, with him and All Might reaching it at roughly the same amount of time.
+                                        </p>
+                                        <p>
+                                            “At least your mum’s making katsudon for us tonight," was All Might's only optimistic comment.
+                                        </p>
+                                        <p>
+                                            Anxiety was still ebbing over Izuku after Tsukauchi had been debriefed in the car.
+                                        </p>
+                                        <p>
+                                            <i>“It seems we share a hobby.”</i> Haunted Izuku on the drive home. As if ripping someone’s Quirk from them and leaving them lying traumatised on the ground was just a fun pastime and not an act of grievous bodily harm.
+                                        </p>
+                                        <p>
+                                            And he’d be dealing with him again in another week.
+                                        </p>
+                                    </div></article>

article_scraper/resources/tests/readability/ars-1/expected.html

@@ -0,0 +1,89 @@
+<article><div id="readability-page-1">
+                            <header>
+                                <h4>
+                                    Biz &amp; IT —
+                                </h4>
+                                
+                                <h2 itemprop="description">
+                                    Two-year-old bug exposes thousands of servers to crippling attack.
+                                </h2>
+                                
+                            </header>
+                            <div itemprop="articleBody">
+                                    <figure>
+                                        <img src="https://cdn.arstechnica.net/wp-content/uploads/2015/04/server-crash-640x426.jpg" alt="Just-released Minecraft exploit makes it easy to crash game servers"/>
+                                        
+                                    </figure>
+                                    
+                                    <p>
+                                        A flaw in the wildly popular online game <em>Minecraft</em> makes it easy for just about anyone to crash the server hosting the game, according to a computer programmer who has released proof-of-concept code that exploits the vulnerability.
+                                    </p>
+                                    <p>
+                                        "I thought a lot before writing this post," Pakistan-based developer Ammar Askar wrote in a <a href="http://blog.ammaraskar.com/minecraft-vulnerability-advisory" target="_blank">blog post published Thursday</a>, 21 months, he said, after privately reporting the bug to <em>Minecraft</em> developer Mojang. "On the one hand I don't want to expose thousands of servers to a major vulnerability, yet on the other hand Mojang has failed to act on it."
+                                    </p>
+                                    <p>
+                                        The bug resides in the <a href="https://github.com/ammaraskar/pyCraft" target="_blank">networking internals of the <em>Minecraft</em> protocol</a>. It allows the contents of inventory slots to be exchanged, so that, among other things, items in players' hotbars are displayed automatically after logging in. <em>Minecraft</em> items can also store arbitrary metadata in a file format known as <a href="http://wiki.vg/NBT" target="_blank">Named Binary Tag (NBT)</a>, which allows complex data structures to be kept in hierarchical nests. Askar has released <a href="https://github.com/ammaraskar/pyCraft/tree/nbt_exploit" target="_blank">proof-of-concept attack code</a> he said exploits the vulnerability to crash any server hosting the game. Here's how it works.
+                                    </p>
+                                    <blockquote>
+                                        <p>
+                                            The vulnerability stems from the fact that the client is allowed to send the server information about certain slots. This, coupled with the NBT format’s nesting allows us to <em>craft</em> a packet that is incredibly complex for the server to deserialize but trivial for us to generate.
+                                        </p>
+                                        <p>
+                                            In my case, I chose to create lists within lists, down to five levels. This is a json representation of what it looks like.
+                                        </p>
+                                        <div>
+                                            <pre><code data-lang="javascript"><span>rekt</span><span>:</span> <span>{</span>
+    <span>list</span><span>:</span> <span>[</span>
+        <span>list</span><span>:</span> <span>[</span>
+            <span>list</span><span>:</span> <span>[</span>
+                <span>list</span><span>:</span> <span>[</span>
+                    <span>list</span><span>:</span> <span>[</span>
+                        <span>list</span><span>:</span> <span>[</span>
+                        <span>]</span>
+                        <span>list</span><span>:</span> <span>[</span>
+                        <span>]</span>
+                        <span>list</span><span>:</span> <span>[</span>
+                        <span>]</span>
+                        <span>list</span><span>:</span> <span>[</span>
+                        <span>]</span>
+                        <span>...</span>
+                    <span>]</span>
+                    <span>...</span>
+                <span>]</span>
+                <span>...</span>
+            <span>]</span>
+            <span>...</span>
+        <span>]</span>
+        <span>...</span>
+    <span>]</span>
+    <span>...</span>
+<span>}</span></code></pre>
+                                        </div>
+                                        <p>
+                                            The root of the object, <code>rekt</code>, contains 300 lists. Each list has a list with 10 sublists, and each of those sublists has 10 of their own, up until 5 levels of recursion. That’s a total of <code>10^5 * 300 = 30,000,000</code> lists.
+                                        </p>
+                                        <p>
+                                            And this isn’t even the theoretical maximum for this attack. Just the nbt data for this payload is 26.6 megabytes. But luckily Minecraft implements a way to compress large packets, lucky us! zlib shrinks down our evil data to a mere 39 kilobytes.
+                                        </p>
+                                        <p>
+                                            Note: in previous versions of Minecraft, there was no protocol wide compression for big packets. Previously, NBT was sent compressed with gzip and prefixed with a signed short of its length, which reduced our maximum payload size to <code>2^15 - 1</code>. Now that the length is a varint capable of storing integers up to <code>2^28</code>, our potential for attack has increased significantly.
+                                        </p>
+                                        <p>
+                                            When the server will decompress our data, it’ll have 27 megs in a buffer somewhere in memory, but that isn’t the bit that’ll kill it. When it attempts to parse it into NBT, it’ll create java representations of the objects meaning suddenly, the sever is having to create several million java objects including ArrayLists. This runs the server out of memory and causes tremendous CPU load.
+                                        </p>
+                                        <p>
+                                            This vulnerability exists on almost all previous and current Minecraft versions as of 1.8.3, the packets used as attack vectors are the <a href="http://wiki.vg/Protocol#Player_Block_Placement" target="_blank">0x08: Block Placement Packet</a> and <a href="http://wiki.vg/Protocol#Creative_Inventory_Action" target="_blank">0x10: Creative Inventory Action</a>.
+                                        </p>
+                                        <p>
+                                            The fix for this vulnerability isn’t exactly that hard, the client should never really send a data structure as complex as NBT of arbitrary size and if it must, some form of recursion and size limits should be implemented.
+                                        </p>
+                                        <p>
+                                            These were the fixes that I recommended to Mojang 2 years ago.
+                                        </p>
+                                    </blockquote>
+                                    <p>
+                                        Ars is asking Mojang for comment and will update this post if company officials respond.
+                                    </p>
+                                    
+                                </div>
+                        </div></article>

article_scraper/resources/tests/readability/ars-1/source.html

@@ -0,0 +1,634 @@
+<!DOCTYPE html>
+<html lang="en-us" xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-us">
+    <head>
+        <title>
+            Just-released Minecraft exploit makes it easy to crash game servers | Ars Technica
+        </title>
+        <script type="text/javascript">
+        //<![CDATA[
+        ars = {"ASSETS":"https:\/\/cdn.arstechnica.net\/wp-content\/themes\/ars\/assets","HOME_URL":"https:\/\/arstechnica.com","LOGIN_URL":"https:\/\/arstechnica.com\/services\/login-desktop.html?v=1","CIVIS":"\/civis","THEME":"light","VIEW":"grid","MOBILE":false,"SUBSCRIBER":false,"PLUS_PLUS":false,"LOGGED":false,"USER_ID":null,"ENV":"production","AD":{"tags":["denial-of-service-attack","exploits","minecraft","vulnerabilities"],"channel":"information-technology","slug":"just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers","template_type":"article","queue":[],"server":"production"},"TOTAL":97063,"UNREAD":0,"RECENT":[1698939,1698645,1698804,1698783,1698769,1698663,1698682,1698690,1698667,1698588,1698619,1697597,1698183,1698597,1698540,1698542,1698370,1698442,1698274,1698421,1698346,1698367,1698356,1698294,1698335],"LOGINS":true,"CROSS":false,"PARSELY":"arstechnica.com","COMMENTS":false,"HOMEPAGE":false,"SITE":1,"READY":[],"SHOW_ADS":true,"IMG_PROXY":"https:\/\/cdn.arstechnica.net\/i\/","CATEGORY":"information-technology","PAGETITLE":"","ZEN_MODE":false};
+        //]]>
+        </script>
+        <link rel="stylesheet" type="text/css" media="all" href="https://cdn.arstechnica.net/wp-content/themes/ars/assets/css/main-130fcfcce0.css" />
+        <link rel="alternate" type="application/rss+xml" href="http://feeds.arstechnica.com/arstechnica/index/" />
+        <link rel="shortcut icon" href="https://cdn.arstechnica.net/favicon.ico" />
+        <link rel="icon" type="image/x-icon" href="https://cdn.arstechnica.net/favicon.ico" />
+        <link rel="apple-touch-icon" sizes="180x180" href="https://cdn.arstechnica.net/wp-content/themes/ars/assets/img/ars-ios-icon-d9a45f558c.png" />
+        <link rel="mask-icon" href="https://cdn.arstechnica.net/wp-content/themes/ars/assets/img/ars-macos-safari-8997f76b21.svg" color="#ff4e00" />
+        <link rel="icon" sizes="192x192" href="https://cdn.arstechnica.net/wp-content/themes/ars/assets/img/material-ars-db41652381.png" />
+        <meta name="application-name" content="Ars Technica" />
+        <meta name="msapplication-starturl" content="http://arstechnica.com/" />
+        <meta name="msapplication-tooltip" content="Ars Technica: Serving the technologist for 1.2 decades" />
+        <meta name="msapplication-task" content="name=News;action-uri=http://arstechnica.com/;icon-uri=https://cdn.arstechnica.net/favicon.ico" />
+        <meta name="msapplication-task" content="name=Features;action-uri=http://arstechnica.com/features/;icon-uri=https://cdn.arstechnica.net/ie-jump-menu/jump-features.ico" />
+        <meta name="msapplication-task" content="name=OpenForum;action-uri=http://arstechnica.com/civis/;icon-uri=https://cdn.arstechnica.net/ie-jump-menu/jump-forum.ico" />
+        <meta name="msapplication-task" content="name=Subscribe;action-uri=http://arstechnica.com/subscriptions/;icon-uri=https://cdn.arstechnica.net/ie-jump-menu/jump-subscribe.ico" />
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+        <meta name="advertising" content="ask" />
+        <meta property="fb:admins" content="592156917" />
+        <meta property="fb:admins" content="108943" />
+        <meta property="fb:pages" content="19374573752" />
+        <meta name="format-detection" content="telephone=no" />
+        <meta name="theme-color" content="#000000" />
+        <meta name="viewport" content="width=device-width,initial-scale=1" /><!-- cache miss 581:single/meta:5a5daf59fa5245a64fe8615caa0b1d1b -->
+        <meta name="parsely-page" content="{&quot;title&quot;:&quot;Just-released Minecraft exploit makes it easy to crash game servers&quot;,&quot;link&quot;:&quot;https:\/\/arstechnica.com\/information-technology\/2015\/04\/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers\/&quot;,&quot;type&quot;:&quot;post&quot;,&quot;author&quot;:&quot;Dan Goodin&quot;,&quot;post_id&quot;:648287,&quot;pub_date&quot;:&quot;2015-04-16T20:02:01Z&quot;,&quot;section&quot;:&quot;Biz &amp; IT&quot;,&quot;tags&quot;:[&quot;denial-of-service-attack&quot;,&quot;exploits&quot;,&quot;minecraft&quot;,&quot;vulnerabilities&quot;,&quot;type: report&quot;],&quot;image_url&quot;:&quot;https:\/\/cdn.arstechnica.net\/wp-content\/uploads\/2015\/04\/server-crash-150x150.jpg&quot;}" />
+        <meta name="parsely-metadata" content="{&quot;type&quot;:&quot;report&quot;,&quot;title&quot;:&quot;Just-released Minecraft exploit makes it easy to crash game servers&quot;,&quot;post_id&quot;:648287,&quot;lower_deck&quot;:&quot;Two-year-old bug exposes thousands of servers to crippling attack.&quot;,&quot;image_url&quot;:&quot;https:\/\/cdn.arstechnica.net\/wp-content\/uploads\/2015\/04\/server-crash-150x150.jpg&quot;,&quot;listing_image_url&quot;:&quot;https:\/\/cdn.arstechnica.net\/wp-content\/uploads\/2015\/04\/server-crash-300x150.jpg&quot;}" />
+        <link rel="canonical" href="https://arstechnica.com/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/" />
+        <link rel="amphtml" href="https://arstechnica.com/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/?amp=1" />
+        <link rel="shorturl" href="https://arstechnica.com/?p=648287" />
+        <meta name="description" content="Two-year-old bug exposes thousands of servers to crippling attack." />
+        <meta name="twitter:card" content="summary_large_image" />
+        <meta name="twitter:url" content="https://arstechnica.com/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/" />
+        <meta name="twitter:title" content="Just-released Minecraft exploit makes it easy to crash game servers" />
+        <meta name="twitter:description" content="Two-year-old bug exposes thousands of servers to crippling attack." />
+        <meta name="twitter:site" content="@arstechnica" />
+        <meta name="twitter:domain" content="arstechnica.com" />
+        <meta property="og:site_name" content="Ars Technica" />
+        <meta name="twitter:image:src" content="https://cdn.arstechnica.net/wp-content/uploads/2015/04/server-crash-640x215.jpg" />
+        <meta name="twitter:image:width" content="640" />
+        <meta name="twitter:image:height" content="215" />
+        <meta name="twitter:creator" content="@dangoodin001" />
+        <meta property="og:url" content="https://arstechnica.com/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/" />
+        <meta property="og:title" content="Just-released Minecraft exploit makes it easy to crash game servers" />
+        <meta property="og:image" content="https://cdn.arstechnica.net/wp-content/uploads/2015/04/server-crash-640x215.jpg" />
+        <meta property="og:description" content="Two-year-old bug exposes thousands of servers to crippling attack." />
+        <meta property="og:type" content="article" /><!-- cache hit 581:single/header:5a5daf59fa5245a64fe8615caa0b1d1b -->
+        <!-- Google Tag Manager DataLayer -->
+
+        <script>
+        <![CDATA[
+        window.dataLayer = window.dataLayer || [];
+        window.dataLayer.push({"event":"data-layer-loaded","user":{"ars_userId":undefined,"amg_userId":undefined,"uID":undefined,"sID":undefined,"loginStatus":false,"subscriberStatus":"none","infinityId":undefined,"registrationSource":undefined,"mdw_cnd_id":undefined,"monthlyVisits":undefined,"accessPaywall":undefined,"view":"grid","theme":"light","show_comments":false},"content":{"pageTemplate":"single","pageType":"article|report","contentCategory":"information-technology","section":"information technology","subsection":undefined,"contributor":"Dan Goodin","contentID":648287,"contentLength":835,"display":"Just-released Minecraft exploit makes it easy to crash game servers","contentSource":"web","pageAssets":undefined,"uniqueContentCount":undefined,"monthlyContentCount":undefined,"publishDate":"2015-04-16T20:02:01+00:00","modifiedDate":"2015-04-16T20:11:02+00:00","keywords":"denial of service attack|exploits|minecraft|vulnerabilities","dataSource":undefined},"marketing":{"campaignName":undefined,"circCampaignId":undefined,"internalCampaignId":undefined,"brand":"Ars Technica","certified_mrc_data":undefined,"condeNastId":undefined},"page":{"pID":undefined,"syndicatorUrl":undefined,"pageURL":"https:\/\/arstechnica.com\/?p=648287","canonical":"https:\/\/arstechnica.com\/information-technology\/2015\/04\/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers\/","canonicalPathName":"\/information-technology\/2015\/04\/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers\/"},"search":{"facets":undefined,"searchTerms":undefined},"site":{"appVersion":"1.0.0"}});
+        ]]>
+        </script><!-- End Google Tag Manager DataLayer -->
+        <!-- Google Tag Manager -->
+
+        <script>
+        <![CDATA[
+        (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
+        new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
+        j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
+        'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
+        })(window,document,'script','dataLayer','GTM-NLXNPCQ');
+        ]]>
+        </script><!-- End Google Tag Manager -->
+        <!-- OneTrust Cookies Consent Notice start -->
+
+        <script src="https://cdn.cookielaw.org/scripttemplates/otSDKStub.js" type="text/javascript" charset="UTF-8" data-domain-script="b10882a1-8446-4e7d-bfb2-ce2c770ad910"></script>
+        <script type="text/javascript">
+        //<![CDATA[
+        function OptanonWrapper(){};
+        //]]>
+        </script>
+        <script src="https://cdn.cookielaw.org/opt-out/otCCPAiab.js" type="text/javascript" charset="UTF-8" ccpa-opt-out-ids="C0002,C0003,C0004,C0005" ccpa-opt-out-geo="ca" ccpa-opt-out-lspa="true"></script><!-- OneTrust Cookies Consent Notice end -->
+
+        <script src="https://www.googletagservices.com/tag/js/gpt.js" id="gpt-script" async="async"></script>
+        <script>
+        <![CDATA[
+        window.googletag=window.googletag||{};window.googletag.cmd=window.googletag.cmd||[];window.cns=window.cns||{};window.cns.queue=[];window.cns.async=function(s,c){cns.queue.push({service:s,callback:c})};window.sparrowQueue=window.sparrowQueue||[];
+        ]]>
+        </script>
+        <link rel="dns-prefetch" href="//aax.amazon-adsystem.com" />
+        <link rel="preconnect" href="//aax.amazon-adsystem.com" crossorigin="" />
+        <link rel="preconnect" href="https://mb.moatads.com" crossorigin="" />
+        <script src="https://c.amazon-adsystem.com/aax2/apstag.js" async="async"></script>
+        <script src="https://cdn.arstechnica.net/cns/prebid.min.js?v=1597375105"></script>
+        <script src="https://js-sec.indexww.com/ht/p/183973-93942139695505.js" async="async"></script>
+        <script src="https://z.moatads.com/condenastprebidheader987326845656/moatheader.js" async="async"></script>
+        <script>
+        <![CDATA[
+        window.cns.pageContext = {"contentType":"article","templateType":"article","channel":"information-technology","subChannel":"","slug":"just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers","server":"production","keywords":{"tags":["denial-of-service-attack","exploits","minecraft","vulnerabilities"],"cm":[],"platform":["wordpress"],"copilotid":""}};
+        ]]>
+        </script>
+        <script src="https://cdn.arstechnica.net/cns/ars-technica.min.js?v=1597375105"></script>
+        <script type="text/javascript" src="https://cdn.arstechnica.net/wp-content/themes/ars/assets/js/ars-32ecec341f.ads.us.js"></script>
+    </head>
+    <body class="post-template-default single single-post postid-648287 single-format-standard grid-view light blog-us">
+        <!-- Google Tag Manager (noscript) -->
+        <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-NLXNPCQ" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript> <!-- End Google Tag Manager (noscript) -->
+        <aside class="ad ad_crown" aria-label="Top of page advertisement"></aside>
+        <div class="site-wrapper">
+            <a class="screen-reader-text skip-link" href="#main" aria-label="Skip to main content">Skip to main content</a>
+            <header class="site-header">
+                <div class="header-left">
+                    <a href="https://arstechnica.com" id="header-logo" title="Ars Technica Homepage"></a>
+                </div>
+                <div class="header-right">
+                    <nav id="header-nav-primary">
+                        <ul>
+                            <li>
+                                <a class="nav-link section-information-technology active" href="/information-technology/">Biz &amp; IT</a>
+                            </li>
+                            <li>
+                                <a class="nav-link section-gadgets" href="/gadgets/">Tech</a>
+                            </li>
+                            <li>
+                                <a class="nav-link section-science" href="/science/">Science</a>
+                            </li>
+                            <li>
+                                <a class="nav-link section-tech-policy" href="/tech-policy/">Policy</a>
+                            </li>
+                            <li>
+                                <a class="nav-link section-cars" href="/cars/">Cars</a>
+                            </li>
+                            <li>
+                                <a class="nav-link section-gaming" href="/gaming/">Gaming &amp; Culture</a>
+                            </li>
+                            <li>
+                                <a class="nav-link store" href="/store/">Store</a>
+                            </li>
+                            <li>
+                                <a class="nav-link forums" href="/civis/">Forums</a>
+                            </li>
+                        </ul>
+                    </nav><a href="/store/product/subscriptions/" class="header-highlight-link">Subscribe</a>
+                    <div class="dropdown" id="header-search">
+                        <a href="/search/" class="dropdown-toggle search-toggle" aria-label="Search" aria-expanded="false"></a>
+                        <div class="dropdown-content">
+                            <form action="/search/" method="get" id="search_form" name="search_form">
+                                <input type="hidden" name="ie" value="UTF-8" /> <input type="text" name="q" id="hdr_search_input" value="" aria-label="Search..." placeholder="Search..." />
+                            </form><a class="nav-search-close">Close</a>
+                        </div>
+                    </div>
+                    <div class="dropdown dropdown-mega" id="header-burger">
+                        <a href="#site-menu" class="dropdown-toggle" aria-label="Menu" aria-expanded="false"></a>
+                        <div id="site-menu" class="dropdown-content">
+                            <section class="burger-navigate">
+                                <h3>
+                                    Navigate
+                                </h3>
+                                <ul>
+                                    <li>
+                                        <a class="nav-link store" href="/store/">Store</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link subscribe" href="/store/product/subscriptions/">Subscribe</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link videos" href="http://video.arstechnica.com/">Videos</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link section-features" href="/features/">Features</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link section-reviews" href="/reviews/">Reviews</a>
+                                    </li>
+                                </ul>
+                                <ul>
+                                    <li>
+                                        <a class="nav-link page-rss-feeds" href="/rss-feeds/">RSS Feeds</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link mobile" href="/?view=mobile">Mobile Site</a>
+                                    </li>
+                                </ul>
+                                <ul>
+                                    <li>
+                                        <a class="nav-link page-about-us" href="/about-us/">About Ars</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link page-staff-directory" href="/staff-directory/">Staff Directory</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link page-contact-us" href="/contact-us/">Contact Us</a>
+                                    </li>
+                                </ul>
+                                <ul>
+                                    <li>
+                                        <a class="nav-link page-advertise-with-us" href="/advertise-with-us/">Advertise with Ars</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link page-reprints" href="/reprints/">Reprints</a>
+                                    </li>
+                                </ul>
+                            </section>
+                            <section class="burger-filter">
+                                <h3>
+                                    Filter by topic
+                                </h3>
+                                <ul id="burger-nav-primary">
+                                    <li>
+                                        <a class="nav-link section-information-technology active" href="/information-technology/">Biz &amp; IT</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link section-gadgets" href="/gadgets/">Tech</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link section-science" href="/science/">Science</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link section-tech-policy" href="/tech-policy/">Policy</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link section-cars" href="/cars/">Cars</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link section-gaming" href="/gaming/">Gaming &amp; Culture</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link store" href="/store/">Store</a>
+                                    </li>
+                                    <li>
+                                        <a class="nav-link forums" href="/civis/">Forums</a>
+                                    </li>
+                                </ul>
+                            </section>
+                            <section class="burger-settings">
+                                <h3>
+                                    Settings
+                                </h3>
+                                <div>
+                                    <div class="burger-layout">
+                                        <p>
+                                            Front page layout
+                                        </p>
+                                        <div class="burger-layout-grid">
+                                            <a rel="nofollow" href="http://arstechnica.com/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/?view=grid" class=""><br />
+                                            Grid
+                                            <div class="faux-radio active"></div></a>
+                                        </div>
+                                        <div class="burger-layout-list">
+                                            <a rel="nofollow" href="http://arstechnica.com/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/?view=archive" class=""><br />
+                                            List
+                                            <div class="faux-radio"></div></a>
+                                        </div>
+                                    </div>
+                                    <div class="burger-theme">
+                                        <p>
+                                            Site theme
+                                        </p>
+                                        <div class="burger-theme-light">
+                                            <a rel="nofollow" href="http://arstechnica.com/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/?theme=light" class=""><span><span>Black on white</span></span>
+                                            <div class="faux-radio active"></div></a>
+                                        </div>
+                                        <div class="burger-theme-dark">
+                                            <a rel="nofollow" href="http://arstechnica.com/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/?theme=dark" class=""><span><span>White on black</span></span>
+                                            <div class="faux-radio"></div></a>
+                                        </div>
+                                    </div>
+                                </div>
+                            </section>
+                        </div>
+                    </div>
+                    <div class="dropdown dropdown-mega" id="header-account">
+                        <a href="https://arstechnica.com/civis/ucp.php?mode=login&amp;return_to=%2Finformation-technology%2F2015%2F04%2Fjust-released-minecraft-exploit-makes-it-easy-to-crash-game-servers%2F" class="dropdown-toggle" aria-expanded="false">Sign in</a>
+                        <div class="dropdown-content">
+                            <section class="profile-activity">
+                                <h3>
+                                    Comment activity
+                                </h3>
+                                <p>
+                                    Sign up or login to join the discussions!
+                                </p>
+                            </section>
+                            <section class="profile-settings">
+                                <form id="login-form" action="https://arstechnica.com/civis/ucp.php?mode=login" method="post" name="login-form">
+                                    <input type="text" name="username" id="username" placeholder="Username or Email" aria-label="Username or Email" /> <input type="password" name="password" id="password" placeholder="Password" aria-label="Password" /> <input type="submit" value="Submit" class="button button-orange button-wide" name="login" /> <label id="remember-label"><input type="checkbox" name="autologin" id="autologin" /> Stay logged in</label> <span>|</span> <a href="/civis/ucp.php?mode=sendpassword">Having trouble?</a> <input type="hidden" name="redirect" value="./ucp.php?mode=login&amp;autoredirect=1&amp;return_to=%2Finformation-technology%2F2015%2F04%2Fjust-released-minecraft-exploit-makes-it-easy-to-crash-game-servers%2F" /> <input type="hidden" name="return_to" value="/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/" />
+                                </form>
+                                <div class="register-account">
+                                    <span>Sign up to comment and more</span> <a href="https://arstechnica.com/civis/ucp.php?mode=register" class="signup-btn button button-wide">Sign up</a>
+                                </div>
+                            </section>
+                        </div>
+                    </div>
+                </div>
+            </header>
+            <main id="main" class="content-wrapper">
+                <script type="text/javascript">
+                //<![CDATA[
+                ars.ARTICLE = {"url":"https:\/\/arstechnica.com\/information-technology\/2015\/04\/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers\/","short_url":"https:\/\/arstechnica.com\/?p=648287","title":"Just-released Minecraft exploit makes it easy to crash game servers","author":329388,"id":648287,"topic":1280621,"pages":1,"current_page":1,"superscroll":false,"promoted":[],"single_page":false,"comments":75,"fullwidth":false,"slug":"just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers","arsStaff":{"104481":{"name":"Aaron Zimmerman","title":"Copyeditor","staff":true},"1002":{"name":"Aurich Lawson","title":"Creative Director","staff":true},"509873":{"name":"Beth Mole","title":"Health Reporter","staff":true},"453791":{"name":"Cathleen O'Grady","title":"Contributing science reporter","staff":true},"102179":{"name":"Chris Lee","title":"Associate writer","staff":true},"821742":{"name":"Corey Gaskin","title":"Senior Commerce Writer","staff":true},"329388":{"name":"Dan Goodin","title":"Security Editor","staff":true},"254631":{"name":"Diana Gitig","title":"Associate Writer","staff":false},"25862":{"name":"Eric Bangeman","title":"Managing Editor","staff":true},"512413":{"name":"Eric Berger","title":"Senior Space Editor","staff":true},"46707":{"name":"Iljitsch van Beijnum","title":"Associate Writer","staff":false},"316010":{"name":"Jason Marlin","title":"Technical Director","staff":true},"746799":{"name":"Jennifer Ouellette","title":"Senior Writer","staff":true},"15365":{"name":"Jeremy Reimer","title":"Senior Niche Technology Historian","staff":false},"4086":{"name":"Jim Salter","title":"Technology Reporter","staff":true},"52979":{"name":"John Timmer","title":"Senior Science Editor","staff":true},"312082":{"name":"Jon Brodkin","title":"Senior IT Reporter","staff":true},"14317":{"name":"Jonathan M. Gitlin","title":"Automotive Editor","staff":true},"786739":{"name":"Kate Cox","title":"Tech Policy Reporter","staff":true},"998":{"name":"Ken Fisher","title":"Editor in Chief","staff":true},"440179":{"name":"Kerry Staurseth","title":"Associate Copyeditor","staff":true},"328283":{"name":"Kyle Orland","title":"Senior Gaming Editor","staff":true},"10243":{"name":"Lee Hutchinson","title":"Senior Technology Editor","staff":true},"173191":{"name":"Matthew Lasar","title":"Associate writer","staff":true},"182268":{"name":"Nate Anderson","title":"Deputy Editor","staff":true},"330533":{"name":"Nathan Mattise","title":"Features Editor","staff":true},"1991":{"name":"Ohrmazd","title":"","staff":false},"391727":{"name":"Ron Amadeo","title":"Reviews Editor","staff":true},"348927":{"name":"Sam Machkovech","title":"Tech Culture Editor","staff":true},"588289":{"name":"Samuel Axon","title":"Senior Reviews Editor","staff":true},"294205":{"name":"Scott K. Johnson","title":"Associate Writer","staff":true},"671621":{"name":"Steven Klein","title":"Developer","staff":false},"173910":{"name":"Timothy B. Lee","title":"Senior tech policy reporter","staff":true}},"tags":["denial-of-service-attack","exploits","minecraft","vulnerabilities"],"zen_mode":false,"vote_sentiments":[{"sentiment_id":"1","sentiment":"agree","direction":"positive","icon":null,"label":"Agree"},{"sentiment_id":"3","sentiment":"interesting","direction":"positive","icon":null,"label":"Interesting"},{"sentiment_id":"5","sentiment":"funny","direction":"positive","icon":null,"label":"Funny"},{"sentiment_id":"6","sentiment":"addsto","direction":"positive","icon":null,"label":"Adds to Story"},{"sentiment_id":"7","sentiment":"disagree","direction":"negative","icon":null,"label":"Disagree"},{"sentiment_id":"8","sentiment":"inaccurate","direction":"negative","icon":null,"label":"Inaccurate"},{"sentiment_id":"11","sentiment":"pointless","direction":"negative","icon":null,"label":"Doesn't Contribute"},{"sentiment_id":"12","sentiment":"abusive","direction":"negative","icon":null,"label":"Abusive"}]};
+                //]]>
+                </script>
+                <article itemscope="itemscope" itemtype="http://schema.org/NewsArticle" class="article-single standalone intro-default" id="">
+                    <div class="column-wrapper">
+                        <div class="left-column">
+                            <header class="article-header">
+                                <h4 class="post-upperdek">
+                                    Biz &amp; IT —
+                                </h4>
+                                <h1 itemprop="headline">
+                                    Just-released <i>Minecraft</i> exploit makes it easy to crash game servers
+                                </h1>
+                                <h2 itemprop="description">
+                                    Two-year-old bug exposes thousands of servers to crippling attack.
+                                </h2>
+                                <section class="post-meta">
+                                    <p class="byline" itemprop="author creator" itemscope="itemscope" itemtype="http://schema.org/Person">
+                                        <a itemprop="url" href="https://arstechnica.com/author/dan-goodin/" rel="author"><span itemprop="name">Dan Goodin</span></a> - <time class="date" data-time="1429214521" datetime="2015-04-16T20:02:01+00:00">Apr 16, 2015 8:02 pm UTC</time>
+                                    </p>
+                                </section>
+                            </header>
+                            <section class="article-guts">
+                                <div itemprop="articleBody" class="article-content post-page">
+                                    <figure class="intro-image intro-left">
+                                        <img src="https://cdn.arstechnica.net/wp-content/uploads/2015/04/server-crash-640x426.jpg" alt="Just-released Minecraft exploit makes it easy to crash game servers" />
+                                        <figcaption class="caption">
+                                            <div class="caption-credit">
+                                                <a rel="nofollow" class="caption-link" href="https://en.wikipedia.org/wiki/Kernel_panic#/media/File:Kernel-panic.jpg">Kevin</a>
+                                            </div>
+                                        </figcaption>
+                                    </figure>
+                                    <aside id="social-left" class="social-left" aria-label="Read the comments or share this article">
+                                        <a title="51 posters participating" class="comment-count icon-comment-bubble-down" href="https://arstechnica.com/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/?comments=1">
+                                        <h4 class="comment-count-before">
+                                            reader comments
+                                        </h4><span class="comment-count-number">75</span> <span class="visually-hidden">with 51 posters participating</span></a>
+                                        <div class="share-links">
+                                            <h4>
+                                                Share this story
+                                            </h4>
+                                            <ul>
+                                                <li>
+                                                    <a href="https://www.facebook.com/sharer.php?u=https%3A%2F%2Farstechnica.com%2F%3Fpost_type%3Dpost%26p%3D648287" target="_blank" class="social-icon share-facebook" title="Share on Facebook"><span class="visually-hidden">Share on Facebook</span></a>
+                                                </li>
+                                                <li>
+                                                    <a href="https://twitter.com/share?text=Just-released+%3Ci%3EMinecraft%3C%2Fi%3E+exploit+makes+it+easy+to+crash+game+servers&amp;url=https%3A%2F%2Farstechnica.com%2F%3Fpost_type%3Dpost%26p%3D648287" target="_blank" class="social-icon share-twitter" title="Share on Twitter"><span class="visually-hidden">Share on Twitter</span></a>
+                                                </li>
+                                                <li>
+                                                    <a href="https://www.reddit.com/submit?url=https%3A%2F%2Farstechnica.com%2F%3Fpost_type%3Dpost%26p%3D648287&amp;title=Just-released+%3Ci%3EMinecraft%3C%2Fi%3E+exploit+makes+it+easy+to+crash+game+servers" target="_blank" class="social-icon share-reddit" title="Share on Reddit"><span class="visually-hidden">Share on Reddit</span></a>
+                                                </li>
+                                            </ul>
+                                        </div>
+                                    </aside><!-- cache miss 581:single/related:5a5daf59fa5245a64fe8615caa0b1d1b --><!-- empty -->
+                                    <p>
+                                        A flaw in the wildly popular online game <em>Minecraft</em> makes it easy for just about anyone to crash the server hosting the game, according to a computer programmer who has released proof-of-concept code that exploits the vulnerability.
+                                    </p>
+                                    <p>
+                                        "I thought a lot before writing this post," Pakistan-based developer Ammar Askar wrote in a <a href="http://blog.ammaraskar.com/minecraft-vulnerability-advisory">blog post published Thursday</a>, 21 months, he said, after privately reporting the bug to <em>Minecraft</em> developer Mojang. "On the one hand I don't want to expose thousands of servers to a major vulnerability, yet on the other hand Mojang has failed to act on it."
+                                    </p>
+                                    <p>
+                                        The bug resides in the <a href="https://github.com/ammaraskar/pyCraft">networking internals of the <em>Minecraft</em> protocol</a>. It allows the contents of inventory slots to be exchanged, so that, among other things, items in players' hotbars are displayed automatically after logging in. <em>Minecraft</em> items can also store arbitrary metadata in a file format known as <a href="http://wiki.vg/NBT">Named Binary Tag (NBT)</a>, which allows complex data structures to be kept in hierarchical nests. Askar has released <a href="https://github.com/ammaraskar/pyCraft/tree/nbt_exploit">proof-of-concept attack code</a> he said exploits the vulnerability to crash any server hosting the game. Here's how it works.
+                                    </p>
+                                    <blockquote>
+                                        <p>
+                                            The vulnerability stems from the fact that the client is allowed to send the server information about certain slots. This, coupled with the NBT format’s nesting allows us to <em>craft</em> a packet that is incredibly complex for the server to deserialize but trivial for us to generate.
+                                        </p>
+                                        <p>
+                                            In my case, I chose to create lists within lists, down to five levels. This is a json representation of what it looks like.
+                                        </p>
+                                        <div class="highlight">
+                                            <pre><code class="language-javascript" data-lang="javascript"><span class="nx">rekt</span><span class="o">:</span> <span class="p">{</span>
+    <span class="nx">list</span><span class="o">:</span> <span class="p">[</span>
+        <span class="nx">list</span><span class="o">:</span> <span class="p">[</span>
+            <span class="nx">list</span><span class="o">:</span> <span class="p">[</span>
+                <span class="nx">list</span><span class="o">:</span> <span class="p">[</span>
+                    <span class="nx">list</span><span class="o">:</span> <span class="p">[</span>
+                        <span class="nx">list</span><span class="o">:</span> <span class="p">[</span>
+                        <span class="p">]</span>
+                        <span class="nx">list</span><span class="o">:</span> <span class="p">[</span>
+                        <span class="p">]</span>
+                        <span class="nx">list</span><span class="o">:</span> <span class="p">[</span>
+                        <span class="p">]</span>
+                        <span class="nx">list</span><span class="o">:</span> <span class="p">[</span>
+                        <span class="p">]</span>
+                        <span class="p">...</span>
+                    <span class="p">]</span>
+                    <span class="p">...</span>
+                <span class="p">]</span>
+                <span class="p">...</span>
+            <span class="p">]</span>
+            <span class="p">...</span>
+        <span class="p">]</span>
+        <span class="p">...</span>
+    <span class="p">]</span>
+    <span class="p">...</span>
+<span class="p">}</span></code></pre>
+                                        </div>
+                                        <p>
+                                            The root of the object, <code>rekt</code>, contains 300 lists. Each list has a list with 10 sublists, and each of those sublists has 10 of their own, up until 5 levels of recursion. That’s a total of <code>10^5 * 300 = 30,000,000</code> lists.
+                                        </p>
+                                        <p>
+                                            And this isn’t even the theoretical maximum for this attack. Just the nbt data for this payload is 26.6 megabytes. But luckily Minecraft implements a way to compress large packets, lucky us! zlib shrinks down our evil data to a mere 39 kilobytes.
+                                        </p>
+                                        <p>
+                                            Note: in previous versions of Minecraft, there was no protocol wide compression for big packets. Previously, NBT was sent compressed with gzip and prefixed with a signed short of its length, which reduced our maximum payload size to <code>2^15 - 1</code>. Now that the length is a varint capable of storing integers up to <code>2^28</code>, our potential for attack has increased significantly.
+                                        </p>
+                                        <p>
+                                            When the server will decompress our data, it’ll have 27 megs in a buffer somewhere in memory, but that isn’t the bit that’ll kill it. When it attempts to parse it into NBT, it’ll create java representations of the objects meaning suddenly, the sever is having to create several million java objects including ArrayLists. This runs the server out of memory and causes tremendous CPU load.
+                                        </p>
+                                        <p>
+                                            This vulnerability exists on almost all previous and current Minecraft versions as of 1.8.3, the packets used as attack vectors are the <a href="http://wiki.vg/Protocol#Player_Block_Placement">0x08: Block Placement Packet</a> and <a href="http://wiki.vg/Protocol#Creative_Inventory_Action">0x10: Creative Inventory Action</a>.
+                                        </p>
+                                        <p>
+                                            The fix for this vulnerability isn’t exactly that hard, the client should never really send a data structure as complex as NBT of arbitrary size and if it must, some form of recursion and size limits should be implemented.
+                                        </p>
+                                        <p>
+                                            These were the fixes that I recommended to Mojang 2 years ago.
+                                        </p>
+                                    </blockquote>
+                                    <p>
+                                        Ars is asking Mojang for comment and will update this post if company officials respond.
+                                    </p>
+                                    <div id="action_button_container"></div>
+                                </div>
+                            </section>
+                        </div>
+                        <div class="xrail">
+                            <div class="xrail-content">
+                                <aside class="ad ad_xrail ad_xrail_top" aria-label="Top sidebar advertisement"></aside>
+                                <aside class="ad_native ad_native_xrail" aria-label="Sidebar native advertisement"></aside>
+                            </div>
+                        </div>
+                    </div>
+                    <div class="column-wrapper">
+                        <div class="left-column">
+                            <div id="social-footer">
+                                <a title="51 posters participating" class="comment-count icon-comment-bubble-down" href="https://arstechnica.com/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/?comments=1">
+                                <h4 class="comment-count-before">
+                                    reader comments
+                                </h4><span class="comment-count-number">75</span> <span class="visually-hidden">with 51 posters participating</span></a>
+                                <div class="share-links">
+                                    <h4>
+                                        Share this story
+                                    </h4>
+                                    <ul>
+                                        <li>
+                                            <a href="https://www.facebook.com/sharer.php?u=https%3A%2F%2Farstechnica.com%2F%3Fpost_type%3Dpost%26p%3D648287" target="_blank" class="social-icon share-facebook" title="Share on Facebook"><span class="visually-hidden">Share on Facebook</span></a>
+                                        </li>
+                                        <li>
+                                            <a href="https://twitter.com/share?text=Just-released+%3Ci%3EMinecraft%3C%2Fi%3E+exploit+makes+it+easy+to+crash+game+servers&amp;url=https%3A%2F%2Farstechnica.com%2F%3Fpost_type%3Dpost%26p%3D648287" target="_blank" class="social-icon share-twitter" title="Share on Twitter"><span class="visually-hidden">Share on Twitter</span></a>
+                                        </li>
+                                        <li>
+                                            <a href="https://www.reddit.com/submit?url=https%3A%2F%2Farstechnica.com%2F%3Fpost_type%3Dpost%26p%3D648287&amp;title=Just-released+%3Ci%3EMinecraft%3C%2Fi%3E+exploit+makes+it+easy+to+crash+game+servers" target="_blank" class="social-icon share-reddit" title="Share on Reddit"><span class="visually-hidden">Share on Reddit</span></a>
+                                        </li>
+                                    </ul>
+                                </div>
+                            </div><!-- cache hit 581:single/author:2814756d09510ff24ad530ca37a5a9a9 -->
+                            <section class="article-author">
+                                <a style="background-image:url('https://cdn.arstechnica.net/wp-content/uploads/2018/10/Dang.jpg');" class="author-photo" href="/author/dan-goodin" tabindex="-1" role="presentation" aria-hidden="true"></a>
+                                <div class="author-bio">
+                                    <section class="author-bio-top">
+                                        <a href="/author/dan-goodin" class="author-name">Dan Goodin</a> Dan is the Security Editor at Ars Technica, which he joined in 2012 after working for The Register, the Associated Press, Bloomberg News, and other publications.
+                                    </section>
+                                    <section class="author-social">
+                                        <strong>Email</strong> <a href="mailto:dan.goodin@arstechnica.com">dan.goodin@arstechnica.com</a> <span class="slashes">//</span> <strong>Twitter</strong> <a href="https://www.twitter.com/dangoodin001" target="_blank">@dangoodin001</a>
+                                    </section>
+                                </div>
+                            </section>
+                        </div>
+                        <div class="xrail"></div>
+                    </div>
+                    <div id="article-footer-wrap">
+                        <aside class="ad ad_fullwidth fullwidth" aria-label="Full width advertisement"></aside>
+                        <section id="comments-area" class="comments-area column-wrapper">
+                            <div class="row comments-row left-column">
+                                <a name="comments-bar" id="comments-bar"></a>
+                                <div id="comments-container"></div>
+                                <div id="comments-posting-container" class="thick-divide-bottom">
+                                    <p id="reply">
+                                        You must <a href="https://arstechnica.com/civis/ucp.php?mode=login&amp;return_to/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/" class="vote_login">login or create an account</a> to comment.
+                                    </p>
+                                </div>
+                            </div>
+                            <div class="xrail xrail-comments">
+                                <div class="xrail-content xrail-content-comments">
+                                    <aside class="ad ad_xrail ad_xrail_comments" aria-label="Comments sidebar advertisement"></aside>
+                                </div>
+                            </div>
+                        </section>
+                        <section class="inline-playlist">
+                            <div class="ars-video-playlist">
+                                <h3 class="ars-video-playlist-module-header">
+                                    Channel <span>Ars Technica</span>
+                                </h3>
+                                <div class="ars-video-playlist-module" data-playlist-id="arstechnica-channel-ars-information-technology" data-video-options="[]"></div>
+                            </div>
+                        </section>
+                        <div class="prev-next-links">
+                            <a href="https://arstechnica.com/tech-policy/2015/04/dozens-of-us-government-online-whistleblower-sites-not-secured-by-https/" rel="prev"><span class="arrow">←</span> Previous story</a> <a href="https://arstechnica.com/gaming/2015/04/hidden-files-suggest-street-fighters-ryu-may-come-to-smash-bros/" rel="next">Next story <span class="arrow">→</span></a>
+                        </div>
+                        <footer id="article-footer">
+                            <div id="recommendations-footer">
+                                <div id="story-recommendations">
+                                    <div class="heading-column">
+                                        <h3>
+                                            Related Stories
+                                        </h3>
+                                    </div>
+                                    <ul id="story-recs" class="rec-wrap"></ul>
+                                </div>
+                                <div id="sponsored-recommendations">
+                                    <div class="heading-column">
+                                        <h3>
+                                            Sponsored Stories
+                                        </h3><a href="http://www.outbrain.com/what-is/default/en" target="_blank">Powered by </a>
+                                    </div>
+                                    <ul id="outbrain-recs"></ul>
+                                </div>
+                                <div id="latest-stories">
+                                    <div class="heading-column">
+                                        <h3>
+                                            Today on Ars
+                                        </h3>
+                                    </div>
+                                    <ul id="latest-recs" class="rec-wrap"></ul>
+                                </div>
+                            </div>
+                        </footer>
+                    </div>
+                </article>
+            </main>
+            <footer class="site-footer">
+                <nav class="nav-footer">
+                    <section>
+                        <ul>
+                            <li>
+                                <a href="/store/">Store</a>
+                            </li>
+                            <li>
+                                <a href="/store/product/subscriptions/">Subscribe</a>
+                            </li>
+                            <li>
+                                <a href="/about-us/">About Us</a>
+                            </li>
+                            <li>
+                                <a href="/rss-feeds/">RSS Feeds</a>
+                            </li>
+                            <li>
+                                <a rel="nofollow" href="http://arstechnica.com/information-technology/2015/04/just-released-minecraft-exploit-makes-it-easy-to-crash-game-servers/?view=mobile">View Mobile Site</a>
+                            </li>
+                        </ul>
+                    </section>
+                    <section>
+                        <ul>
+                            <li>
+                                <a href="/contact-us/">Contact Us</a>
+                            </li>
+                            <li>
+                                <a href="/staff-directory/">Staff</a>
+                            </li>
+                            <li>
+                                <a href="/advertise-with-us/">Advertise with us</a>
+                            </li>
+                            <li>
+                                <a href="/reprints/">Reprints</a>
+                            </li>
+                        </ul>
+                    </section>
+                    <section class="footer-newsletter">
+                        <div class="newsletter-wrapper">
+                            <h3>
+                                <a href="/newsletters/">Newsletter Signup</a>
+                            </h3>
+                            <p>
+                                Join the Ars Orbital Transmission mailing list to get weekly updates delivered to your inbox.
+                            </p><a class="button" href="/newsletters/">Sign me up →</a>
+                        </div>
+                    </section>
+                </nav>
+                <section class="footer-terms-logo">
+                    <div class="cn-logo">
+                        <a href="http://condenast.com/" class="icon icon-logo-cn-us" title="Visit Condé Nast"></a>
+                    </div>
+                    <p id="copyright-terms">
+                        CNMN Collection<br />
+                        WIRED Media Group<br />
+                        © 2020 Condé Nast. All rights reserved. Use of and/or registration on any portion of this site constitutes acceptance of our <a href="https://www.condenast.com/user-agreement/">User Agreement</a> (updated 1/1/20) and <a href="https://www.condenast.com/privacy-policy/">Privacy Policy and Cookie Statement</a> (updated 1/1/20) and <a href="/amendment-to-conde-nast-user-agreement-privacy-policy/">Ars Technica Addendum</a> (effective 8/21/2018). Ars may earn compensation on sales from links on this site. <a href="/affiliate-link-policy/">Read our affiliate link policy</a>.<br />
+                        <a href="https://www.condenast.com/privacy-policy/#california">Your California Privacy Rights</a> | <a id="ot-sdk-btn" class="ot-sdk-show-settings">Do Not Sell My Personal Information</a><br />
+                        The material on this site may not be reproduced, distributed, transmitted, cached or otherwise used, except with the prior written permission of Condé Nast.<br />
+                        <a href="https://www.condenast.com/online-behavioral-advertising-oba-and-how-to-opt-out-of-oba/#clickheretoreadmoreaboutonlinebehavioraladvertising(oba)">Ad Choices</a>
+                    </p>
+                </section>
+            </footer>
+        </div>
+        <script type="text/javascript" src="https://cdn.arstechnica.net/wp-content/themes/ars/assets/js/main-fafdd8b108.js"></script> <!-- cache hit 581:single/javascript-footer:5a5daf59fa5245a64fe8615caa0b1d1b -->
+         
+        <script async="async" type="application/javascript" src="https://embed.actionbutton.co/widget/widget.min.js"></script> <!-- Parse.ly start -->
+        <script id="parsely-cfg" src="//fpa-cdn.arstechnica.com/keys/arstechnica.com/p.js"></script> <!-- Parse.ly end -->
+         
+        <script src="https://player.cnevids.com/interlude/arstechnica.js" async="async"></script> 
+        <script id="conde-polar" src="https://cdn.mediavoice.com/nativeads/script/condenastcorporate/conde-asa-polar-master.js" async="async"></script> 
+        <script>
+        <![CDATA[
+
+        (function () {
+        function DQ() {
+        var queue = window.sparrowQueue;
+        this.push = fn => fn();
+        window.sparrowQueue = this;
+        while (queue.length) {
+          queue.shift()();
+        }
+        }
+        function e(t, e) {
+        var n, a, o;
+        a = !1, n = document.createElement("script"), n.type = "text/javascript", n.src = t, n.onload = n.onreadystatechange = function () {
+          a || this.readyState && "complete" != this.readyState || (a = !0, e ? e() : !0)
+        }, o = document.getElementsByTagName("script")[0], o.parentNode.insertBefore(n, o)
+        }
+        if (location.search.indexOf('no_sparrow') < 0) {
+        e("https://pixel.condenastdigital.com/config/v2/production/ars-technica.config.js", function () {
+          e("https://pixel.condenastdigital.com/sparrow.min.js", function () {
+            if (window.SparrowConfigV2) {
+              window.sparrow = new window.Sparrow(window.SparrowConfigV2);
+              new DQ();
+            }
+          })
+        })
+        }
+        })();
+        ]]>
+        </script> 
+        <script type="text/javascript" src="//s.skimresources.com/js/100098X1555750.skimlinks.js"></script>
+    </body>
+</html>

article_scraper/resources/tests/readability/base-url-base-element-relative/expected.html

@@ -0,0 +1,34 @@
+<article><DIV id="readability-page-1"><article>
+    <h2>Lorem</h2>
+    <p>
+      Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+      tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </p>
+    <p>Links</p>
+    <p><a href="http://fakehost/test/base/foo/bar/baz.html" target="_blank">link</a></p>
+    <p><a href="http://fakehost/test/base/foo/bar/baz.html" target="_blank">link</a></p>
+    <p><a href="http://fakehost/foo/bar/baz.html" target="_blank">link</a></p>
+    <p><a href="#foo">link</a></p>
+    <p><a href="http://fakehost/test/base/baz.html#foo" target="_blank">link</a></p>
+    <p><a href="http://fakehost/foo/bar/baz.html#foo" target="_blank">link</a></p>
+    <p><a href="http://test/foo/bar/baz.html" target="_blank">link</a></p>
+    <p><a href="https://test/foo/bar/baz.html" target="_blank">link</a></p>
+    <p>Images</p>
+    <p><img src="http://fakehost/test/base/foo/bar/baz.png"/></p>
+    <p><img src="http://fakehost/test/base/foo/bar/baz.png"/></p>
+    <p><img src="http://fakehost/foo/bar/baz.png"/></p>
+    <p><img src="http://test/foo/bar/baz.png"/></p>
+    <p><img src="https://test/foo/bar/baz.png"/></p>
+    <h2>Foo</h2>
+    <p>
+      Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </p>
+  </article></DIV></article>

article_scraper/resources/tests/readability/base-url-base-element-relative/source.html

@@ -0,0 +1,44 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8"/>
+  <base href="base/"/>
+  <title>Base URL with base relative test</title>
+</head>
+<body>
+  <article>
+    <h1>Lorem</h1>
+    <div>
+      Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+      tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </div>
+    <p>Links</p>
+    <p><a href="foo/bar/baz.html">link</a></p>
+    <p><a href="./foo/bar/baz.html">link</a></p>
+    <p><a href="/foo/bar/baz.html">link</a></p>
+    <p><a href="#foo">link</a></p>
+    <p><a href="baz.html#foo">link</a></p>
+    <p><a href="/foo/bar/baz.html#foo">link</a></p>
+    <p><a href="http://test/foo/bar/baz.html">link</a></p>
+    <p><a href="https://test/foo/bar/baz.html">link</a></p>
+    <p>Images</p>
+    <p><img src="foo/bar/baz.png"/></p>
+    <p><img src="./foo/bar/baz.png"/></p>
+    <p><img src="/foo/bar/baz.png"/></p>
+    <p><img src="http://test/foo/bar/baz.png"/></p>
+    <p><img src="https://test/foo/bar/baz.png"/></p>
+    <h2>Foo</h2>
+    <div>
+      Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </div>
+  </article>
+</body>
+</html>

article_scraper/resources/tests/readability/basic-tags-cleaning/expected.html

@@ -0,0 +1,20 @@
+<article><DIV id="readability-page-1"><div>
+      <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+        tempor incididunt ut labore et dolore magna aliqua.</p>
+      <p>Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat.</p>
+      
+      <p>Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+    </div><div>
+      <p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat.</p>
+      
+      
+      <p>Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+    </div></DIV></article>

article_scraper/resources/tests/readability/basic-tags-cleaning/source.html

@@ -0,0 +1,36 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8"/>
+  <title>Basic tag cleaning test</title>
+</head>
+<body>
+  <article>
+    <h1>Lorem</h1>
+    <div>
+      <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+        tempor incididunt ut labore et dolore magna aliqua.</p>
+      <p>Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat.</p>
+      <iframe src="about:blank">Iframe fallback test</iframe>
+      <p>Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+    </div>
+    <h2>Foo</h2>
+    <div>
+      <p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat.</p>
+      <object data="foo.swf" type="application/x-shockwave-flash" width="88" height="31">
+        <param movie="foo.swf" />
+      </object>
+      <embed src="foo.swf"/>
+      <p>Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+    </div>
+  </article>
+</body>
+</html>

article_scraper/resources/tests/readability/bbc-1/expected.html

@@ -0,0 +1,22 @@
+<article><div property="articleBody" id="readability-page-1">
+        <p>President Barack Obama has admitted that his failure to pass "common sense gun safety laws" in the US is the greatest frustration of his presidency. </p><p>In an interview with the BBC, Mr Obama said it was "distressing" not to have made progress on the issue "even in the face of repeated mass killings".</p><p>He vowed to keep trying, but the BBC's North America editor Jon Sopel said the president did not sound very confident. </p><p>However, Mr Obama said race relations had improved during his presidency. </p><p>Hours after the interview, a gunman opened fire at a cinema in the US state of Louisiana, killing two people and injuring several others before shooting himself.</p><p>In a wide-ranging interview, President Obama also said:</p><ul>
+<li>
+<a href="http://www.bbc.co.uk/news/uk-politics-33647154" target="_blank">The UK must stay in the EU</a> to have influence on the world stage</li>
+<li>He is confident the Iran nuclear deal will be passed by Congress </li>
+<li>Syria needs a political solution in order to defeat the Islamic State group</li>
+<li>He would speak "bluntly" against corruption <a href="http://www.bbc.co.uk/news/world-us-canada-33646563" target="_blank">and human rights violations in Kenya</a>
+</li>
+<li>He would defend his advocacy of gay rights following protests in Kenya</li>
+<li>Despite racial tensions, the US is becoming more diverse and more tolerant</li>
+</ul><p><a href="http://www.bbc.co.uk/news/world-us-canada-33646542" target="_blank">Read the full transcript of his interview</a></p><p>Mr Obama lands in Kenya later on Friday for his first visit since becoming president. </p><p>But with just 18 months left in power, he said gun control was the area where he has been "most frustrated and most stymied" since coming to power in 2009.</p><p>"If you look at the number of Americans killed since 9/11 by terrorism, it's less than 100. If you look at the number that have been killed by gun violence, it's in the tens of thousands," Mr Obama said. </p><figure><img src="http://ichef.bbci.co.uk/news/555/cpsprodpb/462D/production/_84456971_gettyimages-167501087.jpg" datasrc="http://ichef.bbci.co.uk/news/976/cpsprodpb/462D/production/_84456971_gettyimages-167501087.jpg" alt="Gun control campaigners protest in McPhearson Square in Washington DC - 25 April 2013" height="549" width="976"/>
+                <figcaption>
+                    <span>
+                        The president said he would continue fighting for greater gun control laws
+                    </span>
+                </figcaption></figure><p>"For us not to be able to resolve that issue has been something that is distressing," he added. </p><p>Mr Obama has pushed for stricter gun control throughout his presidency but has been unable to secure any significant changes to the laws. </p><p>After nine African-American churchgoers were killed in South Carolina in June, he admitted "politics in this town" meant there were few options available.</p><figure><img src="http://ichef.bbci.co.uk/news/555/media/images/76020000/jpg/_76020974_line976.jpg" datasrc="http://ichef.bbci.co.uk/news/464/media/images/76020000/jpg/_76020974_line976.jpg" alt="line" height="2" width="464"/></figure><h2>Analysis: Jon Sopel, BBC News, Washington</h2><figure><img src="http://ichef-1.bbci.co.uk/news/555/cpsprodpb/6D3D/production/_84456972_p072315al-0500.jpg" datasrc="http://ichef-1.bbci.co.uk/news/976/cpsprodpb/6D3D/production/_84456972_p072315al-0500.jpg" alt="President Barack Obama participates in an interview with Jon Sopel of BBC in the Roosevelt Room of the White House - 23 July 2015" height="549" width="976"/></figure><p>Nine months ago, the president seemed like a spent force, after taking a beating in the midterm elections, during which members of his own party were reluctant to campaign on his record. </p><p>But the man sat before me today was relaxed and confident, buoyed by a string of "wins" on healthcare, Cuba and Iran, after bitter and ongoing battles with his many critics. </p><p>The only body swerve the president performed was when I asked him <a href="http://www.bbc.co.uk/news/world-us-canada-33643168" target="_blank"> how many minds he had changed on the Iran nuclear deal </a>after an intense sell aimed at Gulf allies and members of US Congress who remain implacably opposed. </p><p>There was a momentary flicker across the president's face as if to say "You think you got me?" before his smile returned and he proceeded to talk about how Congress would come round.</p><p>But notably, he did not give a direct answer to that question, which leaves me with the impression that he has persuaded precisely zero.</p><p><a href="http://www.bbc.co.uk/news/world-us-canada-33646875" target="_blank">Five things we learned from Obama interview</a></p><p><a href="http://www.bbc.co.uk/news/world-us-canada-33646545" target="_blank">The presidential body swerve</a></p><figure><img src="http://ichef.bbci.co.uk/news/555/media/images/76020000/jpg/_76020974_line976.jpg" datasrc="http://ichef.bbci.co.uk/news/464/media/images/76020000/jpg/_76020974_line976.jpg" alt="line" height="2" width="464"/></figure><p>On race relations, Mr Obama said recent concerns around policing and mass incarcerations were "legitimate and deserve intense attention" but insisted progress had been made. </p><p>Children growing up during the eight years of his presidency "will have a different view of race relations in this country and what's possible," he said. </p><p>"There are going to be tensions that arise. But if you look at my daughters' generation, they have an attitude about race that's entirely different than even my generation."</p><p>Talking about how he was feeling after his recent successes, he said "every president, every leader has strengths and weaknesses".  </p><p>"One of my strengths is I have a pretty even temperament.  I don't get too high when it's high and I don't get too low when it's low," he said.   </p><figure><img src="http://ichef-1.bbci.co.uk/news/555/cpsprodpb/142FD/production/_84458628_shirtreuters.jpg" datasrc="http://ichef-1.bbci.co.uk/news/976/cpsprodpb/142FD/production/_84458628_shirtreuters.jpg" alt="Customer looks at Obama shirts at a stall in Nairobi's Kibera slums, 23 July 2015" height="549" width="976"/>
+                <figcaption>
+                    <span>
+                        Kenya is getting ready to welcome the US president
+                    </span>
+                </figcaption></figure><h2>Kenya trip</h2><p>Mr Obama was speaking to the BBC at the White House before departing for Kenya.</p><p>His father was Kenyan and the president is expected to meet relatives in Nairobi.</p><p>Mr Obama has faced criticism in the country after the US legalised gay marriage. However, in his interview, the president said he would not fall silent on the issue.</p><p>"I am not a fan of discrimination and bullying of anybody on the basis of race, on the basis of religion, on the basis of sexual orientation or gender," he said.</p><p>The president also admitted that some African governments, including Kenya's, needed to improve their records on human rights and democracy. However, he defended his decision to engage with and visit those governments. </p><p>"Well, they're not ideal institutions.  But what we found is, is that when we combined blunt talk with engagement, that gives us the best opportunity to influence and open up space for civil society." </p><p>Mr Obama will become the first US president to address the African Union when he travels on to Ethiopia on Sunday.</p>
+    </div></article>

article_scraper/resources/tests/readability/blogger/expected.html

@@ -0,0 +1,84 @@
+<article><div id="readability-page-1" itemprop="description articleBody"><p>
+                                                                                I've written a couple of posts in the past few months but they were all for <a href="http://blog.ioactive.com/search/label/Andrew%20Zonenberg" target="_blank">the blog at work</a> so I figured I'm long overdue for one on Silicon Exposed.</p><h2>
+                                                                                    So what's a GreenPak?</h2>
+                                                                                <br/><p> Silego Technology is a fabless semiconductor company located in the SF Bay area, which makes (among other things) a line of programmable logic devices known as GreenPak. Their <a href="http://www.silego.com/products/greenpak5.html" target="_blank">5th generation parts</a> were just announced, but I started this project before that happened so I'm still targeting the <a href="http://www.silego.com/products/greenpak4.html" target="_blank">4th generation</a>.</p><p> GreenPak devices are kind of like itty bitty <a href="http://www.cypress.com/products/32-bit-arm-cortex-m-psoc" target="_blank">PSoCs</a> - they have a mixed signal fabric with an ADC, DACs, comparators, voltage references, plus a digital LUT/FF fabric and some typical digital MCU peripherals like counters and oscillators (but no CPU).</p><p> It's actually an interesting architecture - FPGAs (including some devices marketed as CPLDs) are a 2D array of LUTs connected via wires to adjacent cells, and true (product term) CPLDs are a star topology of AND-OR arrays connected by a crossbar. GreenPak, on the other hand, is a star topology of LUTs, flipflops, and analog/digital hard IP connected to a crossbar.</p><p> Without further ado, here's a block diagram showing all the cool stuff you get in the SLG46620V:</p><table>
+                                                                                    <tbody>
+                                                                                        <tr>
+                                                                                            <td>
+                                                                                                <a href="https://1.bp.blogspot.com/-YIPC5jkXkDE/Vy7YPSqFKWI/AAAAAAAAAxI/a7D6Ji2GxoUvcrwUkI4RLZcr2LFQEJCTACLcB/s1600/block-diagram.png" imageanchor="1" target="_blank"><img height="512" src="https://1.bp.blogspot.com/-YIPC5jkXkDE/Vy7YPSqFKWI/AAAAAAAAAxI/a7D6Ji2GxoUvcrwUkI4RLZcr2LFQEJCTACLcB/s640/block-diagram.png" width="640"/></a>
+                                                                                            </td>
+                                                                                        </tr>
+                                                                                        <tr>
+                                                                                            <td>SLG46620V block diagram (from device datasheet)</td>
+                                                                                        </tr>
+                                                                                    </tbody>
+                                                                                </table><p>
+                                                                                They're also tiny (the SLG46620V is a 20-pin 0.4mm pitch STQFN measuring 2x3 mm, and the lower gate count SLG46140V is a mere 1.6x2 mm) and probably the cheapest programmable logic device on the market - $0.50 in low volume and less than $0.40 in larger quantities.</p><p> The Vdd range of GreenPak4 is huge, more like what you'd expect from an MCU than an FPGA! It can run on anything from 1.8 to 5V, although performance is only specified at 1.8, 3.3, and 5V nominal voltages. There's also a dual-rail version that trades one of the GPIO pins for a second power supply pin, allowing you to interface to logic at two different voltage levels.</p><p> To support low-cost/space-constrained applications, they even have the configuration memory on die. It's one-time programmable and needs external Vpp to program (presumably Silego didn't want to waste die area on charge pumps that would only be used once) but has a SRAM programming mode for prototyping.</p><p> The best part is that the development software (GreenPak Designer) is free of charge and provided for all major operating systems including Linux! Unfortunately, the only supported design entry method is schematic entry and there's no way to write your design in a HDL.</p><p> While schematics may be fine for quick tinkering on really simple designs, they quickly get unwieldy. The nightmare of a circuit shown below is just a bunch of counters hooked up to LEDs that blink at various rates.</p><table>
+                                                                                    <tbody>
+                                                                                        <tr>
+                                                                                            <td>
+                                                                                                <a href="https://1.bp.blogspot.com/-k3naUT3uXao/Vy7WFac246I/AAAAAAAAAw8/mePy_ostO8QJra5ZJrbP2WGhTlJ0B_r8gCLcB/s1600/schematic-from-hell.png" imageanchor="1" target="_blank"><img height="334" src="https://1.bp.blogspot.com/-k3naUT3uXao/Vy7WFac246I/AAAAAAAAAw8/mePy_ostO8QJra5ZJrbP2WGhTlJ0B_r8gCLcB/s640/schematic-from-hell.png" width="640"/></a>
+                                                                                            </td>
+                                                                                        </tr>
+                                                                                        <tr>
+                                                                                            <td>Schematic from hell!</td>
+                                                                                        </tr>
+                                                                                    </tbody>
+                                                                                </table><p>
+                                                                                As if this wasn't enough of a problem, the largest GreenPak4 device (the SLG46620V) is split into two halves with limited routing between them, and the GUI doesn't help the user manage this complexity at all - you have to draw your schematic in two halves and add "cross connections" between them.</p><p> The icing on the cake is that schematics are a pain to diff and collaborate on. Although GreenPak schematics are XML based, which is a touch better than binary, who wants to read a giant XML diff and try to figure out what's going on in the circuit?</p><p> This isn't going to be a post on the quirks of Silego's software, though - that would be boring. As it turns out, there's one more exciting feature of these chips that I didn't mention earlier: the configuration bitstream is 100% documented in the device datasheet! This is unheard of in the programmable logic world. As Nick of Arachnid Labs <a href="http://www.arachnidlabs.com/blog/2015/03/30/greenpak/" target="_blank">says</a>, the chip is "just dying for someone to write a VHDL or Verilog compiler for it". As you can probably guess by from the title of this post, I've been busy doing exactly that.</p><h2>
+                                                                                    Great! How does it work?</h2>
+                                                                                <br/><p> Rather than wasting time writing a synthesizer, I decided to write a GreenPak technology library for Clifford Wolf's excellent open source synthesis tool, <a href="http://www.clifford.at/yosys/" target="_blank">Yosys</a>, and then make a place-and-route tool to turn that into a final netlist. The post-PAR netlist can then be loaded into GreenPak Designer in order to program the device.</p><p> The first step of the process is to run the "synth_greenpak4" Yosys flow on the Verilog source. This runs a generic RTL synthesis pass, then some coarse-grained extraction passes to infer shift register and counter cells from behavioral logic, and finally maps the remaining logic to LUT/FF cells and outputs a JSON-formatted netlist.</p><p> Once the design has been synthesized, my tool (named, surprisingly, gp4par) is then launched on the netlist. It begins by parsing the JSON and constructing a directed graph of cell objects in memory. A second graph, containing all of the primitives in the device and the legal connections between them, is then created based on the device specified on the command line. (As of now only the SLG46620V is supported; the SLG46621V can be added fairly easily but the SLG46140V has a slightly different microarchitecture which will require a bit more work to support.)</p><p> After the graphs are generated, each node in the netlist graph is assigned a numeric label identifying the type of cell and each node in the device graph is assigned a list of legal labels: for example, an I/O buffer site is legal for an input buffer, output buffer, or bidirectional buffer.</p><table>
+                                                                                    <tbody>
+                                                                                        <tr>
+                                                                                            <td>
+                                                                                                <a href="https://2.bp.blogspot.com/-kIekczO693g/Vy7dBqYifXI/AAAAAAAAAxc/hMNJBs5bedIQOrBzzkhq4gbmhR-n58EQwCLcB/s1600/graph-labels.png" imageanchor="1" target="_blank"><img height="141" src="https://2.bp.blogspot.com/-kIekczO693g/Vy7dBqYifXI/AAAAAAAAAxc/hMNJBs5bedIQOrBzzkhq4gbmhR-n58EQwCLcB/s400/graph-labels.png" width="400"/></a>
+                                                                                            </td>
+                                                                                        </tr>
+                                                                                        <tr>
+                                                                                            <td>Example labeling for a subset of the netlist and device graphs</td>
+                                                                                        </tr>
+                                                                                    </tbody>
+                                                                                </table><p>
+                                                                                The labeled nodes now need to be placed. The initial placement uses a simple greedy algorithm to create a valid (although not necessarily optimal or even routable) placement:</p><ol>
+                                                                                    <li>Loop over the cells in the netlist. If any cell has a LOC constraint, which locks the cell to a specific physical site, attempt to assign the node to the specified site. If the specified node is the wrong type, doesn't exist, or is already used by another constrained node, the constraint is invalid so fail with an error.</li>
+                                                                                    <li>Loop over all of the unconstrained cells in the netlist and assign them to the first unused site with the right label. If none are available, the design is too big for the device so fail with an error.</li>
+                                                                                </ol><p>
+                                                                                Once the design is placed, the placement optimizer then loops over the design and attempts to improve it. A simulated annealing algorithm is used, where changes to the design are accepted unconditionally if they make the placement better, and with a random, gradually decreasing probability if they make it worse. The optimizer terminates when the design receives a perfect score (indicating an optimal placement) or if it stops making progress for several iterations. Each iteration does the following:</p><ol>
+                                                                                    <li>Compute a score for the current design based on the number of unroutable nets, the amount of routing congestion (number of nets crossing between halves of the device), and static timing analysis (not yet implemented, always zero).</li>
+                                                                                    <li>Make a list of nodes that contributed to this score in some way (having some attached nets unroutable, crossing to the other half of the device, or failing timing).</li>
+                                                                                    <li>Remove nodes from the list that are LOC'd to a specific location since we're not allowed to move them.</li>
+                                                                                    <li>Remove nodes from the list that have only one legal placement in the device (for example, oscillator hard IP) since there's nowhere else for them to go.</li>
+                                                                                    <li>Pick a node from the remainder of the list at random. Call this our pivot.</li>
+                                                                                    <li>Find a list of candidate placements for the pivot: </li>
+                                                                                    <ol>
+                                                                                        <li>Consider all routable placements in the other half of the device.</li>
+                                                                                        <li>If none were found, consider all routable placements anywhere in the device.</li>
+                                                                                        <li>If none were found, consider all placements anywhere in the device even if they're not routable.</li>
+                                                                                    </ol>
+                                                                                    <li>Pick one of the candidates at random and move the pivot to that location. If another cell in the netlist is already there, put it in the vacant site left by the pivot.</li>
+                                                                                    <li>Re-compute the score for the design. If it's better, accept this change and start the next iteration.</li>
+                                                                                    <li>If the score is worse, accept it with a random probability which decreases as the iteration number goes up. If the change is not accepted, restore the previous placement.</li>
+                                                                                </ol><p>
+                                                                                After optimization, the design is checked for routability. If any edges in the netlist graph don't correspond to edges in the device graph, the user probably asked for something impossible (for example, trying to hook a flipflop's output to a comparator's reference voltage input) so fail with an error.</p><p> The design is then routed. This is quite simple due to the crossbar structure of the device. For each edge in the netlist:<br/></p><ol>
+                                                                                    <li>If dedicated (non-fabric) routing is used for this path, configure the destination's input mux appropriately and stop.</li>
+                                                                                    <li>If the source and destination are in the same half of the device, configure the destination's input mux appropriately and stop.</li>
+                                                                                    <li>A cross-connection must be used. Check if we already used one to bring the source signal to the other half of the device. If found, configure the destination to route from that cross-connection and stop.</li>
+                                                                                    <li>Check if we have any cross-connections left going in this direction. If they're all used, the design is unroutable due to congestion so fail with an error.</li>
+                                                                                    <li>Pick the next unused cross-connection and configure it to route from the source. Configure the destination to route from the cross-connection and stop.</li>
+                                                                                </ol><p>
+                                                                                Once routing is finished, run a series of post-PAR design rule checks. These currently include the following:</p><ul>
+                                                                                    <li>If any node has no loads, generate a warning</li>
+                                                                                    <li>If an I/O buffer is connected to analog hard IP, fail with an error if it's not configured in analog mode.</li>
+                                                                                    <li>Some signals (such as comparator inputs and oscillator power-down controls) are generated by a shared mux and fed to many loads. If different loads require conflicting settings for the shared mux, fail with an error.</li>
+                                                                                </ul><p>
+                                                                                If DRC passes with no errors, configure all of the individual cells in the netlist based on the HDL parameters. Fail with an error if an invalid configuration was requested.</p><p> Finally, generate the bitstream from all of the per-cell configuration and write it to a file.</p><h2>
+                                                                                    Great, let's get started!</h2><p>
+                                                                                If you don't already have one, you'll need to buy a <a href="http://www.silego.com/buy/index.php?main_page=product_info&amp;products_id=388" target="_blank">GreenPak4 development kit</a>. The kit includes samples of the SLG46620V (among other devices) and a programmer/emulation board. While you're waiting for it to arrive, install <a href="http://www.silego.com/softdoc/software.html" target="_blank">GreenPak Designer</a>.</p><p> Download and install Yosys. Although Clifford is pretty good at merging my pull requests, only <a href="https://github.com/azonenberg/yosys/" target="_blank">my fork on Github</a> is guaranteed to have the most up-to-date support for GreenPak devices so don't be surprised if you can't use a bleeding-edge feature with mainline Yosys.</p><p> Download and install gp4par. You can get it from <a href="https://github.com/azonenberg/openfpga/" target="_blank">the Github repository</a>.</p><p> Write your HDL, compile with Yosys, P&amp;R with gp4par, and import the bitstream into GreenPak Designer to program the target device. The most current gp4par manual is included in LaTeX source form in the source tree and is automatically built as part of the compile process. If you're just browsing, there's a <a href="http://thanatos.virtual.antikernel.net/unlisted/gp4-hdl.pdf" target="_blank">relatively recent PDF version</a> on my web server.</p><p> If you'd like to see the Verilog that produced the nightmare of a schematic I showed above, <a href="https://github.com/azonenberg/openfpga/blob/master/tests/greenpak4/Blinky/Blinky.v" target="_blank">here it is</a>.</p><p> Be advised that this project is still very much a work in progress and there are still a number of SLG46620V features I don't support (see the manual for exact details).</p><h2>
+                                                                                    I love it / it segfaulted / there's a problem in the manual!</h2><p>
+                                                                                Hop in our IRC channel (##openfpga on Freenode) and let me know. Feedback is great, pull requests are even better,</p><h2>
+                                                                                    You're competing with Silego's IDE. Have they found out and sued you yet?</h2><p>
+                                                                                Nope. They're fully aware of what I'm doing and are rolling out the red carpet for me. They love the idea of a HDL flow as an alternative to schematic entry and are pretty amazed at how fast it's coming together.</p><p> After I reported a few bugs in their datasheets they decided to skip the middleman and give me direct access to the engineer who writes their documentation so that I can get faster responses. The last time I found a problem (two different parts of the datasheet contradicted each other) an updated datasheet was in my inbox and on their website by the next day. I only wish Xilinx gave me that kind of treatment!</p><p> They've even <a href="https://twitter.com/SilegoTech/status/717018987771469824" target="_blank">offered me free hardware</a> to help me add support for their latest product family, although I plan to get GreenPak4 support to a more stable state before taking them up on the offer.</p><h2>
+                                                                                    So what's next?</h2>
+                                                                                <br/><p> Better testing, for starters. I have to verify functionality by hand with a DMM and oscilloscope, which is time consuming.</p><p> My contact at Silego says they're going to be giving me documentation on the SRAM emulation interface soon, so I'm going to make a hardware-in-loop test platform that connects to my desktop and the Silego ZIF socket, and lets me load new bitstreams via a scriptable interface. It'll have FPGA-based digital I/O as well as an ADC and DAC on every device pin, plus an adjustable voltage regulator for power, so I can feed in arbitrary mixed-signal test waveforms and write PC-based unit tests to verify correct behavior.</p><p> Other than that, I want to finish support for the SLG46620V in the next month or two. The SLG46621V will be an easy addition since only one pin and the relevant configuration bits have changed from the 46620 (I suspect they're the same die, just bonded out differently).</p><p> Once that's done I'll have to do some more extensive work to add the SLG46140V since the architecture is a bit different (a lot of the combinatorial logic is merged into multi-function blocks). Luckily, the 46140 has a lot in common architecturally with the GreenPak5 family, so once that's done GreenPak5 will probably be a lot easier to add support for.</p><p> My thanks go out to Clifford Wolf, whitequark, the IRC users in ##openfpga, and everyone at Silego I've worked with to help make this possible. I hope that one day this project will become mature enough that Silego will ship it as an officially supported extension to GreenPak Designer, making history by becoming the first modern programmable logic vendor to ship a fully open source synthesis and P&amp;R suite.
+                                                                                </p>
+                                                                            </div></article>

article_scraper/resources/tests/readability/breitbart/expected.html

@@ -0,0 +1,47 @@
+<article><DIV id="readability-page-1"><DIV>
+                            
+                            
+                            
+                            <figure>
+                                <div><p><img itemprop="image" src="http://media.breitbart.com/media/2016/11/GettyImages-621866810-640x480.jpg" alt="Supporters of Republican presidential nominee Donald Trump cheer during election night at the New York Hilton Midtown in New York on November 9, 2016.  / AFP / JIM WATSON        (Photo credit should read JIM WATSON/AFP/Getty Images)" width="640" height="480"/></p><p>JIM WATSON/AFP/Getty Images</p>
+                                </div>
+                            </figure>
+                             <time datetime="2016-12-22T10:43:37Z">22 Dec, 2016</time>
+                            <time datetime="2016-12-22T18:59:12Z">22 Dec, 2016</time>
+                        </DIV><div>
+                            
+                            
+                            <div id="EmailOptin">
+                                <p><span>SIGN UP</span> FOR OUR NEWSLETTER</p>
+                                
+                            </div>
+                            <h2><span>Snopes fact checker and staff writer David Emery posted to Twitter asking if there were “any un-angry Trump supporters?”</span></h2>
+                            <p><span>Emery, a writer for partisan “fact-checking” website Snopes.com which soon will be in charge of labelling </span><a href="http://www.breitbart.com/tech/2016/12/15/facebook-introduce-warning-labels-stories-deemed-fake-news/" target="_blank"><span>“fake news”</span></a><span> alongside ABC News and Politifact, retweeted an article by Vulture magazine relating to the </span><a href="http://www.breitbart.com/big-hollywood/2016/11/19/boycotthamilton-trends-hamilton-cast-members-harass-mike-pence/" target="_blank"><span>protests</span></a><span> of the <em>Hamilton</em> musical following the decision by the cast of the show to make a </span><a href="http://www.breitbart.com/big-hollywood/2016/11/19/tolerance-hamilton-cast-lectures-mike-pence-broadway-stage/" target="_blank"><span>public announcement</span></a><span> to Vice-president elect Mike Pence while he watched the performance with his family.</span></p>
+                            <div id="EmailOptinM">
+                                <p><span>SIGN UP</span> FOR OUR NEWSLETTER</p>
+                                
+                            </div>
+                            <p><span>The tweet from Vulture magazine reads, “</span><a href="https://twitter.com/hashtag/Hamilton?src=hash" target="_blank" rel="noopener"><span>#Hamilton</span></a><span> Chicago show interrupted by angry Trump supporter.” Emery retweeted the story, saying, “Are there un-angry Trump supporters?”</span></p>
+                            
+                            
+
+                            
+
+
+                            <p><span>This isn’t the first time the Snopes.com writer has expressed anti-Trump sentiment on his Twitter page. In another tweet in which Emery links to an article that falsely attributes a quote to President-elect Trump, Emery states, “Incredibly, some people actually think they have to put words in Trump’s mouth to make him look bad.”</span></p>
+                            
+                            
+                            <p><span>Emery also retweeted an article by <em>New York</em> magazine that claimed President-elect Trump relied on lies to win during his campaign and that we now lived in a “post-truth” society. “Before long we’ll all have forgotten what it was like to live in the same universe; or maybe we already have,” Emery tweeted.</span></p>
+
+                            
+
+
+                            
+                            
+                            <p><span>Facebook believe that Emery, along with other Snopes writers, ABC News, and </span><a href="http://www.breitbart.com/tech/2016/12/16/flashback-weekly-standard-data-shows-politifact-has-it-out-for-republicans/" target="_blank"><span>Politifact</span></a><span> are impartial enough to label and silence what they believe to be “fake news” on social media. </span></p>
+                            <p><i><span>Lucas Nolan is a reporter for Breitbart Tech covering issues of free speech and online censorship. Follow him on Twitter </span></i><a href="http://twitter.com/lucasnolan_" target="_blank" rel="noopener"><i><span>@LucasNolan_</span></i></a><i><span> or email him at </span></i><a href="http://www.breitbart.com/wp-admin/blank" target="_blank"><i><span>lnolan@breitbart.com</span></i></a></p>
+
+                            
+
+
+                        </div></DIV></article>

article_scraper/resources/tests/readability/bug-1255978/expected.html

@@ -0,0 +1,136 @@
+<article><div itemprop="articleBody" id="readability-page-1">
+                            <p>Most people go to hotels for the pleasure of sleeping in a giant bed with clean white sheets and waking up to fresh towels in the morning.</p>
+
+                            <p>But those towels and sheets might not be as clean as they look, according to the hotel bosses that responded to an online thread about the things hotel owners don’t want you to know.</p>
+
+                            <p>Zeev Sharon and Michael Forrest Jones both run hotel start-ups in the US. Forrest Jones runs the start-up Beechmont Hotels Corporation, a hotel operating company that consults with hotel owners on how they can improve their business. Sharon is the CEO of Hotelied, a start-up that allows people to sign up for discounts at luxury hotels.</p>
+
+                            <p>But even luxury hotels aren’t always cleaned as often as they should be.</p>
+
+                            <p>Here are some of the secrets that the receptionist will never tell you when you check in, according to answers posted on <a href="https://www.quora.com/What-are-the-things-we-dont-know-about-hotel-rooms" target="_blank">Quora</a>.</p>
+
+                            
+
+                            <div>
+                                <div>
+                                    
+                                    <p><img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2014/03/18/10/bandb2.jpg" alt="bandb2.jpg" title="bandb2.jpg" width="564" height="423"/></p>
+                                    
+                                </div>
+                                <p>Even posh hotels might not wash a blanket in between stays
+                                    
+                                </p>
+                            </div>
+
+                            <p>1. Take any blankets or duvets off the bed</p>
+
+                            <p>Forrest Jones said that anything that comes into contact with any of the previous guest’s skin should be taken out and washed every time the room is made, but that even the fanciest hotels don’t always do so. "Hotels are getting away from comforters. Blankets are here to stay, however. But some hotels are still hesitant about washing them every day if they think they can get out of it," he said.</p>
+
+                            <div>
+                                <div data-video-id="4685984084001" data-embed="default" data-player="2d3d4a83-ba40-464e-9bfb-2804b076bf67" data-account="624246174001" id="4685984084001" role="region" aria-label="video player">
+                                                
+                                                
+                                                
+                                                
+                                                <p><span>Play Video</span></p>
+                                                
+                                                
+                                                
+                                                
+                                                
+                                            </div>
+                                <p>Video shows bed bug infestation at New York hotel</p>
+                            </div>
+
+                            
+
+                            <div>
+                                <div>
+                                    
+                                    <p><img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2015/05/26/11/hotel-door-getty.jpg" alt="hotel-door-getty.jpg" title="hotel-door-getty.jpg" width="564" height="423"/></p>
+                                    
+                                </div>
+                                <p>Forrest Jones advised stuffing the peep hole with a strip of rolled up notepaper when not in use.
+                                    
+                                </p>
+                            </div>
+
+                            <p>2. Check the peep hole has not been tampered with</p>
+
+                            <p>This is not common, but can happen, Forrest Jones said. He advised stuffing the peep hole with a strip of rolled up notepaper when not in use. When someone knocks on the door, the paper can be removed to check who is there. If no one is visible, he recommends calling the front desk immediately. “I look forward to the day when I can tell you to choose only hotels where every employee who has access to guestroom keys is subjected to a complete public records background check, prior to hire, and every year or two thereafter. But for now, I can't,” he said.</p>
+
+                            
+
+                            <div>
+                                <div>
+                                    
+                                    <p><img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2013/07/31/15/luggage-3.jpg" alt="luggage-3.jpg" title="luggage-3.jpg" width="564" height="423"/></p>
+                                    
+                                </div>
+                                <p>Put luggage on the floor
+                                    
+                                </p>
+                            </div>
+
+                            <p>3. Don’t use a wooden luggage rack</p>
+
+                            <p>Bedbugs love wood. Even though a wooden luggage rack might look nicer and more expensive than a metal one, it’s a breeding ground for bugs. Forrest Jones says guests should put the items they plan to take from  bags on other pieces of furniture and leave the bag on the floor.</p>
+
+                            
+
+                            <div>
+                                <div>
+                                    
+                                    <p><img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2015/04/13/11/Lifestyle-hotels.jpg" alt="Lifestyle-hotels.jpg" title="Lifestyle-hotels.jpg" width="564" height="423"/></p>
+                                    
+                                </div>
+                                <p>The old rule of thumb is that for every 00 invested in a room, the hotel should charge in average daily rate
+                                    
+                                </p>
+                            </div>
+
+                            <p>4. Hotel rooms are priced according to how expensive they were to build</p>
+
+                            <p>Zeev Sharon said that the old rule of thumb is that for every $1000 invested in a room, the hotel should charge $1 in average daily rate. So a room that cost $300,000 to build, should sell on average for $300/night.</p>
+
+                            
+
+                            <h3>5. Beware the wall-mounted hairdryer</h3>
+
+                            <p>It contains the most germs of anything in the room. Other studies have said the TV remote and bedside lamp switches are the most unhygienic. “Perhaps because it's something that's easy for the housekeepers to forget to check or to squirt down with disinfectant,” Forrest Jones said.</p>
+
+                            
+
+                            
+
+                            <h3>6. Mini bars almost always lose money</h3>
+
+                            <p>Despite the snacks in the minibar seeming like the most overpriced food you have ever seen, hotel owners are still struggling to make a profit from those snacks. "Minibars almost always lose money, even when they charge $10 for a Diet Coke,” Sharon said.</p>
+
+                            
+
+                            <div>
+                                <div>
+                                    
+                                    <p><img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2014/03/13/16/agenda7.jpg" alt="agenda7.jpg" title="agenda7.jpg" width="564" height="423"/></p>
+                                    
+                                </div>
+                                <p>Towels should always be cleaned between stays
+                                    
+                                </p>
+                            </div>
+
+                            <p>7. Always made sure the hand towels are clean when you arrive</p>
+
+                            <p>Forrest Jones made a discovery when he was helping out with the housekeepers. “You know where you almost always find a hand towel in any recently-vacated hotel room that was occupied by a guy?  On the floor, next to the bed, about halfway down, maybe a little toward the foot of the bed.  Same spot in the floor, next to almost every bed occupied by a man, in every room. I'll leave the rest to your imagination,” he said.</p>
+                            <meta itemprop="datePublished" content="2016-05-08T10:11:51+01:00"/>
+                            <ul>
+                                <li>
+                                    More about:
+                                </li>
+                                <li><a itemprop="keywords" href="http://fakehost/topic/Hotels" target="_blank">Hotels</a></li>
+                                <li><a itemprop="keywords" href="http://fakehost/topic/Hygiene" target="_blank">Hygiene</a></li>
+                            </ul>
+                            
+                            <p><a href="http://fakehost/syndication/reuse-permision-form?url=http://www.independent.co.uk/news/business/news/seven-secrets-that-hotel-owners-dont-want-you-to-know-10506160.html" target="_blank"><img src="http://fakehost/sites/all/themes/ines_themes/independent_theme/img/reuse.png" width="25"/>Reuse content</a>
+                        </p></div></article>

article_scraper/resources/tests/readability/buzzfeed-1/expected.html

@@ -0,0 +1,41 @@
+<article><div id="readability-page-1">
+                                    <div id="superlist_3758406_5547137" rel:buzz_num="1">
+                                        <h2>The mother of a woman who took suspected diet pills bought online has described how her daughter was “literally burning up from within” moments before her death.</h2>
+                                        
+                                        <p> <span>West Merica Police</span></p>
+                                    </div>
+                                    <div id="superlist_3758406_5547213" rel:buzz_num="2">
+                                        <p>Eloise Parry, 21, was taken to Royal Shrewsbury hospital on 12 April after taking a lethal dose of highly toxic “slimming tablets”. </p>
+                                        <p>“The drug was in her system, there was no anti-dote, two tablets was a lethal dose – and she had taken eight,” her mother, Fiona, <a href="https://www.westmercia.police.uk/article/9501/A-tribute-to-Eloise-Aimee-Parry-written-by-her-mother-Fiona-Parry" target="_blank">said in a statement</a> yesterday.</p>
+                                        <p>“As Eloise deteriorated, the staff in A&amp;E did all they could to stabilise her. As the drug kicked in and started to make her metabolism soar, they attempted to cool her down, but they were fighting an uphill battle.</p>
+                                        <p>“She was literally burning up from within.”</p>
+                                        <p>She added: “They never stood a chance of saving her. She burned and crashed.”</p>
+                                    </div>
+                                    <div id="superlist_3758406_5547140">
+                                            <div>
+                                                <div>
+                                                    <p><img src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608056-15.jpg" rel:bf_image_src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608056-15.jpg" height="412" width="203"/></p>
+                                                </div>
+                                                <p>Facebook</p>
+                                            </div>
+                                            <div>
+                                                <div>
+                                                    <p><img src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608057-18.jpg" rel:bf_image_src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608057-18.jpg" height="412" width="412"/></p>
+                                                </div>
+                                                <p>Facebook</p>
+                                            </div>
+                                            
+                                        </div>
+                                    <div id="superlist_3758406_5547284" rel:buzz_num="4">
+                                        <p>West Mercia police <a href="https://www.westmercia.police.uk/article/9500/Warning-Issued-As-Shrewsbury-Woman-Dies-After-Taking-Suspected-Diet-Pills" target="_blank">said the tablets were believed to contain dinitrophenol</a>, known as DNP, which is a highly toxic industrial chemical. </p>
+                                        <p>“We are undoubtedly concerned over the origin and sale of these pills and are working with partner agencies to establish where they were bought from and how they were advertised,” said chief inspector Jennifer Mattinson from the West Mercia police.</p>
+                                        <p>The Food Standards Agency warned people to stay away from slimming products that contained DNP.</p>
+                                        <p>“We advise the public not to take any tablets or powders containing DNP, as it is an industrial chemical and not fit for human consumption,” it said in a statement.</p>
+                                    </div>
+                                    <div id="superlist_3758406_5547219" rel:buzz_num="5">
+                                        <h2>Fiona Parry issued a plea for people to stay away from pills containing the chemical.</h2>
+                                        <br/>
+                                        <p>“[Eloise] just never really understood how dangerous the tablets that she took were,” she said. “Most of us don’t believe that a slimming tablet could possibly kill us.</p>
+                                        <p>“DNP is not a miracle slimming pill. It is a deadly toxin.”</p>
+                                    </div>
+                                </div></article>

article_scraper/resources/tests/readability/citylab-1/expected.html

@@ -0,0 +1,226 @@
+<article><DIV id="readability-page-1"><article itemscope="itemscope" itemtype="https://schema.org/NewsArticle">
+                        <meta itemprop="datePublished" content="2019-04-30T13:39:00-04:00"/>
+                        <meta itemprop="dateModified" content="2019-04-30T13:40:00-04:00"/>
+                        <meta itemprop="mainEntityOfPage" content="https://www.citylab.com/design/2019/04/neon-signage-20th-century-history/588400/"/>
+                        
+                        <figure itemprop="image" itemscope="itemscope" itemtype="http://schema.org/ImageObject">
+                            <picture>
+                                <source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/940.jpg?mod=1556645448" media="(min-width: 1024px)"/>
+                                <source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/lead_large.jpg?mod=1556645448" media="(min-width: 576px)"/>
+                            </picture>
+                            <meta itemprop="height" content="128"/>
+                            <meta itemprop="width" content="300"/>
+                            <meta itemprop="url" content="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448"/>
+                            <picture>
+                                <source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448" media="(max-width: 575px)"/>
+                                <img src="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448" alt="" srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448"/>
+                                
+                            </picture>
+                            <figcaption>
+                                <span itemprop="caption">The Moulin Rouge cabaret in
+                                    Paris</span> <span itemprop="creator">Benoit
+                                    Tessier/Reuters</span>
+                            </figcaption>
+                        </figure>
+                        <div>
+                            <h2 itemprop="headline">
+                                Why Neon Is the Ultimate Symbol of the 20th Century
+                            </h2>
+                            
+                        </div>
+                        <h2 itemprop="description">
+                            The once-ubiquitous form of lighting was novel when it first emerged in the early 1900s,
+                            though it has since come to represent decline.
+                        </h2>
+                        
+                        <section id="article-section-1">
+                            <p>
+                                In the summer of 1898, the Scottish chemist Sir William Ramsay made a discovery that
+                                would eventually give the Moulin Rouge in Paris, the Las Vegas Strip, and New York’s
+                                Times Square their perpetual nighttime glow. Using the boiling point of argon as a
+                                reference point, Ramsay and his colleague Morris W. Travers isolated three more noble
+                                gases and gave them evocative Greek names: neon, krypton, and xenon. In so doing, the
+                                scientists bestowed a label of permanent novelty on the most famous of the trio—neon,
+                                which translates as “new.” This discovery was the foundation on which the French
+                                engineer Georges Claude crafted a new form of illumination over the next decade. He
+                                designed glass tubes in which neon gas could be trapped, then electrified, to create a
+                                light that glowed reliably for more than 1,000 hours.
+                            </p>
+                            <p>
+                                In the 2012 book <em>L’être et le Néon</em>, <a href="https://mitpress.mit.edu/books/being-and-neonness-translation-and-content-revised-augmented-and-updated-edition-luis-de-miranda" target="_blank">which
+                                    has been newly translated into English by Michael Wells</a>, the philosopher Luis de
+                                Miranda weaves a history of neon lighting as both artifact and metaphor. <em>Being and
+                                    Neonness</em>, as the book is called in its English edition, isn’t a typical
+                                material history. There are no photographs. Even de Miranda’s own example of a neon deli
+                                sign spotted in Paris is re-created typographically, with text in all caps and dashes
+                                forming the border of the sign, as one might attempt on Twitter. Fans of Miami Beach’s
+                                restored Art Deco hotels and California’s bowling alleys might be disappointed by the
+                                lack of glossy historical images. Nonetheless, de Miranda makes a convincing case for
+                                neon as a symbol of the grand modern ambitions of the 20th century.
+                            </p>
+                            
+                            <p>
+                                De Miranda beautifully evokes the notion of neon lighting as an icon of the 1900s in his
+                                introduction: “When we hear the word <em>neon</em>, an image pops into our heads: a
+                                combination of light, colors, symbols, and glass. This image is itself a mood. It
+                                carries an atmosphere. It speaks … of the essence of cities, of the poetry of nights, of
+                                the 20th century.” When neon lights debuted in Europe, they seemed dazzlingly
+                                futuristic. But their husky physicality started becoming obsolete by the 1960s, thanks
+                                in part to the widespread use of plastic for fluorescent signs. Neon signs exist today,
+                                though they’ve been eclipsed by newer technologies such as digital billboards, and they
+                                remain charmingly analog: Signs must be made by hand because there’s no cost-effective
+                                way to mass-produce them.
+                            </p>
+                            <p>
+                                In the 1910s, neon started being used for cosmopolitan flash in Paris at precisely the
+                                time and place where the first great modernist works were being created. De Miranda’s
+                                recounting of the ingenuity emerging from the French capital a century ago is thrilling
+                                to contemplate: the cubist art of Pablo Picasso, the radically deconstructed fashions of
+                                Coco Chanel, the stream-of-consciousness poetry of Gertrude Stein, and the genre-defying
+                                music of Claude Debussy—all of which heralded a new age of culture for Europe and for
+                                the world.
+                            </p>
+                        </section>
+                        
+                        <section id="article-section-2">
+                            <p>
+                                Amid this artistic groundswell, Georges Claude premiered his neon lights at the <a href="https://www.mondial-paris.com/en/visiteur/auto" target="_blank">Paris Motor Show</a> in
+                                December 1910, captivating visitors with 40-foot-tall tubes affixed to the building’s
+                                exterior. The lights shone orange-red because neon, by itself, produces that color.
+                                <em>Neon lighting</em> is a catchall term that describes the technology of glass tubing
+                                that contains gas or chemicals that glow when electrified. For example, neon fabricators
+                                use carbon dioxide to make white, and mercury to make blue. Claude acknowledged at the
+                                time that neon didn’t produce the ideal color for a standard light bulb and insisted
+                                that it posed no commercial threat to incandescent bulbs.
+                            </p>
+                            <p>
+                                Of course, the very quality that made neon fixtures a poor choice for interior lighting
+                                made them perfect for signs, de Miranda notes. The first of the neon signs was switched
+                                on in 1912, advertising a barbershop on Paris’s Boulevard Montmartre, and eventually
+                                they were adopted by cinemas and nightclubs. While Claude had a monopoly on neon
+                                lighting throughout the 1920s, the leaking of trade secrets and the expiration of a
+                                series of patents broke his hold on the rapidly expanding technology.
+                            </p>
+                        </section>
+                        
+                        <section id="article-section-3">
+                            <p>
+                                In the following decades, neon’s nonstop glow and vibrant colors turned ordinary
+                                buildings and surfaces into 24/7 billboards for businesses, large and small, that wanted
+                                to convey a sense of always being open. The first examples of neon in the United States
+                                debuted in Los Angeles, where the Packard Motor Car Company commissioned two large
+                                blue-and-orange <span>Packard</span> signs that literally stopped
+                                traffic because they distracted motorists. The lighting also featured heavily at the
+                                Chicago Century of Progress Exposition in 1933 and at the 1939 World’s Fair in New York.
+                                At the latter event, a massive neon sign reading <span>Futurama</span>
+                                lit the way to a General Motors exhibition that heralded “The World of Tomorrow.”
+                            </p>
+                            <figure>
+                                <picture>
+                                    <img alt="" data-srcset="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_8912060228/cbd32b0e1.jpg" src="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_8912060228/cbd32b0e1.jpg"/>
+                                </picture>
+                                <figcaption>
+                                    Workers remove a hammer and sickle from a neon sign that reads “Glory to Communism,”
+                                    visible on the roof of the Communist-run electricity-board headquarters in
+                                    Czechoslovakia in 1989. (AP)
+                                </figcaption>
+                            </figure>
+                            <p>
+                                De Miranda points out that businesses weren’t alone in embracing neon’s ability to
+                                spread messages effectively. By the middle of the century, the lighting was being
+                                adopted for more political purposes. “In the 1960s, the Soviets deployed a vast
+                                ‘neonization’ of the Eastern bloc capitals to emulate capitalist metropolises,” de
+                                Miranda writes. “Because consumer shops were rare in the Polish capital [of Warsaw],
+                                they did not hesitate to illuminate the façades of public buildings.” In other words, as
+                                opposed to the sole use of the more obvious forms of propaganda via posters or slogans,
+                                the mass introduction of neon lighting was a way of getting citizens of Communist cities
+                                to see their surroundings with the pizzazz and nighttime glamour of major Western
+                                capitals.
+                            </p>
+                        </section>
+                        
+                        <section id="article-section-4">
+                            <p>
+                                Neon, around this time, began to be phased out, thanks to cheaper and less
+                                labor-intensive alternatives. In addition, the global economic downturn of the 1970s
+                                yielded a landscape in which older, flickering neon signs, which perhaps their owners
+                                couldn’t afford to fix or replace, came to look like symbols of decline. Where such
+                                signs were once sophisticated and novel, they now seemed dated and even seedy.
+                            </p>
+                            <section>
+                                <h2>
+                                    Cities are changing fast. Keep up with the <b>CityLab Daily</b> newsletter.
+                                </h2><label for="promo-email-input-email">The best way to follow issues you
+                                    care about.</label>
+                                
+                            </section>
+                            <p>
+                                De Miranda understands this evolution by zooming out and looking at the 1900s as the
+                                “neon century.” The author draws a parallel between the physical form of neon lights,
+                                which again are essentially containers for electrified gases, and that of a glass
+                                capsule—suggesting they are a kind of message in a bottle from a time before the First
+                                World War. “Since then, [neon lights] have witnessed all the transformations that have
+                                created the world we live in,” de Miranda writes. “Today, they sometimes seem to
+                                maintain a hybrid status, somewhere between junkyards and museums, not unlike European
+                                capitals themselves.”
+                            </p>
+                            <figure>
+                                <picture>
+                                    <img alt="" data-srcset="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_945361213236/888fdd750.jpg" src="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_945361213236/888fdd750.jpg"/>
+                                </picture>
+                                <figcaption>
+                                    Martin Wartman, a student at Northern Kentucky University, works on a neon sign at
+                                    the Neonworks of Cincinnati workshop connected to the American Sign Museum, in 2016.
+                                    (John Minchillo / AP)
+                                </figcaption>
+                            </figure>
+                            <p>
+                                Another mark of neon’s hybridity: Its obsolescence started just as some contemporary
+                                artists began using the lights in their sculptures. Bruce Nauman’s 1968 work <em><a href="https://www.stedelijk.nl/en/collection/1097-bruce-nauman-my-name-as-though-it-were-written-on-the-surface-of-the-moon" target="_blank">My
+                                        Name as Though It Were Written on the Surface of the Moon</a></em> poked fun at
+                                the space race—another symbol of 20th-century technological innovation whose moment has
+                                passed. The piece uses blue “neon” letters (mercury, actually) to spell out the name
+                                “bruce” in lowercase cursive, with each character repeated several times as if to convey
+                                a person speaking slowly in outer space. The British artist Tracey Emin has made <a href="https://www.artsy.net/collection/tracey-emin-neon-sculptures-and-prints" target="_blank">sculptures</a>
+                                that resemble neon Valentine’s Day candies: They read as garish and sentimental
+                                confections with pink, heart-shaped frames that surround blue text fragments. Drawing on
+                                the nostalgia-inducing quality of neon, the sculptures’ messages are redolent of
+                                old-fashioned movie dialogue, with titles such as “You Loved Me Like a Distant Star” and
+                                “The Kiss Was Beautiful.”
+                            </p>
+                            <p>
+                                Seeing neon lighting tamed in the context of a gallery display fits comfortably with de
+                                Miranda’s notion that neon technology is like a time capsule from another age. In
+                                museums, works of neon art and design coexist with objects that were ahead of their own
+                                time in years past—a poignant fate for a technology that made its name advertising “The
+                                World of Tomorrow.” Yet today neon is also experiencing a kind of craft revival. The
+                                fact that it can’t be mass-produced has made its fabrication something akin to a
+                                cherished artisanal technique. Bars and restaurants hire firms such as Let There Be Neon
+                                in Manhattan, or <a href="https://www.instagram.com/theneonqueen/" target="_blank">the L.A.-based master
+                                    neon artist Lisa Schulte</a>, to create custom signs and works of art. Neon’s story
+                                even continues to glow from inside museums such as California’s <a href="https://www.neonmona.org/" target="_blank">Museum of Neon Art</a> and the Neon Museum in Las
+                                Vegas. If it can still be a vital medium for artists and designers working today,
+                                “neonness” need not only be trapped in the past. It might also capture the mysterious
+                                glow of the near future—just as it did a century ago.
+                            </p>
+                            <p>
+                                <em>This article originally appeared on <a href="https://www.theatlantic.com/entertainment/archive/2019/04/being-and-neonness-neon-lights-symbol-20th-century/588184/" target="_blank">The
+                                        Atlantic</a>.</em>
+                            </p>
+                        </section>
+                        <section data-include="css:https://cdn.citylab.com/static/a/frontend/dist/citylab/css/components/author-article.cf4e8e0b143f.css">
+                            <h4>
+                                About the Author
+                            </h4>
+                            <div itemprop="author">
+                                    <h5 itemprop="name">
+                                        <a href="https://www.citylab.com/authors/sarah-archer/" target="_blank">Sarah Archer</a>
+                                    </h5>
+                                    
+                                    <p itemprop="description">
+                                        <a href="https://www.citylab.com/authors/sarah-archer/" data-omni-click="inherit" target="_blank">Sarah Archer</a> is the author of <em>The
+                                            Midcentury Kitchen</em>.
+                                    </p>
+                                </div>
+                        </section>
+                    </article></DIV></article>

article_scraper/resources/tests/readability/cnet/expected.html

@@ -0,0 +1,46 @@
+<article><div itemprop="articleBody" data-component="lazyloadImages" id="readability-page-1">
+
+
+
+
+                                        <figure section="shortcodeImage"><span><span itemprop="image" itemscope="" itemtype="https://schema.org/ImageObject"><img src="https://cnet1.cbsistatic.com/img/nAMdBzIE1ogVw5bOBZBaiJCt3Ro=/570x0/2014/03/21/863df5d9-e8b8-4b38-851b-5e3f77f2cf0e/mark-zuckerberg-facebook-home-10671610x407.jpg" alt="" width="570" height="0"/><meta itemprop="url" content="https://cnet1.cbsistatic.com/img/nAMdBzIE1ogVw5bOBZBaiJCt3Ro=/570x0/2014/03/21/863df5d9-e8b8-4b38-851b-5e3f77f2cf0e/mark-zuckerberg-facebook-home-10671610x407.jpg"/><meta itemprop="height" content="0"/><meta itemprop="width" content="570"/></span></span>
+                                            <figcaption><span><p>Facebook CEO Mark Zuckerberg, the man with the acquisition plan.</p></span><span>Photo by                                            James Martin/CNET
+                                        </span></figcaption>
+                                            </figure>
+                                            <p>Anyone who has ever been involved in closing a billion-dollar acquisition deal will tell you that you don't go in without a clear, well thought out plan.</p>
+                                            
+                                            <p>Facebook CEO Mark Zuckerberg knows a thing or two about how to seal the deal on blockbuster buys. After all, he's the man behind his company's <a href="https://www.cnet.com/news/facebook-closes-19-billion-deal-for-whatsapp/" target="_blank">$19 billion acquisition</a> of WhatsApp, he <a href="https://www.cnet.com/news/zuckerberg-did-1-billion-instagram-deal-on-his-own/" target="_blank">personally brokered</a> its $1 billion buyout of <a href="https://www.cnet.com/news/why-facebook-plunked-down-1-billion-to-buy-instagram/" target="_blank">Instagram</a> and closed the <a href="https://www.cnet.com/news/facebook-to-buy-oculus-for-2-billion/" target="_blank">$3 billion deal</a> to buy Oculus VR.</p>
+                                            <p>Zuckerberg offered a primer on the strategies he and his company employ when they see an attractive target during testimony Tuesday <a href="https://www.cnet.com/news/zenimax-sues-oculus-over-virtual-reality-rift-tech/" target="_blank">in a lawsuit with ZeniMax Media</a>, which accuses Oculus and Facebook of "misappropriating" trade secrets and copyright infringement. At the heart of the lawsuit is technology that helped create liftoff for virtual reality, one of the <a href="http://www.cbsnews.com/videos/the-reality-of-the-virtual-world/" target="_blank" data-component="externalLink">hottest gadget trends today.</a></p>
+                                            <p>A key Facebook approach is building a long-term relationship with your target, Zuckerberg said at the trial. These deals don't just pop up over night, he said according to a transcript reviewed by <a href="http://www.businessinsider.com/mark-zuckerberg-explains-facebooks-acquisition-strategy-2017-1" target="_blank" data-component="externalLink">Business Insider</a>. They take time to cultivate. </p>
+                                            <blockquote>
+                                                <p>I've been building relationships, at least in Instagram and the WhatsApp cases, for years with the founders and the people that are involved in these companies, which made [it] so that when it became time or when we thought it was the right time to move, we felt like we had a good amount of context and had good relationships so that we could move quickly, which was competitively important and why a lot of these acquisitions, I think, came to us instead of our competitors and ended up being very good acquisitions over time that a lot of competitors wished they had gotten instead. </p>
+                                            </blockquote>
+                                            <p> He also stressed the need assure your target that you have a shared vision about how you will collaborate after the deal is put to bed. Zuckerberg said this was reason Facebook was able to acquire Oculus for less than its original $4 billion asking price.</p>
+                                            <blockquote>If this [deal] is going to happen, it's not going to be because we offer a lot of money, although we're going to have to offer a fair price for the company that is more than what they felt like they could do on their own. But they also need to feel like this was actually going to help their mission.</blockquote>
+
+
+
+
+
+                                            
+                                            <p>When that doesn't work, Zuckerberg said scare tactics is an effective, if undesirable, way of persuading small startups that they face a better chance of survival if they have Facebook to guide their way rather than going it alone.</p>
+                                            <blockquote>That's less my thing, but I think if you are trying to help convince people that they want to join you, helping them understand all the pain that they would have to go through to build it out independently is a valuable tactic. </blockquote>
+                                            <p>It also pays to be weary of competing suitors for your startup, Zuckerberg said, and be willing to move fast to stave off rivals and get the deal done.</p>
+                                            <blockquote>Often, if a company knows we're offering something, they will offer more. So being able to move quickly not only increases our chance of being able to get a deal done if we want to, but it makes it so we don't have end up having to pay a lot more because the process drags out.</blockquote>
+                                            <p>It wasn't clear why these strategies didn't work on Snapchat CEO Evan Spiegel, who <a href="https://www.cnet.com/news/snapchat-said-to-rebuff-3-billion-offer-from-facebook/" target="_blank">famously rebuffed</a> a $3 billion takeover offer from Facebook in 2013.</p>
+
+
+
+
+
+                                            
+                                            <p><em><strong>Tech Enabled:</strong> CNET chronicles tech's role in providing new kinds of accessibility. Check it out <a href="https://www.cnet.com/tech-enabled/" target="_blank">here</a>.</em><em><strong><br/></strong></em></p>
+                                            <p><em><strong>Technically Literate:</strong> Original works of short fiction with unique perspectives on tech, exclusively on CNET. <a href="https://www.cnet.com/technically-literate/" target="_blank">Here</a>.</em></p>
+
+
+
+                                            
+
+
+                                            
+                                    </div></article>

article_scraper/resources/tests/readability/cnn/expected.html

@@ -0,0 +1,51 @@
+<article><div id="readability-page-1">
+                            
+                            
+                            <h2>The U.S. has long been heralded as a land of opportunity -- a place where anyone can succeed regardless of the economic class they were born into.</h2>
+                            <p> But a new report released on Monday by <a href="http://web.stanford.edu/group/scspi-dev/cgi-bin/" target="_blank">Stanford University's Center on Poverty and Inequality</a> calls that into question. </p>
+                            
+                            <p> The report assessed poverty levels, income and wealth inequality, economic mobility and unemployment levels among 10 wealthy countries with social welfare programs. </p>
+                            <div id="smartassetcontainer">
+                                                
+
+
+                                                
+
+                                                <p>
+                                                    Powered by SmartAsset.com
+                                                </p>
+
+                                                
+
+                                                
+
+
+                                                
+
+
+                                            </div>
+                            <p> Among its key findings: the class you're born into matters much more in the U.S. than many of the other countries. </p>
+                            <p> As the <a href="http://web.stanford.edu/group/scspi-dev/cgi-bin/publications/state-union-report" target="_blank">report states</a>: "[T]he birth lottery matters more in the U.S. than in most well-off countries." </p>
+                            
+                            <p> But this wasn't the only finding that suggests the U.S. isn't quite living up to its reputation as a country where everyone has an equal chance to get ahead through sheer will and hard work. </p>
+                            <p> <a href="http://money.cnn.com/2016/01/11/news/economy/rich-taxes/index.html?iid=EL" target="_blank"><span>Related: Rich are paying more in taxes but not as much as they used to</span></a> </p>
+                            
+                            <p> The report also suggested the U.S. might not be the "jobs machine" it thinks it is, when compared to other countries. </p>
+                            <p> It ranked near the bottom of the pack based on the levels of unemployment among men and women of prime working age. The study determined this by taking the ratio of employed men and women between the ages of 25 and 54 compared to the total population of each country. </p>
+                            <p> The overall rankings of the countries were as follows:<span> <br/>1. Finland <span> <br/>2. Norway<span> <br/>3. Australia <span> <br/>4. Canada<span> <br/>5. Germany<span> <br/>6. France<span> <br/>7. United Kingdom <span> <br/>8. Italy<span> <br/>9. Spain<span> <br/>10. United States </span></span>
+                                </span>
+                                </span>
+                                </span>
+                                </span>
+                                </span>
+                                </span>
+                                </span>
+                                </span>
+                            </p>
+                            <p> The low ranking the U.S. received was due to its extreme levels of wealth and income inequality and the ineffectiveness of its "safety net" -- social programs aimed at reducing poverty. </p>
+                            <p> <a href="http://money.cnn.com/2016/01/05/news/economy/chicago-segregated/index.html?iid=EL" target="_blank"><span>Related: Chicago is America's most segregated city</span></a> </p>
+                            <p> The report concluded that the American safety net was ineffective because it provides only half the financial help people need. Additionally, the levels of assistance in the U.S. are generally lower than in other countries. </p>
+                            
+                            
+                            <p><span> CNNMoney (New York) </span> <span>First published February 1, 2016: 1:28 AM ET</span> </p>
+                        </div></article>

article_scraper/resources/tests/readability/comment-inside-script-parsing/expected.html

@@ -0,0 +1,18 @@
+<article><DIV id="readability-page-1"><div>
+      <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+        tempor incididunt ut labore et dolore magna aliqua.</p>
+      <p>Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat.</p>
+      <p>Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+    </div><div>
+      <p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat.</p>
+      <p>Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur.
+      Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+    </div></DIV></article>

article_scraper/resources/tests/readability/comment-inside-script-parsing/source.html

@@ -0,0 +1,34 @@
+<html>
+  <head><title>Test script parsing</title></head>
+<body>
+  <script>
+    <!--
+    Silly test
+    <script src="foo.js"></script>
+    -->
+  </script>
+  <article>
+    <h1>Lorem</h1>
+    <div>
+      <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+        tempor incididunt ut labore et dolore magna aliqua.</p>
+      <p>Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat.</p>
+      <p>Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+    </div>
+    <h2>Foo</h2>
+    <div>
+      <p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat.</p>
+      <p>Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur.
+      Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+    </div>
+  </article>
+</body>
+</html>

article_scraper/resources/tests/readability/daringfireball-1/expected.html

@@ -0,0 +1,30 @@
+<article><div id="readability-page-1">
+                <h2>About This Site</h2>
+                <p>Daring Fireball is written and produced by John Gruber.</p>
+                <p>
+                    <a href="http://fakehost/graphics/author/addison-bw.jpg" target="_blank"> <img src="http://fakehost/graphics/author/addison-bw-425.jpg" alt="Photograph of the author."/></a>
+                    <br/><em>Portrait by <a href="http://superbiate.com/inquiries/" target="_blank">George Del Barrio</a></em> </p>
+                <h2>Mac Apps</h2>
+                <ul>
+                    <li><a href="http://www.barebones.com/products/bbedit/" target="_blank">BBEdit</a></li>
+                    <li><a href="http://www.flyingmeat.com/acorn/" target="_blank">Acorn</a></li>
+                    <li><a href="http://www.red-sweater.com/marsedit/" target="_blank">MarsEdit</a></li>
+                    <li><a href="http://aged-and-distilled.com/napkin/" target="_blank">Napkin</a></li>
+                    <li><a href="http://www.barebones.com/products/Yojimbo/" target="_blank">Yojimbo</a></li>
+                    <li><a href="http://www.panic.com/transmit/" target="_blank">Transmit</a></li>
+                    <li><a href="http://latenightsw.com/sd4/index.html" target="_blank">Script Debugger</a></li>
+                    <li><a href="http://www.ambrosiasw.com/utilities/snapzprox/" target="_blank">Snapz Pro X</a></li>
+                    <li><a href="http://nightly.webkit.org/" target="_blank">WebKit</a></li>
+                </ul>
+                <h2>iPhone Apps</h2>
+                <ul>
+                    <li><a href="http://vesperapp.co/" target="_blank">Vesper</a></li>
+                </ul>
+                <h2>Server Software</h2>
+                <p>The Daring Fireball website is hosted by <a href="http://joyent.com/" target="_blank">Joyent</a>.</p>
+                <p>Articles and links are published through <a href="http://movabletype.org/" target="_blank">Movable Type</a>. In addition to my own SmartyPants and Markdown plug-ins, Daring Fireball uses several excellent Movable Type plug-ins, including Brad Choate’s <a href="http://bradchoate.com/weblog/2003/06/24/regular-expressions" target="_blank">MT-Regex</a> and <a href="http://bradchoate.com/weblog/2004/10/20/mtifempty" target="_blank">MT-IfEmpty</a>, and <a href="http://bumppo.net/projects/amputator/" target="_blank">Nat Irons’s Amputator</a>.</p>
+                <p>Stats are tracked using <a href="http://haveamint.com/" target="_blank">Mint</a>. Additional web nerdery, including the membership system, is fueled by <a href="http://perl.org/" target="_blank">Perl</a>, <a href="http://www.php.net/" target="_blank">PHP</a>, and <a href="http://www.mysql.com/" target="_blank">MySQL</a>.</p>
+                <h2>Web Standards</h2>
+                <p>Web standards are important, and Daring Fireball adheres to them. Specifically, Daring Fireball’s HTML markup should validate as either <a href="http://www.whatwg.org/specs/web-apps/current-work/" target="_blank">HTML 5</a> or XHTML 4.01 Transitional, its layout is constructed using <a href="http://jigsaw.w3.org/css-validator/validator?uri=http://daringfireball.net/css/fireball_screen.css" target="_blank">valid CSS</a>, and its syndicated feed is <a href="http://feedvalidator.org/check?url=http%3A%2F%2Fdaringfireball.net%2Findex.xml" target="_blank">valid Atom</a>.</p>
+                <p>If Daring Fireball looks goofy in your browser, you’re likely using a shitty browser that doesn’t support web standards. Internet Explorer, I’m looking in your direction. If you complain about this, I will laugh at you, because I do not care. If, however, you are using a modern, standards-compliant browser and have trouble viewing or reading Daring Fireball, please do let me know.</p>
+            </div></article>

article_scraper/resources/tests/readability/daringfireball-1/source.html

@@ -0,0 +1,151 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+
+<head>
+    <meta http-equiv="content-type" content="text/html; charset=utf-8"/>
+    <title>Daring Fireball: Colophon</title>
+    <meta name="viewport" content="width=600, initial-scale=0.5, minimum-scale=0.45"/>
+    <link rel="apple-touch-icon-precomposed" href="/graphics/apple-touch-icon.png"/>
+    <link rel="shortcut icon" href="/graphics/favicon.ico?v=005"/>
+    <link rel="stylesheet" type="text/css" media="screen" href="/css/fireball_screen.css?v1.6"/>
+    <link rel="stylesheet" type="text/css" media="screen" href="/css/ie_sucks.php"/>
+    <link rel="stylesheet" type="text/css" media="print" href="/css/fireball_print.css?v01"/>
+    <link rel="alternate" type="application/atom+xml" href="/feeds/main"/>
+    <script src="/js/js-global/FancyZoom.js" type="text/javascript"></script>
+    <script src="/js/js-global/FancyZoomHTML.js" type="text/javascript"></script>
+    <link rel="shortcut icon" href="/favicon.ico"/> </head>
+
+<body onload="setupZoom()">
+    <div id="Box">
+        <div id="Banner">
+            <a href="/" title="Daring Fireball: Home"><img src="/graphics/logos/" alt="Daring Fireball" height="56"/></a>
+        </div>
+        <div id="Sidebar">
+            <p>By <strong>John&nbsp;Gruber</strong></p>
+            <ul>
+                <!--&#9733;-->
+                <li><a href="/archive/" title="Previous articles.">Archive</a></li>
+                <li>
+                    <script type="text/javascript">
+                        // <![CDATA[
+                        function ReadCookie(name) {
+                            var nameEQ = name + "=";
+                            var ca = document.cookie.split(';');
+                            for (var i = 0; i < ca.length; i++) {
+                                var c = ca[i];
+                                while (c.charAt(0) == ' ') c = c.substring(1, c.length);
+                                if (c.indexOf(nameEQ) == 0) return c.substring(nameEQ.length, c.length);
+                            }
+                            return null;
+                        }
+                        var display_linked_list = ReadCookie('displayLinkedList');
+                        var li_linked = '<a href="/linked/" title="The Linked List.">Linked List<\/a>';
+                        if (display_linked_list == "hide") {
+                            // Linked List is off on home page, so show it in menu:
+                            document.write(li_linked + "<\/li>\n<li>");
+                        } else {
+                            // Default to not putting separate LL item in sidebar:
+                        }
+                        // ]]>
+                    </script>
+                </li>
+                <li><a href="/thetalkshow/" title="The world’s most popular podcast.">The Talk Show</a></li>
+                <li><a href="http://vesperapp.co/" title="A simple, elegant iOS app for collecting notes, ideas, thoughts, and images.">Vesper: <em>Now for iPad</em></a></li>
+                <li><a href="/projects/" title="Software projects, including SmartyPants and Markdown.">Projects</a></li>
+                <li><a href="/contact/" title="How to send email regarding Daring Fireball.">Contact</a></li>
+                <li><a href="/colophon/" title="About this site and the tools used to produce it.">Colophon</a></li>
+                <li><a href="/feeds/">RSS Feed</a></li>
+                <li><a href="https://twitter.com/daringfireball">Twitter</a></li>
+                <li><a href="/feeds/sponsors/">Sponsorship</a></li>
+            </ul>
+        </div>
+        <!-- Sidebar -->
+        <div id="Main">
+            <div class="article">
+                <h1>About This Site</h1>
+                <p>Daring Fireball is written and produced by John Gruber.</p>
+                <p>
+                    <a href="/graphics/author/addison-bw.jpg"> <img src="/graphics/author/addison-bw-425.jpg" alt="Photograph of the author." style="border: 0;"/></a>
+                    <br/><em>Portrait by <a href="http://superbiate.com/inquiries/">George Del Barrio</a></em> </p>
+                <h2>Mac Apps</h2>
+                <ul>
+                    <li><a href="http://www.barebones.com/products/bbedit/">BBEdit</a></li>
+                    <li><a href="http://www.flyingmeat.com/acorn/">Acorn</a></li>
+                    <li><a href="http://www.red-sweater.com/marsedit/">MarsEdit</a></li>
+                    <li><a href="http://aged-and-distilled.com/napkin/">Napkin</a></li>
+                    <li><a href="http://www.barebones.com/products/Yojimbo/">Yojimbo</a></li>
+                    <li><a href="http://www.panic.com/transmit/">Transmit</a></li>
+                    <li><a href="http://latenightsw.com/sd4/index.html">Script Debugger</a></li>
+                    <li><a href="http://www.ambrosiasw.com/utilities/snapzprox/">Snapz Pro X</a></li>
+                    <li><a href="http://nightly.webkit.org/">WebKit</a></li>
+                </ul>
+                <h2>iPhone Apps</h2>
+                <ul>
+                    <li><a href="http://vesperapp.co/">Vesper</a></li>
+                </ul>
+                <h2>Server Software</h2>
+                <p>The Daring Fireball website is hosted by <a href="http://joyent.com/">Joyent</a>.</p>
+                <p>Articles and links are published through <a href="http://movabletype.org/">Movable Type</a>. In addition to my own SmartyPants and Markdown plug-ins, Daring Fireball uses several excellent Movable Type plug-ins, including Brad Choate’s <a href="http://bradchoate.com/weblog/2003/06/24/regular-expressions">MT-Regex</a> and <a href="http://bradchoate.com/weblog/2004/10/20/mtifempty">MT-IfEmpty</a>, and <a href="http://bumppo.net/projects/amputator/">Nat Irons’s Amputator</a>.</p>
+                <p>Stats are tracked using <a href="http://haveamint.com/">Mint</a>. Additional web nerdery, including the membership system, is fueled by <a href="http://perl.org/">Perl</a>, <a href="http://www.php.net/">PHP</a>, and <a href="http://www.mysql.com/">MySQL</a>.</p>
+                <h2>Web Standards</h2>
+                <p>Web standards are important, and Daring Fireball adheres to them. Specifically, Daring Fireball’s HTML markup should validate as either <a href="http://www.whatwg.org/specs/web-apps/current-work/">HTML 5</a> or XHTML 4.01 Transitional, its layout is constructed using <a href="http://jigsaw.w3.org/css-validator/validator?uri=http://daringfireball.net/css/fireball_screen.css">valid CSS</a>, and its syndicated feed is <a href="http://feedvalidator.org/check?url=http%3A%2F%2Fdaringfireball.net%2Findex.xml">valid Atom</a>.</p>
+                <p>If Daring Fireball looks goofy in your browser, you’re likely using a shitty browser that doesn’t support web standards. Internet Explorer, I’m looking in your direction. If you complain about this, I will laugh at you, because I do not care. If, however, you are using a modern, standards-compliant browser and have trouble viewing or reading Daring Fireball, please do let me know.</p>
+            </div>
+            <!-- article -->
+            <div id="Footer">
+                <form id="SiteSearch" action="https://daringfireball.net/search" method="get" style="margin-bottom: 2.5em;">
+                    <div>
+                        <input name="q" type="text" value="" style="margin-right: 8px; width: 66%;"/>
+                        <input type="submit" value="Search"/> </div>
+                </form>
+                <p class="smallprint"> <a href="/preferences/" title="Customize the font size and presentation options for this web site.">Display Preferences</a>
+                    <br/>
+                    <br/> Copyright © 2002–2015 The Daring Fireball Company LLC. </p>
+            </div>
+            <div id="SidebarTheDeck">
+                <script type="text/javascript">
+                    // <![CDATA[
+                    (function(id) {
+                        document.write('<script type="text/javascript" src="' + '//connect.decknetwork.net/deck' + id + '_js.php?' + (new Date().getTime()) + '"></' + 'script>');
+                    })("DF");
+                    var deckDiv = document.getElementById("SidebarTheDeck");
+                    var sidebarDiv = document.getElementById("Sidebar");
+                    sidebarDiv.appendChild(deckDiv);
+                    // ]]>
+                </script>
+                <p id="ViaTheDeck">
+                    <a href="http://decknetwork.net/"> <img src="//daringfireball.net/graphics/madison/via_the_deck.png" alt="Ads via The Deck" class="the_deck_promo" width="70"/> <span>Ads via The Deck</span> </a>
+                </p>
+            </div>
+            <!-- Google Analytics -->
+            <script type="text/javascript">
+                var _gaq = _gaq || [];
+                _gaq.push(['_setAccount', 'UA-593949-1']);
+                _gaq.push(['_trackPageview']);
+                (function() {
+                    var ga = document.createElement('script');
+                    ga.type = 'text/javascript';
+                    ga.async = true;
+                    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+                    var s = document.getElementsByTagName('script')[0];
+                    s.parentNode.insertBefore(ga, s);
+                })();
+            </script>
+            <!-- Asynchronously load Mint -->
+            <script type="text/javascript">
+                (function() {
+                    var ma = document.createElement('script');
+                    ma.type = 'text/javascript';
+                    ma.src = '/mint/?js';
+                    ma.async = true;
+                    var s = document.getElementsByTagName('script')[0];
+                    s.parentNode.insertBefore(ma, s);
+                })();
+            </script>
+        </div>
+        <!-- Main -->
+    </div>
+    <!-- box -->
+</body>
+
+</html>

article_scraper/resources/tests/readability/dev418/expected.html

@@ -0,0 +1,80 @@
+<article><DIV id="readability-page-1">
+        
+        <p>
+            Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+        </p>
+        <hr/>
+        <h2>
+            Single &lt;img&gt;
+        </h2>
+        <p>
+            <img src="http://fakehost/test/base/florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image"/>
+        </p>
+        <p>
+            Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+        </p>
+        <hr/>
+        <h2>
+            Single &lt;figure&gt;
+        </h2>
+        <figure>
+            <img src="http://fakehost/test/base/florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image"/>
+            <figcaption>
+                Caption of the figure
+            </figcaption>
+        </figure>
+        <p>
+            Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+        </p>
+        <hr/>
+        <h2>
+            &lt;ul&gt; List of &lt;img&gt;
+        </h2>
+        <ul>
+            <li>
+                <img src="http://fakehost/test/base/florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image"/>
+            </li>
+            <li>
+                <img src="http://fakehost/test/base/florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image"/>
+            </li>
+            <li>
+                <img src="http://fakehost/test/base/florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image"/>
+            </li>
+        </ul>
+        <p>
+            Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+        </p>
+        <hr/>
+        <h2>
+            &lt;ul&gt; List of &lt;figure&gt;
+        </h2>
+        <ul>
+            <li>
+                <figure>
+                    <img src="http://fakehost/test/base/florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image"/>
+                    <figcaption>
+                        Caption of the figure
+                    </figcaption>
+                </figure>
+            </li>
+            <li>
+                <figure>
+                    <img src="http://fakehost/test/base/florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image"/>
+                    <figcaption>
+                        Caption of the figure
+                    </figcaption>
+                </figure>
+            </li>
+            <li>
+                <figure>
+                    <img src="http://fakehost/test/base/florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image"/>
+                    <figcaption>
+                        Caption of the figure
+                    </figcaption>
+                </figure>
+            </li>
+        </ul>
+        <p>
+            Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+        </p>
+    </DIV></article>

article_scraper/resources/tests/readability/dev418/source.html

@@ -0,0 +1,104 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+    <head>
+        <meta charset="utf-8" />
+        <title>
+            Readability Test
+        </title>
+        <style>
+        <![CDATA[
+        body {
+                font-family: sans-serif;
+                margin: 0 auto;
+                max-width: 900px;
+                padding: 1em 2em;
+        }
+        img {
+                max-width: 100%;
+        }
+        ]]>
+        </style>
+    </head>
+    <body>
+        <h1>
+            Readability Test
+        </h1>
+        <p>
+            Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+        </p>
+        <hr />
+        <h2>
+            Single &lt;img&gt;
+        </h2>
+        <p>
+            <img src="florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image" />
+        </p>
+        <p>
+            Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+        </p>
+        <hr />
+        <h2>
+            Single &lt;figure&gt;
+        </h2>
+        <figure>
+            <img src="florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image" />
+            <figcaption>
+                Caption of the figure
+            </figcaption>
+        </figure>
+        <p>
+            Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+        </p>
+        <hr />
+        <h2>
+            &lt;ul&gt; List of &lt;img&gt;
+        </h2>
+        <ul>
+            <li>
+                <img src="florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image" />
+            </li>
+            <li>
+                <img src="florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image" />
+            </li>
+            <li>
+                <img src="florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image" />
+            </li>
+        </ul>
+        <p>
+            Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+        </p>
+        <hr />
+        <h2>
+            &lt;ul&gt; List of &lt;figure&gt;
+        </h2>
+        <ul>
+            <li>
+                <figure>
+                    <img src="florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image" />
+                    <figcaption>
+                        Caption of the figure
+                    </figcaption>
+                </figure>
+            </li>
+            <li>
+                <figure>
+                    <img src="florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image" />
+                    <figcaption>
+                        Caption of the figure
+                    </figcaption>
+                </figure>
+            </li>
+            <li>
+                <figure>
+                    <img src="florian-giorgio-P1U7-ZgKeOM-unsplash.jpg" alt="An image" />
+                    <figcaption>
+                        Caption of the figure
+                    </figcaption>
+                </figure>
+            </li>
+        </ul>
+        <p>
+            Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+        </p>
+    </body>
+</html>

article_scraper/resources/tests/readability/dropbox-blog/expected.html

@@ -0,0 +1,525 @@
+<article><div id="readability-page-1">
+                            <div>
+                                <p>
+                                    I joined Dropbox not long after graduating with a Master’s degree in computer science. Aside from an internship, this was my first big-league engineering job. My team had already begun designing a critical internal service that most of our software would use: It would handle asynchronous computing requests behind the scenes, powering everything from dragging a file into a Dropbox folder to scheduling a marketing campaign.
+                                </p>
+                                <p>
+                                    This Asynchronous Task Framework (ATF) would replace multiple bespoke async systems used by different engineering teams. It would reduce redundant development, incompatibilities, and reliance on legacy software. There were no open-source projects or buy-not-build solutions that worked well for our use case and scale, so we had to create our own. ATF is both an important and interesting challenge, though, so we were happy to design, build and deploy our own in-house service.
+                                </p>
+                                <p>
+                                    ATF not only had to work well, it had to work well at scale: It would be a foundational building block of Dropbox infrastructure. It would need to handle 10,000 async tasks per second from the start, and be architected for future growth. It would need to support nearly 100 unique async task types from the start, again with room to grow. There were at least two dozen engineering teams that would want to use it for entirely different parts of our codebase, for many products and services. 
+                                </p>
+                                <p>
+                                    As any engineer would, we Googled to see what other companies with mega-scale services had done to handle async tasks. We were disappointed to find little material published by engineers who built supersized async services.
+                                </p>
+                                <p>
+                                    Now that ATF is deployed and currently serving 9,000 async tasks scheduled per second and in use by 28 engineering teams internally, we’re glad to fill that information gap. We’ve documented Dropbox ATF thoroughly, as a reference and guide for the engineering community seeking their own async solutions.
+                                </p>
+                            </div>
+                            <div>
+                                <P id="introduction">
+                                    <h2>
+                                        Introduction
+                                    </h2>
+                                </P>
+                            </div>
+                            <div>
+                                <p>
+                                    Scheduling asynchronous tasks on-demand is a critical capability that powers many features and internal platforms at Dropbox. Async Task Framework (ATF) is the infrastructural system that supports this capability at Dropbox through a callback-based architecture. ATF enables developers to define callbacks, and schedule tasks that execute against these pre-defined callbacks.
+                                </p>
+                                <p>
+                                    Since its introduction over a year ago, ATF has gone on to become an important building block in the Dropbox infrastructure, used by nearly 30 internal teams across our codebase. It currently supports 100+ use cases which require either immediate or delayed task scheduling. 
+                                </p>
+                            </div>
+                            <div>
+                                <P id="glossary">
+                                    <h2>
+                                        Glossary
+                                    </h2>
+                                </P>
+                            </div>
+                            <div>
+                                <p>
+                                    Some basic terms repeatedly used in this post, defined as used in the context of this discussion.
+                                </p>
+                                <p>
+                                    <b>Lambda:</b> A callback implementing business logic.
+                                </p>
+                                <p>
+                                    <span><b>Task:</b> Unit of execution of a lambda. Each asynchronous job scheduled with ATF is a task.</span>
+                                </p>
+                                <p>
+                                    <span><b>Collection:</b> A labeled subset of tasks belonging to a lambda. If <span>send email</span> is implemented as a lambda, then <span>password reset email</span> and <span>marketing email</span> would be collections.</span>
+                                </p>
+                                <p>
+                                    <span><b> Priority:</b> Labels defining priority of execution of tasks within a lambda. </span>
+                                </p>
+                            </div>
+                            <div>
+                                <P id="features">
+                                    <h2>
+                                        Features
+                                    </h2>
+                                </P>
+                            </div>
+                            <div>
+                                <p>
+                                    <b>Task scheduling</b><br/>
+                                    Clients can schedule tasks to execute at a specified time. Tasks can be scheduled for immediate execution, or delayed to fit the use case.
+                                </p>
+                                <p>
+                                    <b>Priority based execution</b><br/>
+                                    Tasks should be associated with a priority. Tasks with higher priority should get executed before tasks with a lower priority once they are ready for execution.
+                                </p>
+                                <p>
+                                    <b>Task gating</b><br/>
+                                    ATF enables the the gating of tasks based on lambda, or a subset of tasks on a lambda based on collection. Tasks can be gated to be completely dropped or paused until a suitable time for execution.
+                                </p>
+                                <p>
+                                    <b>Track task status</b><br/>
+                                    Clients can query the status of a scheduled task.
+                                </p>
+                            </div>
+                            <div>
+                                <P id="system-guarantees">
+                                    <h2>
+                                        System guarantees
+                                    </h2>
+                                </P>
+                            </div>
+                            <div>
+                                <p>
+                                    <b>At-least once task execution<br/></b> The ATF system guarantees that a task is executed at least once after being scheduled. Execution is said to be complete once the user-defined callback signals task completion to the ATF system.
+                                </p>
+                                <p>
+                                    <b>No concurrent task execution<br/></b> The ATF system guarantees that at most one instance of a task will be actively executing at any given in point. This helps users write their callbacks without designing for concurrent execution of the same task from different locations.
+                                </p>
+                                <p>
+                                    <b>Isolation<br/></b> Tasks in a given lambda are isolated from the tasks in other lambdas. This isolation spans across several dimensions, including worker capacity for task execution and resource use for task scheduling. Tasks on the same lambda but different priority levels are also isolated in their resource use for task scheduling.
+                                </p>
+                                <p>
+                                    <b>Delivery latency<br/></b> 95% of tasks begin execution within five seconds from their scheduled execution time.
+                                </p>
+                                <p>
+                                    <b>High availability for task scheduling<br/></b> The ATF service is 99.9% available to accept task scheduling requests from any client.
+                                </p>
+                            </div>
+                            <div>
+                                <P id="-lambda-requirements">
+                                    <h2>
+                                        Lambda requirements
+                                    </h2>
+                                </P>
+                            </div>
+                            <div>
+                                <p>
+                                    Following are some restrictions we place on the callback logic (lambda):
+                                </p>
+                                <p>
+                                    <b>Idempotence</b><br/>
+                                    A single task on a lambda can be executed multiple times within the ATF system. Developers should ensure that their lambda logic and correctness of task execution in clients are not affected by this.
+                                </p>
+                                <p>
+                                    <b>Resiliency</b><br/>
+                                    Worker processes which execute tasks might die at any point during task execution. ATF retries abruptly interrupted tasks, which could also be retried on different hosts. Lambda owners must design their lambdas such that retries on different hosts do not affect lambda correctness.
+                                </p>
+                                <p>
+                                    <b>Terminal state handling<br/></b> ATF retries tasks until they are signaled to be complete from the lambda logic. Client code can mark a task as successfully completed, fatally terminated, or retriable. It is critical that lambda owners design clients to signal task completion appropriately to avoid misbehavior such as infinite retries. 
+                                </p>
+                            </div>
+                            <div>
+                                <P id="architecture">
+                                    <h2>
+                                        Architecture
+                                    </h2>
+                                </P>
+                            </div>
+                            <div>
+                                    <figure>
+                                        <img src="http://fakehost/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x844px-1.png" aria-hidden="false" alt="Async Task Framework (ATF) [Fig 1]" height="1688" width="1440"/>
+                                        <figcaption>
+                                            Async Task Framework (ATF) [Fig 1]
+                                        </figcaption>
+                                    </figure>
+                                </div>
+                            <div>
+                                <p>
+                                    In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.) In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.) Dropbox <a href="https://dropbox.tech/infrastructure/courier-dropbox-migration-to-grpc" target="_blank">uses gRPC</a> for remote calls and our in-house <a href="https://dropbox.tech/infrastructure/reintroducing-edgestore" target="_blank">Edgestore</a> to store tasks.
+                                </p>
+                                <p>
+                                    ATF consists of the following components: 
+                                </p>
+                                <ul>
+                                    <li>Frontend
+                                    </li>
+                                    <li>Task Store
+                                    </li>
+                                    <li>Store Consumer
+                                    </li>
+                                    <li>Queue
+                                    </li>
+                                    <li>Controller
+                                    </li>
+                                    <li>Executor
+                                    </li>
+                                    <li>Heartbeat and Status Controller (HSC)<span><br/></span>
+                                    </li>
+                                </ul>
+                                <p>
+                                    <span><b>Frontend</b><br/>
+                                    This is the service that schedules requests via an RPC interface. The frontend accepts RPC requests from clients and schedules tasks by interacting with ATF’s task store described below.</span><br/>
+                                </p>
+                                <p>
+                                    <b>Task Store<br/></b> ATF tasks are stored in and triggered from the task store. The task store could be any generic data store with indexed querying capability. In ATF’s case, We use our in-house metadata store Edgestore to power the task store. More details can be found in the <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model" target="_blank">D</a><a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model" target="_blank">ata</a> <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model" target="_blank">M</a><a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model" target="_blank">odel</a> section below.
+                                </p>
+                                <p>
+                                    <b>Store Consumer<br/></b> The Store Consumer is a service that periodically polls the task store to find tasks that are ready for execution and pushes them onto the right queues, as described in the queue section below. These could be tasks that are newly ready for execution, or older tasks that are ready for execution again because they either failed in a retriable way on execution, or were dropped elsewhere within the ATF system. 
+                                </p>
+                                <p>
+                                    Below is a simple walkthrough of the Store Consumer’s function: 
+                                </p>
+                            </div>
+                            <div>
+                                
+                                <pre><code>repeat every second:
+  1. poll tasks ready for execution from task store
+  2. push tasks onto the right queues
+  3. update task statuses</code></pre>
+                            </div>
+                            <div>
+                                <p>
+                                    The Store Consumer polls tasks that failed in earlier execution attempts. This helps with the at-least-once guarantee that the ATF system provides. More details on how the Store Consumer polls new and previously failed tasks is presented in the <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=342792671048375002388848&amp;h2=Lifecycle-of-a-task" target="_blank">Lifecycle of a task</a> section below.
+                                </p>
+                                <p>
+                                    <b>Queue<br/></b> ATF uses AWS <a href="https://aws.amazon.com/sqs/" target="_blank">Simple Queue Service</a> (SQS) to queue tasks internally. These queues act as a buffer between the Store Consumer and Controllers (described below). Each <span>&lt;lambda, priority&gt;</span>  pair gets a dedicated SQS queue. The total number of SQS queues used by ATF is <span>#lambdas x #priorities</span>.
+                                </p>
+                                <p>
+                                    <b>Controller<br/></b> Worker hosts are physical hosts dedicated for task execution. Each worker host has one controller process responsible for polling tasks from SQS queues in a background thread, and then pushing them onto process local buffered queues. The Controller is only aware of the lambdas it is serving and thus polls only the limited set of necessary queues. 
+                                </p>
+                                <p>
+                                    The Controller serves tasks from its process local queue as a response to <span>NextWork</span> RPCs. This is the layer where execution level task prioritization occurs. The Controller has different process level queues for tasks of different priorities and can thus prioritize tasks in response to <span>NextWork</span> RPCs.
+                                </p>
+                                <p>
+                                    <b>Executor<br/></b> The Executor is a process with multiple threads, responsible for the actual task execution. Each thread within an Executor process follows this simple loop:
+                                </p>
+                            </div>
+                            <div>
+                                
+                                <pre><code>while True:
+  w = get_next_work()
+  do_work(w)</code></pre>
+                            </div>
+                            <div>
+                                <p>
+                                    Each worker host has a single Controller process and multiple executor processes. Both the Controller and Executors work in a “pull” model, in which active loops continuously long-poll for new work to be done.
+                                </p>
+                                <p>
+                                    <b>Heartbeat and Status Controller (HSC)</b><br/>
+                                    The HSC serves RPCs for claiming a task for execution (<span>ClaimTask</span>), setting task status after execution (<span>SetResults</span>) and heartbeats during task execution (<span>Heartbeat</span>). <span>ClaimTask</span> requests originate from the Controllers in response to <span>NextWork</span> requests. <span>Heartbeat</span> and <span>SetResults</span> requests originate from executor processes during and after task execution. The HSC interacts with the task store to update the task status on the kind of request it receives.
+                                </p>
+                            </div>
+                            <div>
+                                <P id="data-model">
+                                    <h2>
+                                        Data model
+                                    </h2>
+                                </P>
+                            </div>
+                            <div>
+                                <p>
+                                    ATF uses our in-house metadata store, Edgestore, as a task store. Edgestore objects can be Entities or Associations (<span>assoc</span>), each of which can have user-defined attributes. Associations are used to represent relationships between entities. Edgestore supports indexing only on attributes of associations.
+                                </p>
+                                <p>
+                                    Based on this design, we have two kinds of ATF-related objects in Edgestore. The ATF association stores scheduling information, such as the next scheduled timestamp at which the Store Consumer should poll a given task (either for the first time or for a retry). The ATF entity stores all task related information that is used to track the task state and payload for task execution. We query on associations from the Store Consumer in a pull model to pick up tasks ready for execution.
+                                </p>
+                            </div>
+                            <div>
+                                <P id="lifecycle-of-a-task">
+                                    <h2>
+                                        Lifecycle of a task
+                                    </h2>
+                                </P>
+                            </div>
+                            <div>
+                                <ol>
+                                    <li>Client performs a <span>Schedule</span> RPC call to <b>Frontend</b> with task information, including execution time. 
+                                    </li>
+                                    <li>Frontend creates Edgestore <span>entity</span> and <span>assoc</span> for the task. 
+                                    </li>
+                                    <li>When it is time to process the task, <b>Store Consumer</b> pulls the task from <b>Edgestore</b> and pushes it to a related <b>SQS</b> queue. 
+                                    </li>
+                                    <li>
+                                        <b>Executor</b> makes <span>NextWork</span> RPC call to <b>Controller</b>, which pulls tasks from the <b>SQS</b> queue, makes a <span>ClaimTask</span> RPC to the HSC and then returns the task to the <b>Executor</b>. 
+                                    </li>
+                                    <li>
+                                        <b>Executor</b> invokes the callback for the task. While processing, <b>Executor</b> performs <span>Heartbeat</span> RPC calls to <b>Heartbeat and Status Controller (HSC)</b>. Once processing is done, <b>Executor</b> performs <span>TaskStatus</span> RPC call to <b>HSC</b>. 
+                                    </li>
+                                    <li>Upon getting <span>Heartbeat</span> and <span>TaskStatus</span> RPC calls, <b>HSC</b> updates the <b>Edgestore</b> entity and <span>assoc</span>.
+                                    </li>
+                                </ol>
+                                <p>
+                                    Every state update in the lifecycle of a task is accompanied by an update to the next trigger timestamp in the <span>assoc</span>. This ensures that the Store Consumer pulls the task again if there is no change in state of the task within the next trigger timestamp. This helps ATF achieve its at-least-once delivery guarantee by ensuring that no task is dropped.
+                                </p>
+                                <p>
+                                    Following are the task entity and association states in ATF and their corresponding timestamp updates:
+                                </p>
+                                <table>
+                                    <tbody>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <b>Entity status</b>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <b>Assoc status</b>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <b>next trigger timestamp in Assoc</b>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <b>Comment</b>
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span>new</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span>new</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span>scheduled_timestamp</span> of the task
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    Pick up new tasks that are ready. 
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span>enqueued</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span>started</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span>enqueued_timestamp</span> + <span>enqueue_timeout</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    Re-enqueue task if it has been in <span>enqueued</span> state for too long. This can happen if the queue loses data or the controller goes down after polling the queue and before the task is claimed.
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span>claimed</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span>started</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span>claimed_timestamp</span> + <span>claim_timeout</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    Re-enqueue if task is claimed but never transfered to <span>processing</span>. This can happen if Controller is down after claiming a task. Task status is changed to <span>enqueued</span> after re-enqueue.
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span>processing</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span>started</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span>heartbeat_timestamp</span> + <span>heartbeat_timeout</span>`
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    Re-enqueue if task hasn’t sent <span>heartbeat</span> for too long. This can happen if Executor is down. Task status is changed to <span>enqueued</span> after re-enqueue. 
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span>retriable failure</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    started
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    compute <span>next_timestamp</span> according to backoff logic
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    Exponential backoff for tasks with retriable failure. 
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span>success</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span>completed</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    N/A
+                                                </p>
+                                            </td>
+                                            
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span>fatal_failure</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span>completed</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    N/A
+                                                </p>
+                                            </td>
+                                            
+                                        </tr>
+                                    </tbody>
+                                </table>
+                                <p>
+                                    The store consumer polls for tasks based on the following query:
+                                </p>
+                                <p>
+                                    <span>assoc_status= &amp;&amp; next_timestamp&lt;=time.now()<br/></span>
+                                </p>
+                                <p>
+                                    Below is the state machine that defines task state transitions: <br/>
+                                </p>
+                            </div>
+                            <div>
+                                    <figure>
+                                        <img src="http://fakehost/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x225px-2.png" aria-hidden="false" alt="Task State Transitions [Fig 2]" height="450" width="1440"/>
+                                    </figure>
+                                </div>
+                            <div>
+                                <P id="-achieving-guarantees">
+                                    <h2>
+                                        Achieving guarantees
+                                    </h2>
+                                </P>
+                            </div>
+                            <div>
+                                <p>
+                                    <b>At-least-once task execution<br/></b> At-least-once execution is guaranteed in ATF by retrying a task until it completes execution (which is signaled by a <span>Success</span> or a <span>FatalFailure</span> state). All ATF system errors are implicitly considered retriable failures, and lambda owners have an option of marking tasks with a <span>RetriableFailure</span> state. Tasks might be dropped from the ATF execution pipeline in different parts of the system through transient RPC failures and failures on dependencies like Edgestore or SQS. These transient failures at different parts of the system do not affect the at-least-once guarantee, though, because of the system of timeouts and re-polling from Store Consumer.
+                                </p>
+                                <p>
+                                    <b>No concurrent task execution<br/></b> Concurrent task execution is avoided through a combination of two methods in ATF. First, tasks are explicitly claimed through an exclusive task state (<span>Claimed</span>) before starting execution. Once the task execution is complete, the task status is updated to one of <span>Success</span>, <span>FatalFailure</span> or <span>RetriableFailure</span>. A task can be claimed only if its existing task state is <span>Enqueued</span> (retried tasks go to the <span>Enqueued</span> state as well once they are re-pushed onto SQS).
+                                </p>
+                                <p>
+                                    However, there might be situations where once a long running task starts execution, its heartbeats might fail repeatedly yet the task execution continues. ATF would retry this task by polling it from the store consumer because the heartbeat timeouts would’ve expired. This task can then be claimed by another worker and lead to concurrent execution. <br/>
+                                </p>
+                                <p>
+                                    To avoid this situation, there is a termination logic in the Executor processes whereby an Executor process terminates itself as soon as three consecutive heartbeat calls fail. Each heartbeat timeout is large enough to eclipse three consecutive heartbeat failures. This ensures that the Store Consumer cannot pull such tasks before the termination logic ends them—the second method that helps achieve this guarantee.
+                                </p>
+                                <p>
+                                    <b>Isolation<br/></b> Isolation of lambdas is achieved through dedicated worker clusters, dedicated queues, and dedicated per-lambda scheduling quotas. In addition, isolation across different priorities within the same lambda is likewise achieved through dedicated queues and scheduling bandwidth.
+                                </p>
+                                <p>
+                                    <b>Delivery latency<br/></b> ATF use cases do not require ultra-low task delivery latencies. Task delivery latencies on the order of a couple of seconds are acceptable. Tasks ready for execution are periodically polled by the Store Consumer and this period of polling largely controls the task delivery latency. Using this as a tuning lever, ATF can achieve different delivery latencies as required. Increasing poll frequency reduces task delivery latency and vice versa. Currently, we have calibrated ATF to poll for ready tasks once every two seconds.
+                                </p>
+                            </div>
+                            <div>
+                                <P id="ownership-model">
+                                    <h2>
+                                        Ownership model
+                                    </h2>
+                                </P>
+                            </div>
+                            <p>
+                                    ATF is designed to be a self-serve framework for developers at Dropbox. The design is very intentional in driving an ownership model where lambda owners own all aspects of their lambdas’ operations. To promote this, all lambda worker clusters are owned by the lambda owners. They have full control over operations on these clusters, including code deployments and capacity management. Each executor process is bound to one lambda. Owners have the option of deploying multiple lambdas on their worker clusters simply by spawning new executor processes on their hosts.
+                                </p>
+                            <div>
+                                <P id="-extending-atf">
+                                    <h2>
+                                        Extending ATF
+                                    </h2>
+                                </P>
+                            </div>
+                            <div>
+                                <p>
+                                    As described above, ATF provides an infrastructural building block for scheduling asynchronous tasks. With this foundation established, ATF can be extended to support more generic use cases and provide more features as a framework. Following are some examples of what could be built as an extension to ATF. 
+                                </p>
+                                <p>
+                                    <b>Periodic task execution<br/></b> Currently, ATF is a system for one-time task scheduling. Building support for periodic task execution as an extension to this framework would be useful in unlocking new capabilities for our clients.
+                                </p>
+                                <p>
+                                    <b>Better support for task chaining<br/></b> Currently, it is possible to chain tasks on ATF by scheduling a task onto ATF that then schedules other tasks onto ATF during its execution. Although it is possible to do this in the current ATF setup, visibility and control on this chaining is absent at the framework level. Another natural extension here would be to better support task chaining through framework-level visibility and control, to make this use case a first class concept in the ATF model.
+                                </p>
+                                <p>
+                                    <b>Dead letter queues for misbehaving tasks<br/></b> One common source of maintenance overhead we observe on ATF is that some tasks get stuck in infinite retry loops due to occasional bugs in lambda logic. This requires manual intervention from the ATF framework owners in some cases where there are a large number of tasks stuck in such loops, occupying a lot of the scheduling bandwidth in the system. Typical manual actions in response to such a situation include pausing execution of the lambdas with misbehaving tasks, or dropping them outright.
+                                </p>
+                                <p>
+                                    One way to reduce this operational overhead and provide an easy interface for lambda owners to recover from such incidents would be to create dead letter queues filled with such misbehaving tasks. The ATF framework could impose a maximum number of retries before tasks are pushed onto the dead letter queue. We could create and expose tools that make it easy to reschedule tasks from the dead letter queue back into the ATF system, once the associated lambda bugs are fixed.<br/>
+                                </p>
+                            </div>
+                            <div>
+                                <P id="conclusion">
+                                    <h2>
+                                        Conclusion
+                                    </h2>
+                                </P>
+                            </div>
+                            <p>
+                                    We hope this post helps engineers elsewhere to develop better async task frameworks of their own. Many thanks to everyone who worked on this project: Anirudh Jayakumar, Deepak Gupta, Dmitry Kopytkov, Koundinya Muppalla, Peng Kang, Rajiv Desai, Ryan Armstrong, Steve Rodrigues, Thomissa Comellas, Xiaonan Zhang and Yuhuan Du.<br/>
+                                     
+                                </p>
+                        </div></article>

article_scraper/resources/tests/readability/dropbox-blog/source.html

@@ -0,0 +1,868 @@
+<!DOCTYPE html>
+<html lang="en" xml:lang="en" data-cms-lang="en-us" xmlns="http://www.w3.org/1999/xhtml">
+    <head>
+        <meta charset="utf-8" />
+        <title>
+            How we designed Dropbox ATF: an async task framework - Dropbox
+        </title>
+        <meta name="data-tags" content="Async,Edgestore,Infrastructure,Task Scheduling" />
+        <meta name="data-tagTaxonomy" content="Async; Edgestore; Infrastructure; Task Scheduling;" />
+        <meta name="page-id" content="infrastructure-asynchronous-task-scheduling-at-dropbox" />
+        <meta name="topic" content="Infrastructure" />
+        <meta name="publishDate" content="2020-11-11 12:00:00.000-0600" />
+        <meta name="author" content="Arun Sai Krishnan" />
+        <link rel="canonical" href="https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" />
+        <link rel="icon" href="https://cfl.dropboxstatic.com/static/images/favicon-vflUeLeeY.ico" type="image/x-icon" />
+        <meta property="og:url" content="https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" />
+        <meta property="og:type" content="article" />
+        <meta property="og:title" content="How we designed Dropbox’s ATF - an async task framework" />
+        <meta property="og:image" content="https://aem.dropbox.com/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-Social.png" />
+        <meta name="twitter:card" content="summary_large_image" />
+        <meta content="width=device-width,initial-scale=1.0,user-scalable=no" name="viewport" />
+        <link rel="alternate" hreflang="en-us" href="https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" /><!-- /* Enable rebrand styles  */
+<sly data-sly-use.inheritUtil="com.dropbox.aem.common.models.utils.InheritanceUtilUse"
+     data-sly-test.pageStyle="" /> -->
+        <link rel="stylesheet" href="/cms/etc.clientlibs/settings/wcm/designs/dropbox-common/clientlib-cms-common.757d73acbd22d3e2bf4eeb953c16c4d5.css" type="text/css" />
+        <link rel="stylesheet" href="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-all.fc2ae6db413129b3901dd5a89e64f347.css" type="text/css" /><!--Knotch Integration should be added in header-->
+
+        <script src="https://www.knotch-cdn.com/unit/latest/knotch.min.js" data-account="33c0d4ac-b5bc-4168-a95b-e963ec65974d" async="async"></script>
+        <link rel="stylesheet" href="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content.22c503f9a8a000fceab6a403af5ce96f.css" type="text/css" />
+        <style>
+        <![CDATA[
+        body.stormcrow-animate{opacity:1;}
+        ]]>
+        </style>
+    </head>
+    <body class="tech-blog-article-page__page stormcrow-animate" data-article-uuid="d4052e45-cbcb-4ebb-b834-eb377ab543e8">
+        <input type="hidden" id="wcmRunmode" name="wcmRunmode" value="publish,prod" /> 
+        <script type="text/javascript">
+        //<![CDATA[
+
+            var attr = "tealium_event$cms".split(",");
+
+            var utag_data = {}
+            attr.forEach(function (item) {
+                if (item && item.indexOf("$") > -1)
+                    utag_data[item.split("$")[0]] = item.split("$")[1];
+            })
+        //]]>
+        </script> 
+        <script type="text/javascript">
+        //<![CDATA[
+
+            (function (a, b, c, d) {
+                a = "\/\/tags.tiqcdn.com\/utag\/dropbox\/tech\u002Dblog\/prod\/utag.js";
+                b = document;
+                c = 'script';
+                d = b.createElement(c);
+                d.src = a;
+                d.type = 'text/java' + c;
+                d.async = true;
+                a = b.getElementsByTagName(c)[0];
+                a.parentNode.insertBefore(d, a);
+            })();
+        //]]>
+        </script>
+        <header class="dr-header">
+            <div class="dr-header__sticky-container">
+                <section class="dr-header__section dr-flex dr-flex--align-center dr-padding-right-40 dr-padding-left-40 dr-header__sticky-content-container dr-header__sticky-content-container--opened dr-container--surface">
+                    <div class="dr-flex-1">
+                        <a class="dr-link dr-link--no-underline dr-link--no-underline-hover dr-typography-t1" href="https://dropbox.tech/">Dropbox.Tech</a>
+                    </div><button class="dr-header__item dr-nav__menu-toggle-button dr-button dr-hide-from-md dr-flex dr-flex-align-center dr-flex-justify-center" data-dr-tooltip="Menu" data-dr-tooltip-theme="white"><svg viewbox="0 0 28 28" fill="none" xmlns="http://www.w3.org/2000/svg" class="dr-width-100 dr-height-100">
+                    <rect x="0.501831" y="8" width="28" height="2" fill="white"></rect>
+                    <rect x="0.500977" y="18" width="28" height="2" fill="white"></rect></svg></button>
+                    <nav class="dr-show-block-from-md dr-nav__nav">
+                        <button class="dr-button dr-nav__menu-close-button"><svg width="20" height="20" viewbox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+                        <path fill-rule="evenodd" clip-rule="evenodd" d="M19.2875 2.15983L17.6683 0.566406L9.82597 8.28403L2.33211 0.909344L0.71294 2.50277L8.2068 9.87746L0.666992 17.2974L2.28617 18.8908L9.82597 11.4709L17.7143 19.2337L19.3334 17.6403L11.4451 9.87746L19.2875 2.15983Z" fill="white"></path></svg></button>
+                        <ul class="dr-unstyled-list dr-typography-t2 dr-flex dr-nav__nav-list">
+                            <li class="dr-header__item dr-position-relative dr-header__item--with-subnav dr-header__list-item">
+                                <button class="dr-button dr-button--link dr-header__link--with-subnav">Topics</button>
+                                <ul class="dr-unstyled-list dr-display-none dr-header__subnav dr-position-absolute dr-container--surface dr-padding-top-30 dr-padding-left-40 dr-padding-bottom-20 dr-padding-right-40 dr-font-weight-500">
+                                    <li class="dr-header__list-item dr-header__list-item--subnav">
+                                        <a href="https://dropbox.tech/application" class="dr-display-block dr-link dr-link--no-underline dr-container--application dr-link--primary">Application</a>
+                                    </li>
+                                    <li class="dr-header__list-item dr-header__list-item--subnav">
+                                        <a href="https://dropbox.tech/frontend" class="dr-display-block dr-link dr-link--no-underline dr-container--frontend dr-link--primary">Front End</a>
+                                    </li>
+                                    <li class="dr-header__list-item dr-header__list-item--subnav">
+                                        <a href="https://dropbox.tech/infrastructure" class="dr-display-block dr-link dr-link--no-underline dr-container--infrastructure dr-link--primary">Infrastructure</a>
+                                    </li>
+                                    <li class="dr-header__list-item dr-header__list-item--subnav">
+                                        <a href="https://dropbox.tech/machine-learning" class="dr-display-block dr-link dr-link--no-underline dr-container--machine-learning dr-link--primary">Machine Learning</a>
+                                    </li>
+                                    <li class="dr-header__list-item dr-header__list-item--subnav">
+                                        <a href="https://dropbox.tech/mobile" class="dr-display-block dr-link dr-link--no-underline dr-container--mobile dr-link--primary">Mobile</a>
+                                    </li>
+                                    <li class="dr-header__list-item dr-header__list-item--subnav">
+                                        <a href="https://dropbox.tech/security" class="dr-display-block dr-link dr-link--no-underline dr-container--security dr-link--primary">Security</a>
+                                    </li>
+                                </ul>
+                            </li>
+                            <li class="dr-header__item dr-header__list-item">
+                                <a href="https://dropbox.tech/developers" class="dr-link dr-link--no-underline dr-nav__main-category">Developers</a>
+                            </li>
+                            <li class="dr-header__item dr-header__list-item">
+                                <a class="dr-link dr-link--no-underline dr-header__link" href="http://dropbox.com/jobs" target="_blank">Jobs</a>
+                            </li>
+                        </ul>
+                    </nav><button data-dark-mode-switcher="" class="dr-header__item dr-header__dark-mode-switcher dr-button dr-button--link dr-cursor-pointer" data-dr-tooltip="Dark Mode" data-dr-tooltip-theme="white" type="button"><img alt="" height="30" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-all/resources/button_dark-mode-new.svg" width="30" class="dr-header__mode-image" /></button> <button class="dr-header__item dr-header__search-button dr-button dr-button--link dr-cursor-pointer" data-dr-tooltip="Search" data-dr-tooltip-theme="white" type="button"><img alt="" height="20" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-all/resources/button_search-new.svg" width="20" /></button> <!--search-result-page-only-->
+                     <!--search-result-page-only-->
+                    <div class="dr-header__search dr-display-none">
+                        <button class="dr-header__search-close-button dr-header__item dr-button dr-button--link dr-cursor-pointer" type="button"><svg width="20" height="20" viewbox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+                        <path fill-rule="evenodd" clip-rule="evenodd" d="M19.2875 2.15983L17.6683 0.566406L9.82597 8.28403L2.33211 0.909344L0.71294 2.50277L8.2068 9.87746L0.666992 17.2974L2.28617 18.8908L9.82597 11.4709L17.7143 19.2337L19.3334 17.6403L11.4451 9.87746L19.2875 2.15983Z" fill="white"></path></svg></button>
+                        <div class="dr-header__search-form-container">
+                            <form action="https://dropbox.tech/search-results.html" class="dr-header__search-form dr-container__content dr-width-100">
+                                <input autocomplete="off" class="dr-header__search-input dr-typography-t3" name="q" placeholder="Search" required="true" type="text" />
+                                <p class="dr-header__search-hint dr-margin-top-30 dr-margin-bottom-0 dr-typography-t5 dr-display-none">
+                                    // Press enter to search
+                                </p>
+                            </form>
+                        </div>
+                    </div>
+                </section>
+            </div>
+        </header>
+        <div class="dr-article-hero">
+            <div class="dr-article-hero__background-container dr-container--infrastructure">
+                <picture class="dr-article-hero__background dr-article-hero__background--regular"><source media="( max-width: 375px )" srcset="/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-375x150-light.png" /> <source media="( max-width: 376px ) and ( max-width: 1199px )" srcset="/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-1024x250-light.png" /> <img class="dr-article-hero__background-image" src="/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-1440x305-light.png" alt="" /></picture> <picture class="dr-article-hero__background dr-article-hero__background--dark"><source media="( max-width: 375px )" srcset="/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-375x150-dark.png" /> <source media="( max-width: 376px ) and ( max-width: 1199px )" srcset="/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-1024x250-dark.png" /> <img class="dr-article-hero__background-image" src="/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-1440x305-dark.png" alt="" /></picture>
+            </div>
+            <section class="dr-container__content">
+                <h1 class="dr-display-inline dr-typography-t15 dr-container--surface dr-article-hero__title">
+                    <span class="dr-article-hero__title-container dr-container--infrastructure">How we designed Dropbox ATF: an async task framework</span>
+                </h1>
+                <div class="dr-typography-no-space dr-margin-top-10 dr-margin-md-top-20">
+                    <span class="dr-typography-t5">// By Arun Sai Krishnan • Nov 11, 2020</span>
+                </div>
+            </section>
+        </div>
+        <div class="dr-article-content">
+            <div class="dr-article-content__scroll-tracker-container dr-container--infrastructure">
+                <div class="dr-article-content__scroll-tracker"></div>
+            </div>
+            <div class="dr-container__content">
+                <div class="dr-article-content__content-container dr-padding-md-left-80 dr-padding-md-right-80 dr-typography-t12">
+                    <nav class="dr-article-content__side-nav dr-article-content__side-nav--initial dr-typography-t5">
+                        <ol class="dr-article-content__side-nav-list dr-margin-0">
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#introduction" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Introduction</a>
+                            </li>
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#glossary" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Glossary</a>
+                            </li>
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#features" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Features</a>
+                            </li>
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#system-guarantees" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">System guarantees</a>
+                            </li>
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#-lambda-requirements" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Lambda requirements</a>
+                            </li>
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#architecture" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Architecture</a>
+                            </li>
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#data-model" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Data model</a>
+                            </li>
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#lifecycle-of-a-task" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Lifecycle of a task</a>
+                            </li>
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#-achieving-guarantees" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Achieving guarantees</a>
+                            </li>
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#ownership-model" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Ownership model</a>
+                            </li>
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#-extending-atf" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Extending ATF</a>
+                            </li>
+                            <li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
+                                <a href="#conclusion" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Conclusion</a>
+                            </li>
+                        </ol>
+                    </nav>
+                    <div class="dr-article-content__content">
+                        <div class="aem-Grid aem-Grid--12 aem-Grid--default--12">
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    I joined Dropbox not long after graduating with a Master’s degree in computer science. Aside from an internship, this was my first big-league engineering job. My team had already begun designing a critical internal service that most of our software would use: It would handle asynchronous computing requests behind the scenes, powering everything from dragging a file into a Dropbox folder to scheduling a marketing campaign.
+                                </p>
+                                <p>
+                                    This Asynchronous Task Framework (ATF) would replace multiple bespoke async systems used by different engineering teams. It would reduce redundant development, incompatibilities, and reliance on legacy software. There were no open-source projects or buy-not-build solutions that worked well for our use case and scale, so we had to create our own. ATF is both an important and interesting challenge, though, so we were happy to design, build and deploy our own in-house service.
+                                </p>
+                                <p>
+                                    ATF not only had to work well, it had to work well at scale: It would be a foundational building block of Dropbox infrastructure. It would need to handle 10,000 async tasks per second from the start, and be architected for future growth. It would need to support nearly 100 unique async task types from the start, again with room to grow. There were at least two dozen engineering teams that would want to use it for entirely different parts of our codebase, for many products and services.&#160;
+                                </p>
+                                <p>
+                                    As any engineer would, we Googled to see what other companies with mega-scale services had done to handle async tasks. We were disappointed to find little material published by engineers who built supersized async services.
+                                </p>
+                                <p>
+                                    Now that ATF is deployed and currently serving 9,000 async tasks scheduled per second and in use by 28 engineering teams internally, we’re glad to fill that information gap. We’ve documented Dropbox ATF thoroughly, as a reference and guide for the engineering community seeking their own async solutions.
+                                </p>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="introduction">
+                                    <h2 class="dr-article-content__section-title">
+                                        Introduction
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    Scheduling asynchronous tasks on-demand is a critical capability that powers many features and internal platforms at Dropbox. Async Task Framework (ATF) is the infrastructural system that supports this capability at Dropbox through a callback-based architecture. ATF enables developers to define callbacks, and schedule tasks that execute against these pre-defined callbacks.
+                                </p>
+                                <p>
+                                    Since its introduction over a year ago, ATF has gone on to become an important building block in the Dropbox infrastructure, used by nearly 30 internal teams across our codebase. It currently supports 100+ use cases which require either immediate or delayed task scheduling.&#160;
+                                </p>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="glossary">
+                                    <h2 class="dr-article-content__section-title">
+                                        Glossary
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    Some basic terms repeatedly used in this post, defined as used in the context of this discussion.
+                                </p>
+                                <p>
+                                    <b>Lambda:</b> A callback implementing business logic.
+                                </p>
+                                <p>
+                                    <span><b>Task:</b> Unit of execution of a lambda. Each asynchronous job scheduled with ATF is a task.</span>
+                                </p>
+                                <p>
+                                    <span><b>Collection:</b> A labeled subset of tasks belonging to a lambda. If <span class="dr-code">send email</span> is implemented as a lambda, then <span class="dr-code">password reset email</span> and <span class="dr-code">marketing email</span> would be collections.</span>
+                                </p>
+                                <p>
+                                    <span><b>&#160;Priority:</b> Labels defining priority of execution of tasks within a lambda.&#160;</span>
+                                </p>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="features">
+                                    <h2 class="dr-article-content__section-title">
+                                        Features
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    <b>Task scheduling</b><br />
+                                    Clients can schedule tasks to execute at a specified time. Tasks can be scheduled for immediate execution, or delayed to fit the use case.
+                                </p>
+                                <p>
+                                    <b>Priority based execution</b><br />
+                                    Tasks should be associated with a priority. Tasks with higher priority should get executed before tasks with a lower priority once they are ready for execution.
+                                </p>
+                                <p>
+                                    <b>Task gating</b><br />
+                                    ATF enables the the gating of tasks based on lambda, or a subset of tasks on a lambda based on collection. Tasks can be gated to be completely dropped or paused until a suitable time for execution.
+                                </p>
+                                <p>
+                                    <b>Track task status</b><br />
+                                    Clients can query the status of a scheduled task.
+                                </p>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="system-guarantees">
+                                    <h2 class="dr-article-content__section-title">
+                                        System guarantees
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    <b>At-least once task execution<br /></b> The ATF system guarantees that a task is executed at least once after being scheduled. Execution is said to be complete once the user-defined callback signals task completion to the ATF system.
+                                </p>
+                                <p>
+                                    <b>No concurrent task execution<br /></b> The ATF system guarantees that at most one instance of a task will be actively executing at any given in point. This helps users write their callbacks without designing for concurrent execution of the same task from different locations.
+                                </p>
+                                <p>
+                                    <b>Isolation<br /></b> Tasks in a given lambda are isolated from the tasks in other lambdas. This isolation spans across several dimensions, including worker capacity for task execution and resource use for task scheduling. Tasks on the same lambda but different priority levels are also isolated in their resource use for task scheduling.
+                                </p>
+                                <p>
+                                    <b>Delivery latency<br /></b> 95% of tasks begin execution within five seconds from their scheduled execution time.
+                                </p>
+                                <p>
+                                    <b>High availability for task scheduling<br /></b> The ATF service is 99.9% available to accept task scheduling requests from any client.
+                                </p>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="-lambda-requirements">
+                                    <h2 class="dr-article-content__section-title">
+                                        Lambda requirements
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    Following are some restrictions we place on the callback logic (lambda):
+                                </p>
+                                <p>
+                                    <b>Idempotence</b><br />
+                                    A single task on a lambda can be executed multiple times within the ATF system. Developers should ensure that their lambda logic and correctness of task execution in clients are not affected by this.
+                                </p>
+                                <p>
+                                    <b>Resiliency</b><br />
+                                    Worker processes which execute tasks might die at any point during task execution. ATF retries abruptly interrupted tasks, which could also be retried on different hosts. Lambda owners must design their lambdas such that retries on different hosts do not affect lambda correctness.
+                                </p>
+                                <p>
+                                    <b>Terminal state handling<br /></b> ATF retries tasks until they are signaled to be complete from the lambda logic. Client code can mark a task as successfully completed, fatally terminated, or retriable. It is critical that lambda owners design clients to signal task completion appropriately to avoid misbehavior such as infinite retries.&#160;
+                                </p>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="architecture">
+                                    <h2 class="dr-article-content__section-title">
+                                        Architecture
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="image c04-image aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-image image cq-dd-image">
+                                    <figure class="dr-margin-0 dr-display-inline-block">
+                                        <img src="/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x844px-1.png" aria-hidden="false" alt="Async Task Framework (ATF) [Fig 1]" height="1688" width="1440" />
+                                        <figcaption class="dr-typography-t5 dr-color-ink-60">
+                                            Async Task Framework (ATF) [Fig 1]
+                                        </figcaption>
+                                    </figure>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.)&#160;In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.) Dropbox <a href="https://dropbox.tech/infrastructure/courier-dropbox-migration-to-grpc">uses gRPC</a> for remote calls and our in-house <a href="https://dropbox.tech/infrastructure/reintroducing-edgestore">Edgestore</a> to store tasks.
+                                </p>
+                                <p>
+                                    ATF consists of the following components:&#160;
+                                </p>
+                                <ul>
+                                    <li>Frontend
+                                    </li>
+                                    <li>Task Store
+                                    </li>
+                                    <li>Store Consumer
+                                    </li>
+                                    <li>Queue
+                                    </li>
+                                    <li>Controller
+                                    </li>
+                                    <li>Executor
+                                    </li>
+                                    <li>Heartbeat and Status Controller (HSC)<span><br /></span>
+                                    </li>
+                                </ul>
+                                <p>
+                                    <span><b>Frontend</b><br />
+                                    This is the service that schedules requests via an RPC interface. The frontend accepts RPC requests from clients and schedules tasks by interacting with ATF’s task store described below.</span><br />
+                                </p>
+                                <p>
+                                    <b>Task Store<br /></b> ATF tasks are stored in and triggered from the task store. The task store could be any generic data store with indexed querying capability. In ATF’s case, We use our in-house metadata store Edgestore to power the task store. More details can be&#160;found in the <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">D</a><a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">ata</a> <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">M</a><a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">odel</a> section below.
+                                </p>
+                                <p>
+                                    <b>Store Consumer<br /></b> The Store Consumer is a service that periodically polls the task store to find tasks that are ready for execution and pushes them onto the right queues, as described in the queue section below. These could be tasks that are newly ready for execution, or older tasks that are ready for execution again because they either failed in a retriable way on execution, or were dropped elsewhere within the ATF system.&#160;
+                                </p>
+                                <p>
+                                    Below is a simple walkthrough of the Store Consumer’s function:&#160;
+                                </p>
+                            </div>
+                            <div class="dr-code-container aem-GridColumn aem-GridColumn--default--12">
+                                <button class="dr-code-container__copy-button dr-button dr-typography-t17">Copy</button>
+                                <pre class="dr-code-container__pre"><code class="dr-code-container__code dr-typography-t5">repeat every second:
+  1. poll tasks ready for execution from task store
+  2. push tasks onto the right queues
+  3. update task statuses</code></pre>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    The Store Consumer polls tasks that failed in earlier execution attempts. This helps with the at-least-once guarantee that the ATF system provides. More details on how the Store Consumer polls new and previously failed tasks is presented in the <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=342792671048375002388848&amp;h2=Lifecycle-of-a-task">Lifecycle of a task</a> section below.
+                                </p>
+                                <p>
+                                    <b>Queue<br /></b> ATF uses AWS <a href="https://aws.amazon.com/sqs/" style="background-color: rgb(255,255,255);">Simple Queue Service</a> (SQS) to queue tasks internally. These queues act as a buffer between the Store Consumer and Controllers (described below). Each <span class="dr-code">&lt;lambda, priority&gt;</span> &#160;pair gets a dedicated SQS queue. The total number of SQS queues used by ATF is <span class="dr-code">#lambdas x #priorities</span>.
+                                </p>
+                                <p>
+                                    <b>Controller<br /></b> Worker hosts are physical hosts dedicated for task execution. Each worker host has one controller process responsible for polling tasks from SQS queues in a background thread, and then pushing them onto process local buffered queues. The Controller is only aware of the lambdas it is serving and thus polls only the limited set of necessary queues.&#160;
+                                </p>
+                                <p>
+                                    The Controller serves tasks from its process local queue as a response to <span class="dr-code">NextWork</span> RPCs. This is the layer where execution level task prioritization occurs. The Controller has different process level queues for tasks of different priorities and can thus prioritize tasks in response to <span class="dr-code">NextWork</span> RPCs.
+                                </p>
+                                <p>
+                                    <b>Executor<br /></b> The Executor is a process with multiple threads, responsible for the actual task execution. Each thread within an Executor process follows this simple loop:
+                                </p>
+                            </div>
+                            <div class="dr-code-container aem-GridColumn aem-GridColumn--default--12">
+                                <button class="dr-code-container__copy-button dr-button dr-typography-t17">Copy</button>
+                                <pre class="dr-code-container__pre"><code class="dr-code-container__code dr-typography-t5">while True:
+  w = get_next_work()
+  do_work(w)</code></pre>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    Each worker host has a single Controller process and multiple executor processes. Both the Controller and Executors work in a “pull” model, in which active loops continuously long-poll for new work to be done.
+                                </p>
+                                <p>
+                                    <b>Heartbeat and Status Controller (HSC)</b><br />
+                                    The HSC serves RPCs for claiming a task for execution (<span class="dr-code">ClaimTask</span>), setting task status after execution (<span class="dr-code">SetResults</span>) and heartbeats during task execution (<span class="dr-code">Heartbeat</span>). <span class="dr-code">ClaimTask</span> requests originate from the Controllers in response to <span class="dr-code">NextWork</span> requests. <span class="dr-code">Heartbeat</span> and <span class="dr-code">SetResults</span> requests originate from executor processes during and after task execution. The HSC interacts with the task store to update the task status on the kind of request it receives.
+                                </p>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="data-model">
+                                    <h2 class="dr-article-content__section-title">
+                                        Data model
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    ATF uses our in-house metadata store, Edgestore, as a task store. Edgestore objects can be Entities or Associations (<span class="dr-code">assoc</span>), each of which can have user-defined attributes. Associations are used to represent relationships between entities. Edgestore supports indexing only on attributes of associations.
+                                </p>
+                                <p>
+                                    Based on this design, we have two kinds of ATF-related objects in Edgestore. The ATF association stores scheduling information, such as the next scheduled timestamp at which the Store Consumer should poll a given task (either for the first time or for a retry). The ATF entity stores all task related information that is used to track the task state and payload for task execution. We query on associations from the Store Consumer in a pull model to pick up tasks ready for execution.
+                                </p>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="lifecycle-of-a-task">
+                                    <h2 class="dr-article-content__section-title">
+                                        Lifecycle of a task
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <ol>
+                                    <li>Client performs a <span class="dr-code">Schedule</span> RPC call to <b>Frontend</b> with task information, including execution time.&#160;
+                                    </li>
+                                    <li>Frontend creates Edgestore <span class="dr-code">entity</span> and <span class="dr-code">assoc</span> for the task.&#160;
+                                    </li>
+                                    <li>When it is time to process the task, <b>Store Consumer</b> pulls the task from <b>Edgestore</b> and pushes it to a related <b>SQS</b> queue.&#160;
+                                    </li>
+                                    <li>
+                                        <b>Executor</b> makes <span class="dr-code">NextWork</span> RPC call to <b>Controller</b>, which pulls tasks from the <b>SQS</b> queue, makes a <span class="dr-code">ClaimTask</span> RPC to the HSC and then returns the task to the <b>Executor</b>.&#160;
+                                    </li>
+                                    <li>
+                                        <b>Executor</b> invokes the callback for the task. While processing, <b>Executor</b> performs <span class="dr-code">Heartbeat</span> RPC calls to <b>Heartbeat and Status Controller (HSC)</b>. Once processing is done, <b>Executor</b> performs <span class="dr-code">TaskStatus</span> RPC call to <b>HSC</b>.&#160;
+                                    </li>
+                                    <li>Upon getting <span class="dr-code">Heartbeat</span> and <span class="dr-code">TaskStatus</span> RPC calls, <b>HSC</b> updates the <b>Edgestore</b> entity and <span class="dr-code">assoc</span>.
+                                    </li>
+                                </ol>
+                                <p>
+                                    Every state update in the lifecycle of a task is accompanied by an update to the next trigger timestamp in the <span class="dr-code">assoc</span>. This ensures that the Store Consumer pulls the task again if there is no change in state of the task within the next trigger timestamp. This helps ATF achieve its at-least-once delivery guarantee by ensuring that no task is dropped.
+                                </p>
+                                <p>
+                                    Following are the task entity and association states in ATF and their corresponding timestamp updates:
+                                </p>
+                                <table>
+                                    <tbody>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <b>Entity status</b>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <b>Assoc status</b>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <b>next trigger timestamp in Assoc</b>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <b>Comment</b>
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">new</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">new</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">scheduled_timestamp</span> of the task
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    Pick up new tasks that are ready.&#160;
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">enqueued</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">started</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">enqueued_timestamp</span> + <span class="dr-code">enqueue_timeout</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    Re-enqueue task if it has been in <span class="dr-code">enqueued</span> state for too long. This can happen if the queue loses data or the controller goes down after polling the queue and before the task is claimed.
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">claimed</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">started</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">claimed_timestamp</span> + <span class="dr-code">claim_timeout</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    Re-enqueue if task is claimed but never transfered to <span class="dr-code">processing</span>. This can happen if Controller is down after claiming a task. Task status is changed to <span class="dr-code">enqueued</span> after re-enqueue.
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">processing</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">started</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">heartbeat_timestamp</span> + <span class="dr-code">heartbeat_timeout</span>`
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    Re-enqueue if task hasn’t sent <span class="dr-code">heartbeat</span> for too long. This can happen if Executor is down. Task status is changed to <span class="dr-code">enqueued</span> after re-enqueue.&#160;
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">retriable failure</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    started
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    compute <span class="dr-code">next_timestamp</span> according to backoff logic
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    Exponential backoff for tasks with retriable failure.&#160;
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">success</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">completed</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    N/A
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    &#160;
+                                                </p>
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">fatal_failure</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    <span class="dr-code">completed</span>
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    N/A
+                                                </p>
+                                            </td>
+                                            <td>
+                                                <p>
+                                                    &#160;
+                                                </p>
+                                            </td>
+                                        </tr>
+                                    </tbody>
+                                </table>
+                                <p>
+                                    The store consumer polls for tasks based on the following query:
+                                </p>
+                                <p>
+                                    <span class="dr-code">assoc_status= &amp;&amp; next_timestamp&lt;=time.now()<br /></span>
+                                </p>
+                                <p>
+                                    Below is the state machine that defines task state transitions:&#160;<br />
+                                </p>
+                            </div>
+                            <div class="image c04-image aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-image image cq-dd-image">
+                                    <figure class="dr-margin-0 dr-display-inline-block">
+                                        <img src="/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x225px-2.png" aria-hidden="false" alt="Task State Transitions [Fig 2]" height="450" width="1440" />
+                                    </figure>
+                                </div>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="-achieving-guarantees">
+                                    <h2 class="dr-article-content__section-title">
+                                        Achieving guarantees
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    <b>At-least-once task execution<br /></b> At-least-once execution is guaranteed in ATF by retrying a task until it completes execution (which is signaled by a <span class="dr-code">Success</span> or a <span class="dr-code">FatalFailure</span> state). All ATF system errors are implicitly considered retriable failures, and lambda owners have an option of marking tasks with a <span class="dr-code">RetriableFailure</span> state. Tasks might be dropped from the ATF execution pipeline in different parts of the system through transient RPC failures and failures on dependencies like Edgestore or SQS. These transient failures at different parts of the system do not affect the at-least-once guarantee, though, because of the system of timeouts and re-polling from Store Consumer.
+                                </p>
+                                <p>
+                                    <b>No concurrent task execution<br /></b> Concurrent task execution is avoided through a combination of two methods in ATF. First, tasks are explicitly claimed through an exclusive task state (<span class="dr-code">Claimed</span>) before starting execution. Once the task execution is complete, the task status is updated to one of <span class="dr-code">Success</span>, <span class="dr-code">FatalFailure</span> or <span class="dr-code">RetriableFailure</span>. A task can be claimed only if its existing task state is <span class="dr-code">Enqueued</span> (retried tasks go to the <span class="dr-code">Enqueued</span> state as well once they are re-pushed onto SQS).
+                                </p>
+                                <p>
+                                    However, there might be situations where once a long running task starts execution, its heartbeats might fail repeatedly yet the task execution continues. ATF would retry this task by polling it from the store consumer because the heartbeat timeouts would’ve expired. This task can then be claimed by another worker and lead to concurrent execution.&#160;<br />
+                                </p>
+                                <p>
+                                    To avoid this situation, there is a termination logic in the Executor processes whereby an Executor process terminates itself as soon as three consecutive heartbeat calls fail. Each heartbeat timeout is large enough to eclipse three consecutive heartbeat failures. This ensures that the Store Consumer cannot pull such tasks before the termination logic ends them—the second method that helps achieve this guarantee.
+                                </p>
+                                <p>
+                                    <b>Isolation<br /></b> Isolation of lambdas is achieved through dedicated worker clusters, dedicated queues, and dedicated per-lambda scheduling quotas. In addition, isolation across different priorities within the same lambda is likewise achieved through dedicated queues and scheduling bandwidth.
+                                </p>
+                                <p>
+                                    <b>Delivery latency<br /></b> ATF use cases do not require ultra-low task delivery latencies. Task delivery latencies on the order of a couple of seconds are acceptable. Tasks ready for execution are periodically polled by the Store Consumer and this period of polling largely controls the task delivery latency. Using this as a tuning lever, ATF can achieve different delivery latencies as required. Increasing poll frequency reduces task delivery latency and vice versa. Currently, we have calibrated ATF to poll for ready tasks once every two seconds.
+                                </p>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="ownership-model">
+                                    <h2 class="dr-article-content__section-title">
+                                        Ownership model
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    ATF is designed to be a self-serve framework for developers at Dropbox. The design is very intentional in driving an ownership model where lambda owners own all aspects of their lambdas’ operations. To promote this, all lambda worker clusters are owned by the lambda owners. They have full control over operations on these clusters, including code deployments and capacity management. Each executor process is bound to one lambda. Owners have the option of deploying multiple lambdas on their worker clusters simply by spawning new executor processes on their hosts.
+                                </p>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="-extending-atf">
+                                    <h2 class="dr-article-content__section-title">
+                                        Extending ATF
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    As described above, ATF provides an infrastructural building block for scheduling asynchronous tasks. With this foundation established, ATF can be extended to support more generic use cases and provide more features as a framework. Following are some examples of what could be built as an extension to ATF.&#160;
+                                </p>
+                                <p>
+                                    <b>Periodic task execution<br /></b> Currently, ATF is a system for one-time task scheduling. Building support for periodic task execution as an extension to this framework would be useful in unlocking new capabilities for our clients.
+                                </p>
+                                <p>
+                                    <b>Better support for task chaining<br /></b> Currently, it is possible to chain tasks on ATF by scheduling a task onto ATF that then schedules other tasks onto ATF during its execution. Although it is possible to do this in the current ATF setup, visibility and control on this chaining is absent at the framework level. Another natural extension here would be to better support task chaining through framework-level visibility and control, to make this use case a first class concept in the ATF model.
+                                </p>
+                                <p>
+                                    <b>Dead letter queues for misbehaving tasks<br /></b> One common source of maintenance overhead we observe on ATF is that some tasks get stuck in infinite retry loops due to occasional bugs in lambda logic. This requires manual intervention from the ATF framework owners in some cases where there are a large number of tasks stuck in such loops, occupying a lot of the scheduling bandwidth in the system. Typical manual actions in response to such a situation include pausing execution of the lambdas with misbehaving tasks, or dropping them outright.
+                                </p>
+                                <p>
+                                    One way to reduce this operational overhead and provide an easy interface for lambda owners to recover from such incidents would be to create dead letter queues filled with such misbehaving tasks. The ATF framework could impose a maximum number of retries before tasks are pushed onto the dead letter queue. We could create and expose tools that make it easy to reschedule tasks from the dead letter queue back into the ATF system, once the associated lambda bugs are fixed.<br />
+                                </p>
+                            </div>
+                            <div class="section aem-GridColumn aem-GridColumn--default--12">
+                                <div class="dr-article-content__section" id="conclusion">
+                                    <h2 class="dr-article-content__section-title">
+                                        Conclusion
+                                    </h2>
+                                </div>
+                            </div>
+                            <div class="text parbase aem-GridColumn aem-GridColumn--default--12">
+                                <p>
+                                    We hope this post helps engineers elsewhere to develop better async task frameworks of their own. Many thanks to everyone who worked on this project: Anirudh Jayakumar, Deepak Gupta, Dmitry Kopytkov, Koundinya Muppalla, Peng Kang, Rajiv Desai, Ryan Armstrong, Steve Rodrigues, Thomissa Comellas, Xiaonan Zhang and Yuhuan Du.<br />
+                                    &#160;
+                                </p>
+                            </div>
+                        </div>
+                    </div>
+                    <hr class="dr-typography-t5 dr-margin-top-50 dr-article-content__divider" />
+                    <div class="dr-typography-t5"></div>
+                    <div class="dr-typography-t5 dr-margin-top-20">
+                        // Tags<br />
+                        <ul class="dr-unstyled-list dr-margin-top-10 dr-typography-t4">
+                            <li class="dr-container--infrastructure dr-display-inline-block dr-margin-right-10 dr-margin-bottom-10">
+                                <a class="dr-link dr-pill dr-pill--primary dr-link--no-underline" href="https://dropbox.tech/infrastructure">Infrastructure</a>
+                            </li>
+                            <li class="dr-display-inline-block dr-margin-right-10">
+                                <a class="dr-link dr-pill dr-link--no-underline" href="https://dropbox.tech/tag-results.task-scheduling">Task Scheduling</a>
+                            </li>
+                            <li class="dr-display-inline-block dr-margin-right-10">
+                                <a class="dr-link dr-pill dr-link--no-underline" href="https://dropbox.tech/tag-results.async">Async</a>
+                            </li>
+                            <li class="dr-display-inline-block dr-margin-right-10">
+                                <a class="dr-link dr-pill dr-link--no-underline" href="https://dropbox.tech/tag-results.edgestore">Edgestore</a>
+                            </li>
+                        </ul>
+                    </div>
+                    <div class="dr-typography-t5 dr-margin-top-20 dr-hide-from-md">
+                        // Copy link<br />
+                        <div class="dr-article-content__social-links-tooltip dr-display-none">
+                            Link copied
+                        </div><button class="dr-button dr-button--link dr-link dr-link--no-underline dr-article-content__copy-link" data-dr-tooltip="Copy link"><img alt="Copy link" class="dr-display-block dr-invert-on-theme-dark" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content/resources/copy.svg" /></button>
+                    </div>
+                    <div class="dr-article-content__social-links">
+                        <ul class="dr-article-content__social-links-list dr-unstyled-list">
+                            <li class="dr-margin-bottom-20">
+                                <div class="dr-article-content__social-links-tooltip dr-typography-t5 dr-display-none">
+                                    Link copied
+                                </div><button class="dr-button dr-display-block dr-link dr-link--no-underline dr-article-content__copy-link dr-button--link" data-dr-tooltip="Copy link" data-dr-tooltip-position="cl" data-dr-tooltip-theme="bw"><img alt="Copy link" class="dr-display-block dr-invert-on-theme-dark" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content/resources/copy.svg" /></button>
+                            </li>
+                            <li class="dr-margin-bottom-20">
+                                <a class="dr-link dr-display-block dr-link--no-underline dr-article-content__share-link dr-article-content__twitter-link" data-dr-tooltip="Share on Twitter" data-dr-tooltip-position="cl" data-dr-tooltip-theme="bw" href="https://twitter.com/intent/tweet/?text=How%20we%20designed%20Dropbox%20ATF%3A%20an%20async%20task%20framework&amp;url=https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" target="_blank"><img alt="Share on Twitter" class="dr-display-block dr-invert-on-theme-dark" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content/resources/twitter.svg" /></a>
+                            </li>
+                            <li class="dr-margin-bottom-20">
+                                <a class="dr-link dr-display-block dr-link--no-underline dr-article-content__share-link dr-article-content__facebook-link" data-dr-tooltip="Share on Facebook" data-dr-tooltip-position="cl" data-dr-tooltip-theme="bw" href="https://facebook.com/sharer/sharer.php?u=https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" target="_blank"><img alt="Share on Facebook" class="dr-display-block dr-invert-on-theme-dark" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content/resources/facebook.svg" /></a>
+                            </li>
+                            <li>
+                                <a class="dr-link dr-display-block dr-link--no-underline dr-article-content__share-link dr-article-content__linkedin-link" data-dr-tooltip="Share on Linkedin" data-dr-tooltip-position="cl" data-dr-tooltip-theme="bw" href="https://www.linkedin.com/shareArticle?mini=true&amp;url=https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox&amp;title=How%20we%20designed%20Dropbox%20ATF%3A%20an%20async%20task%20framework&amp;source=https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" target="_blank"><img alt="Share on Linkedin" class="dr-display-block dr-invert-on-theme-dark" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content/resources/linkedin.svg" /></a>
+                            </li>
+                        </ul>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="aem-Grid aem-Grid--12 aem-Grid--default--12">
+            <div class="plain-html c17-plain-html aem-GridColumn aem-GridColumn--default--12">
+                <div class="knotch_placeholder"></div>
+            </div>
+        </div>
+        <footer class="dr-footer">
+            <div class="dr-container--surface">
+                <section class="dr-container__content dr-footer__container">
+                    <div class="dr-newsletter-subscription__succeed dr-display-none dr-typography-t5">
+                        <hr class="dr-newsletter-subscription__form-divider" />
+                        <div class="dr-margin-bottom-30 dr-margin-top-30">
+                            <!--// Thanks for subscribing.-->
+                            <div class="dr-show-block-from-lg">
+                                <img src="/cms/content/dam/dropbox/tech-blog/en-us/subscribe/thanksforsubscribing_desktop.png" title="subscription__success" alt="subscription__success" />
+                            </div>
+                            <div class="dr-show-block-from-md dr-hide-from-lg dr-hide-from-sm">
+                                <img src="/cms/content/dam/dropbox/tech-blog/en-us/subscribe/thanksforsubscribing_tablet.png" title="subscription__success" alt="subscription__success" />
+                            </div>
+                            <div class="dr-show-block-from-sm dr-hide-from-lg dr-hide-from-md">
+                                <img src="/cms/content/dam/dropbox/tech-blog/en-us/subscribe/thanksforsubscribing_mobile.png" title="subscription__success" alt="subscription__success" />
+                            </div>
+                        </div>
+                        <hr class="dr-newsletter-subscription__form-divider" />
+                    </div>
+                    <form role="form" class="dr-typography-t5 dr-newsletter-subscription__form" novalidate="">
+                        <hr class="dr-newsletter-subscription__form-divider" />
+                        <div class="dr-margin-top-30 dr-margin-bottom-30 dr-margin-md-top-10 dr-margin-md-bottom-10">
+                            // Subscribe to email updates by category
+                        </div>
+                        <div class="dr-margin-left-25">
+                            <p class="dr-newsletter-subscription__topic-error dr-display-none dr-color-tangerine">
+                                Select at least one topic
+                            </p><label class="dr-newsletter-subscription__form-label" for="newsletterForm.application"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.application" name="categories[ ]" type="checkbox" value="Application" data-mid="127814" />Application</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.frontend"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.frontend" name="categories[ ]" type="checkbox" value="Front End" data-mid="127842" />Front End</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.infrastructure"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.infrastructure" name="categories[ ]" type="checkbox" value="Infrastructure" data-mid="127826" />Infrastructure</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.machine-learning"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.machine-learning" name="categories[ ]" type="checkbox" value="Machine Learning" data-mid="127830" />Machine Learning</label><br class="dr-show-block-from-md" />
+                            <label class="dr-newsletter-subscription__form-label" for="newsletterForm.mobile"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.mobile" name="categories[ ]" type="checkbox" value="Mobile" data-mid="127834" />Mobile</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.security"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.security" name="categories[ ]" type="checkbox" value="Security" data-mid="127838" />Security</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.developers"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.developers" name="categories[ ]" type="checkbox" value="Developers" data-mid="129642" />Developers</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.all"><input class="dr-newsletter-subscription__form-checkbox dr-newsletter-subscription__form-checkbox--all dr-input" id="newsletterForm.all" type="checkbox" />All</label>
+                        </div>
+                        <p class="dr-newsletter-subscription__error dr-display-none dr-color-tangerine">
+                            Error occurred!<br />
+                            Please try again later
+                        </p>
+                        <p class="dr-newsletter-subscription__email-error dr-display-none dr-color-tangerine">
+                            Enter a valid address
+                        </p>
+                        <div class="dr-newsletter-subscription__email-container dr-margin-bottom-20 dr-margin-top-40 dr-margin-md-top-0">
+                            <div>
+                                // Type your email address
+                            </div><input autocomplete="off" class="dr-newsletter-subscription__form-input dr-flex-1" name="email" type="email" />
+                            <div class="dr-newsletter-subscription__actions-container">
+                                <div class="dr-newsletter-subscription__loading dr-display-none">
+                                    Submitting...
+                                </div><button type="submit" disabled="disabled" class="dr-newsletter-subscription__form-submit dr-button dr-typography-t5">Subscribe</button>
+                            </div>
+                        </div>
+                        <hr class="dr-newsletter-subscription__form-divider" />
+                    </form>
+                    <div class="dr-grid dr-grid--md-2">
+                        <div>
+                            <a href="https://dropbox.com" target="_blank" class="dr-margin-bottom-20 dr-display-block"><img alt="Dropbox" height="40" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-all/resources/logo_dropbox.svg" width="164" /></a>
+                        </div>
+                        <ul class="dr-footer-links dr-unstyled-list dr-typography-t10 dr-grid dr-grid--2 dr-grid--column-gap-15">
+                            <li>
+                                <a class="dr-link dr-link--no-underline" href="http://dropbox.com/jobs" target="_blank">Jobs</a>
+                            </li>
+                            <li>
+                                <a class="dr-link dr-link--no-underline" href="https://medium.com/@Dropbox" target="_blank">Medium</a>
+                            </li>
+                            <li>
+                                <a class="dr-link dr-link--no-underline" href="https://www.dropbox.com/privacy" target="_blank">Privacy</a>
+                            </li>
+                            <li>
+                                <a class="dr-link dr-link--no-underline" href="https://twitter.com/Dropbox" target="_blank">twitter</a>
+                            </li>
+                            <li>
+                                <a class="dr-link dr-link--no-underline" href="https://www.dropbox.com/terms" target="_blank">Terms</a>
+                            </li>
+                            <li>
+                                <a class="dr-link dr-link--no-underline" href="https://www.instagram.com/dropbox" target="_blank">Instagram</a>
+                            </li>
+                            <li>
+                                <a class="dr-link dr-link--no-underline" href="https://blog.dropbox.com/" target="_blank">Work In Progress</a>
+                            </li>
+                        </ul>
+                    </div>
+                </section>
+            </div>
+        </footer>
+        <div id="u04-snapengage-config" data-snapengage-widget-id="d5c1efed-d0ef-4fca-8c7d-faff398ad272" data-proactive-chat="false" style="display:none;"></div>
+        <script type="text/javascript" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-common/clientlib-cms-common.7f3cf4624fd698d8bfec572c3c993880.js"></script> 
+        <script type="text/javascript" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-all.3230e3eaa6e5a90686710bfde829f620.js"></script> 
+        <script type="text/javascript" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content.2c12dd2925c2dcad6bde22d2ff271137.js"></script> 
+        <script type="application/javascript">
+        <![CDATA[
+        document.body.classList.remove('stormcrow-animate');
+        ]]>
+        </script> <noscript></noscript>
+    </body>
+</html>

article_scraper/resources/tests/readability/ebb-org/source.html

@@ -0,0 +1,485 @@
+<!DOCTYPE html>
+<html lang="en-US" xml:lang="en-US" xmlns="http://www.w3.org/1999/xhtml">
+    <head>
+        <meta content="text/html; charset=utf-8" http-equiv="Content-Type" />
+        <title>
+            On Recent Controversial Events - Bradley M. Kuhn ( Brad ) ( bkuhn )
+        </title>
+        <meta content="Bradley M. Kuhn (http://ebb.org/bkuhn/)" name="author" />
+        <link href="http://ebb.org/bkuhn/rss.xml" rel="alternate" title="Whole Website RSS for Bradley M. Kuhn" type="application/rss+xml" />
+        <link href="http://ebb.org/bkuhn/blog/rss.xml" rel="alternate" title="Blog RSS for Bradley M. Kuhn" type="application/rss+xml" />
+        <link href="http://ebb.org/bkuhn/blog/rss.xml" rel="alternate" title="Articles / Interviews RSS for Bradley M. Kuhn" type="application/rss+xml" />
+        <link href="/css/screen.css" rel="stylesheet" type="text/css" />
+        <link href="mailto:bkuhn@ebb.org" rel="made" />
+        <meta content="kuhn, brad, bradley, bkuhn, hacker, gnu, free, software, linux, unix, perl, liberal, pgp, gpg, geek, system, administrator, network, free software, developer, new york, NY, computer, open source, hacker, software, freedom, software freedom" http-equiv="Keywords" name="Keywords" />
+        <meta content="The website of Bradley M. Kuhn, aka Brad, aka bkuhn. This site includes his GPG keys, resume, blog, projects list, software, interviews, speeches and writing." http-equiv="Description" name="Description" />
+    </head>
+    <body>
+        <div id="site">
+            <div id="sidebar">
+                <div id="title">
+                    <p>
+                        <a href="/bkuhn">Bradley M. Kuhn</a> <a href="http://ebb.org/bkuhn/rss.xml"><img alt="[RSS of Whole Site]" border="0" src="/images/feed-icon-14x14.png" /></a>
+                    </p>
+                </div>
+                <div id="menu">
+                    <ul>
+                        <li>
+                            <a href="/bkuhn/contact">Contact</a>
+                        </li>
+                        <li>
+                            <a href="/bkuhn/blog">Blog</a> &#160; <a href="http://ebb.org/bkuhn/blog/rss.xml"><img alt="[RSS of Blog]" border="0" src="/images/feed-icon-14x14.png" /></a>
+                        </li>
+                        <li>
+                            <a href="http://identi.ca/bkuhn/">Pump.io Social Network</a>
+                        </li><!-- %li=  link_to("About", "/bkuhn/about") -->
+                        <!-- %li=  link_to("Speeches / Talks", "/bkuhn/speeches/") -->
+                        <li>
+                            <a href="/bkuhn/articles">Interviews / Articles</a> &#160; <a href="http://ebb.org/bkuhn/articles/rss.xml"><img alt="[RSS of Articles]" border="0" src="/images/feed-icon-14x14.png" /></a>
+                        </li>
+                        <li>
+                            <a href="/bkuhn/code/">Software</a>
+                        </li>
+                        <li>
+                            <a href="/bkuhn/resume">Résumé</a>
+                        </li>
+                    </ul>
+                </div>
+                <h2>
+                    <a href="/bkuhn/tags">Tag</a> Cloud
+                </h2>
+                <ol id="tag-cloud">
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#accounting">accounting</a>
+                    </li>
+                    <li class="tier-4" title="24 posts">
+                        <a href="/bkuhn/tags.html#advocacy">advocacy</a>
+                    </li>
+                    <li class="tier-4" title="16 posts">
+                        <a href="/bkuhn/tags.html#agpl">agpl</a>
+                    </li>
+                    <li class="tier-3" title="6 posts">
+                        <a href="/bkuhn/tags.html#android">android</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#apache">apache</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#apple">apple</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#apt">apt</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#artistic">artistic</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#asterisk">asterisk</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#automotive">automotive</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#autonomous">autonomous</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#award">award</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#bilski">bilski</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#canonical">canonical</a>
+                    </li>
+                    <li class="tier-3" title="7 posts">
+                        <a href="/bkuhn/tags.html#cla">cla</a>
+                    </li>
+                    <li class="tier-5" title="39 posts">
+                        <a href="/bkuhn/tags.html#community">community</a>
+                    </li>
+                    <li class="tier-3" title="6 posts">
+                        <a href="/bkuhn/tags.html#compliance">compliance</a>
+                    </li>
+                    <li class="tier-4" title="15 posts">
+                        <a href="/bkuhn/tags.html#conferences">conferences</a>
+                    </li>
+                    <li class="tier-5" title="37 posts">
+                        <a href="/bkuhn/tags.html#conservancy">conservancy</a>
+                    </li>
+                    <li class="tier-3" title="8 posts">
+                        <a href="/bkuhn/tags.html#copyleft">copyleft</a>
+                    </li>
+                    <li class="tier-5" title="56 posts">
+                        <a href="/bkuhn/tags.html#copyright">copyright</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#cow-orking">cow-orking</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#cpp">cpp</a>
+                    </li>
+                    <li class="tier-3" title="9 posts">
+                        <a href="/bkuhn/tags.html#debian">debian</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#denounce">denounce</a>
+                    </li>
+                    <li class="tier-2" title="4 posts">
+                        <a href="/bkuhn/tags.html#development">development</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#diversity">diversity</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#emacs">emacs</a>
+                    </li>
+                    <li class="tier-2" title="4 posts">
+                        <a href="/bkuhn/tags.html#encryption">encryption</a>
+                    </li>
+                    <li class="tier-3" title="10 posts">
+                        <a href="/bkuhn/tags.html#enforcement">enforcement</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#exceptions">exceptions</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#faif">faif</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#fdl">fdl</a>
+                    </li>
+                    <li class="tier-4" title="11 posts">
+                        <a href="/bkuhn/tags.html#for-profit">for-profit</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#fosdem">fosdem</a>
+                    </li>
+                    <li class="tier-4" title="13 posts">
+                        <a href="/bkuhn/tags.html#fsf">fsf</a>
+                    </li>
+                    <li class="tier-2" title="4 posts">
+                        <a href="/bkuhn/tags.html#gcc">gcc</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#git">git</a>
+                    </li>
+                    <li class="tier-4" title="13 posts">
+                        <a href="/bkuhn/tags.html#gnome">gnome</a>
+                    </li>
+                    <li class="tier-3" title="6 posts">
+                        <a href="/bkuhn/tags.html#gnu">gnu</a>
+                    </li>
+                    <li class="tier-2" title="4 posts">
+                        <a href="/bkuhn/tags.html#google">google</a>
+                    </li>
+                    <li class="tier-6" title="107 posts">
+                        <a href="/bkuhn/tags.html#gpl">gpl</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#gpl-compatibility">gpl-compatibility</a>
+                    </li>
+                    <li class="tier-5" title="47 posts">
+                        <a href="/bkuhn/tags.html#gpl-enforcement">gpl-enforcement</a>
+                    </li>
+                    <li class="tier-2" title="4 posts">
+                        <a href="/bkuhn/tags.html#gplv3">gplv3</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#guadec">guadec</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#identica">identica</a>
+                    </li>
+                    <li class="tier-4" title="17 posts">
+                        <a href="/bkuhn/tags.html#infringement">infringement</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#java">java</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#javascript">javascript</a>
+                    </li>
+                    <li class="tier-2" title="4 posts">
+                        <a href="/bkuhn/tags.html#jvm">jvm</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#launchpad">launchpad</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#ldap">ldap</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#lgpl">lgpl</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#libreoffice">libreoffice</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#libreplanet">libreplanet</a>
+                    </li>
+                    <li class="tier-6" title="72 posts">
+                        <a href="/bkuhn/tags.html#licensing">licensing</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#lindows">lindows</a>
+                    </li>
+                    <li class="tier-4" title="11 posts">
+                        <a href="/bkuhn/tags.html#linux">linux</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#maemo">maemo</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#mail">mail</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#meego">meego</a>
+                    </li>
+                    <li class="tier-4" title="17 posts">
+                        <a href="/bkuhn/tags.html#microsoft">microsoft</a>
+                    </li>
+                    <li class="tier-3" title="6 posts">
+                        <a href="/bkuhn/tags.html#mobile">mobile</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#moblin">moblin</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#mono">mono</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#motorola">motorola</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#mta">mta</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#mysql">mysql</a>
+                    </li>
+                    <li class="tier-4" title="13 posts">
+                        <a href="/bkuhn/tags.html#net-services">net-services</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#nlp">nlp</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#nokia">nokia</a>
+                    </li>
+                    <li class="tier-4" title="23 posts">
+                        <a href="/bkuhn/tags.html#non-profit">non-profit</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#np-complete">np-complete</a>
+                    </li>
+                    <li class="tier-3" title="7 posts">
+                        <a href="/bkuhn/tags.html#open-core">open-core</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#open-foam">open-foam</a>
+                    </li>
+                    <li class="tier-2" title="4 posts">
+                        <a href="/bkuhn/tags.html#oracle">oracle</a>
+                    </li>
+                    <li class="tier-2" title="4 posts">
+                        <a href="/bkuhn/tags.html#parrot">parrot</a>
+                    </li>
+                    <li class="tier-4" title="16 posts">
+                        <a href="/bkuhn/tags.html#patents">patents</a>
+                    </li>
+                    <li class="tier-3" title="6 posts">
+                        <a href="/bkuhn/tags.html#perl">perl</a>
+                    </li>
+                    <li class="tier-3" title="6 posts">
+                        <a href="/bkuhn/tags.html#perljvm">perljvm</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#permissive-license">permissive-license</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#piracy">piracy</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#podcast">podcast</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#podjango">podjango</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#poker">poker</a>
+                    </li>
+                    <li class="tier-4" title="14 posts">
+                        <a href="/bkuhn/tags.html#politics">politics</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#postfix">postfix</a>
+                    </li>
+                    <li class="tier-4" title="13 posts">
+                        <a href="/bkuhn/tags.html#proprietary">proprietary</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#qt">qt</a>
+                    </li>
+                    <li class="tier-2" title="4 posts">
+                        <a href="/bkuhn/tags.html#replicant">replicant</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#requiem">requiem</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#rtlinux">rtlinux</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#scale">SCALE</a>
+                    </li>
+                    <li class="tier-3" title="6 posts">
+                        <a href="/bkuhn/tags.html#sco">sco</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#scotus">scotus</a>
+                    </li>
+                    <li class="tier-3" title="5 posts">
+                        <a href="/bkuhn/tags.html#security">security</a>
+                    </li>
+                    <li class="tier-3" title="6 posts">
+                        <a href="/bkuhn/tags.html#sexism">sexism</a>
+                    </li>
+                    <li class="tier-2" title="4 posts">
+                        <a href="/bkuhn/tags.html#sflc">sflc</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#slicing">slicing</a>
+                    </li>
+                    <li class="tier-3" title="7 posts">
+                        <a href="/bkuhn/tags.html#social-justice">social-justice</a>
+                    </li>
+                    <li class="tier-3" title="6 posts">
+                        <a href="/bkuhn/tags.html#software">software</a>
+                    </li>
+                    <li class="tier-6" title="108 posts">
+                        <a href="/bkuhn/tags.html#software-freedom">software-freedom</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#speeches">speeches</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#stet">stet</a>
+                    </li>
+                    <li class="tier-2" title="4 posts">
+                        <a href="/bkuhn/tags.html#talks">talks</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#tcl">tcl</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#teaching">teaching</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#tech-press">tech-press</a>
+                    </li>
+                    <li class="tier-5" title="35 posts">
+                        <a href="/bkuhn/tags.html#technology">technology</a>
+                    </li>
+                    <li class="tier-2" title="2 posts">
+                        <a href="/bkuhn/tags.html#thesis">thesis</a>
+                    </li>
+                    <li class="tier-3" title="5 posts">
+                        <a href="/bkuhn/tags.html#trademarks">trademarks</a>
+                    </li>
+                    <li class="tier-3" title="10 posts">
+                        <a href="/bkuhn/tags.html#ubuntu">ubuntu</a>
+                    </li>
+                    <li class="tier-1" title="1 post">
+                        <a href="/bkuhn/tags.html#voip">voip</a>
+                    </li>
+                    <li class="tier-2" title="3 posts">
+                        <a href="/bkuhn/tags.html#xen">xen</a>
+                    </li>
+                </ol>
+                <h2>
+                    Powered by
+                </h2><a href="https://gitorious.org/bkuhn/jekyll/source/HEAD:">A Very Old Fork of Jekyll</a> <a href="https://gitorious.org/bkuhn/website/source/HEAD:">"Source Code" for this site</a>
+            </div>
+            <div id="contentWithSidebar">
+                <div id="post">
+                    <h1>
+                        On Recent Controversial Events
+                    </h1>
+                    <p class="topAttributionWithDate">
+                        Tuesday 15 October 2019 by Bradley M. Kuhn
+                    </p>
+                    <p>
+                        The last 33 days have been unprecedentedly difficult for the software freedom community and for me personally. Folks have been emailing, phoning, texting, tagging me on social media (— the last of which has been funny, because all my social media accounts are placeholder accounts). But, just about everyone has urged me to comment on the serious issues that the software freedom community now faces. Until now, I have stayed silent regarding all these current topics: from Richard M. Stallman (RMS)'s public statements, to <a href="https://www.fsf.org/news/richard-m-stallman-resigns">his resignation from the Free Software Foundation (FSF)</a>, to the Epstein scandal and its connection to MIT. I've also avoided generally commenting on software freedom organizational governance during this period. I did this for good reason, which is explained below. However, in this blog post, I now share my primary comments on the matters that seem to currently be of the utmost attention of the Open Source and Free Software communities.
+                    </p>
+                    <p>
+                        I have been silent the last month because, until two days ago, I was an at-large member of <a href="https://www.fsf.org/about/staff-and-board">FSF's Board of Directors</a>, and a <a href="https://static.fsf.org/nosvn/fsf-amended-bylaws-current.pdf">Voting Member</a> of the FSF. As a member of FSF's two leadership bodies, I was abiding by a reasonable request from the FSF management and my duty to the organization. Specifically, the FSF asked that all communication during the crisis <a href="https://www.fsf.org/news/richard-m-stallman-resigns">come</a> <a href="https://www.fsf.org/news/fsf-and-gnu">directly</a> from FSF officers and not from at-large directors and/or Voting Members. Furthermore, the FSF management asked all Directors and Voting Members to remain silent on this entire matter — even on issues only tangentially related to the current situation, and even when speaking in our own capacity (e.g., on our own blogs like this one). The FSF is an important organization, and I take any request from the FSF seriously — so I abided fully with their request.
+                    </p>
+                    <p>
+                        The situation was further complicated because folks at my employer, Software Freedom Conservancy (where I also serve on the <a href="https://sfconservancy.org/about/board/#bkuhn">Board of Directors</a>) had strong opinions about this matter as well. Fortunately, the FSF and Conservancy both had already created clear protocols for what I should do if ever there was a disagreement or divergence of views between Conservancy and FSF. I therefore was recused fully from the planning, drafting, and timing of Conservancy's statement on this matter. I thank my colleagues at the Conservancy for working so carefully to keep me entirely outside the loop on their statement and to diligently assure that it was straight-forward for me to manage any potential organizational disagreements. I also thank those at the FSF who outlined clear protocols (ahead of time, back in March 2019) in case a situation like this ever came up. I also know my colleagues at Conservancy care deeply, as I do, about the health and welfare of the FSF and its mission of fighting for universal software freedom for all. None of us want, nor have, any substantive disagreement over software freedom issues.
+                    </p>
+                    <p>
+                        I take very seriously my duty to the various organizations where I have (or have had) affiliations. More generally, I champion non-profit organizational transparency. Unfortunately, the current crisis left me in a quandary between the overarching goal of community transparency and abiding by FSF management's directives. Now that I've left the FSF Board of Directors, FSF's Voting Membership, and all my FSF volunteer roles (which ends my 22-year uninterrupted affiliation with the FSF), I can now comment on the substantive issues that face not just the FSF, but the Free Software community as a whole, while continuing to adhere to my past duty of acting in FSF's best interest. In other words, my affiliation with the FSF has come to an end for many good and useful reasons. The end to this affiliation allows me to speak directly about the core issues at the heart of the community's current crisis.
+                    </p>
+                    <p>
+                        Firstly, all these events — from RMS' public comments on the MIT mailing list, to RMS' resignation from the FSF to RMS' discussions about the next steps for the GNU project — <em>seem</em> to many to have happened ridiculously quickly. But it wasn't actually fast at all. In fact, these events were culmination of issues that were slowly growing in concern to many people, including me.
+                    </p>
+                    <p>
+                        For the last two years, I had been a loud internal voice in the FSF leadership regarding RMS' Free-Software-unrelated public statements; I felt strongly that it was in the best interest of the FSF to actively seek to limit such statements, and that it was my duty to FSF to speak out about this within the organization. Those who only learned of this story in the last month (understandably) believed <a href="https://medium.com/@selamjie/remove-richard-stallman-fec6ec210794">Selam G.'s Medium post</a> raised an entirely new issue. <a href="https://web.archive.org/web/20161107050933/https://www.stallman.org/archives/2016-jul-oct.html#31_October_2016_(Down's_syndrome)">In</a> <a href="https://web.archive.org/web/20170202025227/https://www.stallman.org/archives/2016-nov-feb.html#14_December_2016_(Campaign_of_bull-headed_prudery)">fact</a>, <a href="https://web.archive.org/web/20170224174306/https://www.stallman.org/archives/2016-nov-feb.html#23_February_2017_(A_violent_sex_offender)">RMS'</a> <a href="https://web.archive.org/web/20170612074722/http://stallman.org/archives/2017-mar-jun.html#26_May_2017_(Prudish_ignorantism)">views</a> <a href="https://web.archive.org/web/20170616044924/https://www.stallman.org/archives/2017-mar-jun.html#13_June_2017_(Sex_offender_registry)">and</a> <a href="https://web.archive.org/web/20171020041022/http://stallman.org/archives/2017-jul-oct.html#10_October_2017_(Laws_against_having_sex_with_an_animal)">statements</a> <a href="https://web.archive.org/web/20180131020215/https://stallman.org/archives/2017-jul-oct.html#29_October_2017_(Pestering_women)">posted</a> <a href="https://web.archive.org/web/20180104112431/https://www.stallman.org/archives/2017-nov-feb.html#27_November_2017_(Roy_Moore's_relationships)">on</a> <a href="https://web.archive.org/web/20180509120046/https://stallman.org/archives/2018-mar-jun.html#30_April_2018_(UN_peacekeepers_in_South_Sudan)">stallman.org</a> <a href="https://web.archive.org/web/20180911075211/https://www.stallman.org/archives/2018-jul-oct.html#17_July_2018_(The_bullshitter's_flirting)">about</a> <a href="https://web.archive.org/web/20180911075211/https://www.stallman.org/archives/2018-jul-oct.html#21_August_2018_(Age_and_attraction)">sexual</a> <a href="https://web.archive.org/web/20180924231708/https://stallman.org/archives/2018-jul-oct.html#23_September_2018_(Cody_Wilson)">morality</a> <a href="https://web.archive.org/web/20180919100154/https://stallman.org/antiglossary.html#assult">escalated</a> <a href="https://web.archive.org/web/20181113161736/https://www.stallman.org/archives/2018-sep-dec.html#6_November_2018_(Sex_according_to_porn)">for</a> <a href="https://web.archive.org/web/20190325024048/https://stallman.org/archives/2019-jan-apr.html#14_February_2019_(Respecting_peoples_right_to_say_no)">the</a> <a href="https://www.stallman.org/archives/2019-may-aug.html#11_June_2019_(Stretching_meaning_of_terms)">worse</a> <a href="https://web.archive.org/web/20190801201704/https://stallman.org/archives/2019-may-aug.html#12_June_2019_(Declining_sex_rates)">over</a> <a href="https://web.archive.org/web/20190801201704/https://stallman.org/archives/2019-may-aug.html#30_July_2019_(Al_Franken)">the</a> <a href="https://web.archive.org/web/20190903050208/https://stallman.org/archives/2019-jul-oct.html#27_August_2019_(Me-too_frenzy)">last</a> <a href="https://web.archive.org/web/20191011023557/https://stallman.org/archives/2019-jul-oct.html#21_September_2019_(Sex_workers)">few</a> <a href="https://web.archive.org/web/20180924231708/https://stallman.org/archives/2018-jul-oct.html#23_September_2018_(Cody_Wilson)">years</a>. When the escalation started, I still considered RMS both a friend and colleague, and I attempted to argue with him at length to convince him that some of his positions were harmful to sexual assault survivors and those who are sex trafficked, and to the people who devote their lives in service to such individuals. More importantly to the FSF, I attempted to persuade RMS that launching a controversial campaign on sexual behavior and morality was counter to his and FSF's mission to advance software freedom, and told RMS that my duty as an FSF Director was to assure the best outcome for the FSF, which <acronym title="in my opinion">IMO</acronym> didn't include having a leader who made such statements. Not only is human sexual behavior not a topic on which RMS has adequate academic expertise, but also his positions appear to ignore significant research and widely available information on the subject. Many of his comments, while occasionally politically intriguing, lack empathy for people who experienced trauma.
+                    </p>
+                    <p>
+                        IMO, this is not and has never been a Free Speech issue. I do believe freedom of speech links directly to software freedom: indeed, I see the freedom to publish software under Free licenses as almost a corollary to the freedom of speech. However, we do not need to follow leadership from those whose views we fundamentally disagree. Moreover, organizations need not and should not elevate spokespeople and leaders who speak regularly on unrelated issues that organizations find do not advance their mission, and/or that alienate important constituents. I, like many other software freedom leaders, curtail my public comments on issues not related to <acronym title="Free and Open Source Software">FOSS</acronym>. (Indeed, I would not even be commenting on <em>this issue</em> if it had not become a central issue of concern to the software freedom community.) Leaders have power, and they must exercise the power of their words with <a href="https://lwn.net/Articles/770966/">restraint, not with impunity</a>.
+                    </p>
+                    <p>
+                        RMS has consistently argued that there was a campaign of “prudish intimidation” — seeking to keep him quiet about his views on sexuality. After years of conversing with RMS about how his non-software-freedom views were a distraction, an indulgence, and downright problematic, his general response was to make even more public comments of this nature. The issue is not about RMS' right to say what he believes, nor is it even about whether or not you agree or disagree with RMS' statements. The question is whether an organization should have a designated leader who is on a sustained, public campaign advocating about an unrelated issue that many consider controversial. It really doesn't matter what your view about the controversial issue is; a leader who refuses to stop talking loudly about unrelated issues eventually creates an untenable distraction from the radical activism you're actively trying to advance. The message of universal software freedom is a radical cause; it's basically impossible for one individual to effectively push forward two unrelated controversial agendas at once. In short, the radical message of software freedom became overshadowed by RMS' radical views about sexual morality.
+                    </p>
+                    <p>
+                        And here is where I say the thing that may infuriate many but it's what I believe: I think RMS took a useful step by resigning some of his leadership roles at the FSF. I thank RMS for taking that step, and I wish the FSF Directors well in their efforts to assure that the FSF becomes a welcoming organization to all who care about universal software freedom. The <a href="https://www.fsf.org/about/">FSF's mission</a> is essential to our technological future, and we should all support that mission. I care deeply about that mission myself and have worked and will continue to work in our community in the best interest of the mission.
+                    </p>
+                    <p>
+                        I'm admittedly struggling to find a way to work again with RMS, given his views on sexual morality and his behaviors stemming from those views. I explicitly do not agree with <a href="https://web.archive.org/web/20180919100154/https://stallman.org/antiglossary.html#assult">this “(re-)definition” of sexual assault</a>. Furthermore, I believe uninformed statements about sexual assault are irresponsible and cause harm to victims. #MeToo is <strong><a href="https://web.archive.org/web/20190903050208/https://stallman.org/archives/2019-jul-oct.html#27_August_2019_(Me-too_frenzy)">not a “frenzy”</a></strong>; it is a global movement by individuals who have been harmed seeking to hold both bad actors <em>and</em> society-at-large accountable for ignoring systemic wrongs. Nevertheless, I still am proud of the <a href="https://www.gnu.org/philosophy/freedom-or-power.en.html">essay that I co-wrote with RMS</a> and still find <a href="https://www.gnu.org/gnu/manifesto.en.html">many</a> <a href="https://www.gnu.org/philosophy/free-sw.html">of</a> <a href="https://www.gnu.org/philosophy/why-free.html">RMS'</a> <a href="https://www.gnu.org/philosophy/pragmatic.html">other</a> <a href="https://www.gnu.org/philosophy/microsoft-old.html">essays</a> <a href="https://www.gnu.org/philosophy/gpl-american-way.html">compelling</a>, <a href="https://www.gnu.org/licenses/why-not-lgpl.html">important</a>, <a href="https://www.gnu.org/philosophy/stallman-kth.en.html">and</a> <a href="https://www.gnu.org/philosophy/who-does-that-server-really-serve.en.html">relevant</a>.
+                    </p>
+                    <p>
+                        I want the FSF to succeed in its mission and enter a new era of accomplishments. I've spent the last 22 years, without a break, dedicating substantial time, effort, care and loyalty to the various FSF roles that I've had: including employee, volunteer, at-large Director, and Voting Member. Even though my duties to the FSF are done, and my relationship with the FSF is no longer formal, I still think the FSF is a valuable institution worth helping and saving, specifically because the FSF was founded for a mission that I deeply support. And we should also realize that RMS — a human being (who is flawed like the rest of us) — invented that mission.
+                    </p>
+                    <p>
+                        As culture change becomes more rapid, I hope we can find reasonable nuance and moderation on our complex analysis about people and their disparate views, while we also hold individuals fully accountable for their actions. That's the difficulty we face in the post-post-modern culture of the early twenty-first century. Most importantly, I believe we must find a way to stand firm for software freedom while also making a safe environment for victims of sexual assault, sexual abuse, gaslighting, and other deplorable actions.
+                    </p>
+                    <p class="bottomAttributionWithDate">
+                        Posted on Tuesday 15 October 2019 at 09:11 by Bradley M. Kuhn.
+                    </p>
+                    <p class="comments">
+                        Submit comments on this post to <a href="mailto:bkuhn@ebb.org">&lt;bkuhn@ebb.org&gt;</a>.
+                    </p>
+                    <div id="prevlink">
+                        <a class="previous" href="/bkuhn/blog/2019/05/23/github-sponsors.html">← <strong>Previous</strong>: Chasing Quick Fixes To Sustainability</a>
+                    </div>
+                </div>
+            </div>
+            <p id="footer"></p>
+            <hr />
+            <br />
+            <p>
+                <a href="http://creativecommons.org/licenses/by-sa/3.0/us/" rel="license"><img alt="Creative Commons License" src="http://i.creativecommons.org/l/by-sa/3.0/us/88x31.png" style="border-width:10" /></a> This website and all documents on it are licensed under a <a href="http://creativecommons.org/licenses/by-sa/3.0/us/" rel="license">Creative Commons Attribution-Share Alike 3.0 United States License</a> .
+            </p>
+            <hr />
+            <p>
+                <code>#include &lt;std/disclaimer.h&gt;</code><br />
+                <code>use Standard::Disclaimer;</code><br />
+                <code>from standard import disclaimer</code><br />
+                <code>SELECT full_text FROM standard WHERE type = 'disclaimer';</code>
+            </p>
+            <p>
+                Both previously and presently, I have been employed by and/or done work for various organizations that also have views on Free, Libre, and Open Source Software. As should be blatantly obvious, this is my website, not theirs, so please do not assume views and opinions here belong to any such organization. Since I do co-own ebb.org with my wife, it may not be so obvious that these aren't her views and opinions, either.
+            </p>
+            <p align="right">
+                — bkuhn
+            </p>
+            <hr />
+            <p>
+                ebb <sup>℠</sup> is a service mark of Bradley M. Kuhn.
+            </p>
+            <address>
+                <a href="http://ebb.org/bkuhn/">Bradley M. Kuhn</a> <a href="mailto:bkuhn@ebb.org">&lt;bkuhn@ebb.org&gt;</a>
+            </address>
+        </div>
+    </body>
+</html>

article_scraper/resources/tests/readability/ehow-1/expected.html

@@ -0,0 +1,98 @@
+<article><div id="readability-page-1">
+            
+            <div>
+                    <p>Glass cloche terrariums are not only appealing to the eye, but they also preserve a bit of nature in your home and serve as a simple, yet beautiful, piece of art. Closed terrariums are easy to care for, as they retain much of their own moisture and provide a warm environment with a consistent level of humidity. You won’t have to water the terrariums unless you see that the walls are not misting up. Small growing plants that don’t require a lot of light work best such as succulents, ferns, moss, even orchids.</p>
+                    <figure> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg" alt="Glass cloche terrariums" title="Glass cloche terrariums" data-credit="Lucy Akins " longdesc="http://s3.amazonaws.com/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg"/> </figure>
+                    <figcaption> Glass cloche terrariums (Lucy Akins) </figcaption>
+                </div>
+            <div id="relatedContentUpper" data-module="rcp_top">
+                <header>
+                    <h3>Other People Are Reading</h3> </header>
+                
+            </div>
+            <div> <p><span>What You'll Need:</span></p><ul>
+                            <li>Cloche</li>
+                            <li>Planter saucer, small shallow dish or desired platform</li>
+                            <li>Floral foam oasis</li>
+                            <li>Ruler </li>
+                            <li>Spoon</li>
+                            <li>Floral wire pins or paper clips</li>
+                            <li>Small plants (from a florist or nursery)</li>
+                            <li>Moss</li>
+                            <li>Tweezers</li>
+                            <li>Other small decorative items (optional)</li>
+                        </ul>
+                    </div>
+            <div>
+                    <div> <p><span>Step 1</span></p><p>Measure the circumference of your cloche and cut the foam oasis about 3/4 inch (2 cm) smaller. Place the foam oasis into a container full of water and allow to soak until it sinks to the bottom. Dig out a hole on the oasis large enough to fit your plant, being careful not to pierce all the way through to the bottom.</p>
+                    </div>
+                    <figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/fc249ef6-4d27-41b4-8c21-15f7a8512b50.jpg" alt="Dig a hole in the oasis." data-credit="Lucy Akins"/> </figure>
+                    <figcaption> Dig a hole in the oasis. (Lucy Akins) </figcaption>
+                </div>
+            
+            
+            <div>
+                    <div> <p><span>Step 2</span></p><p>Insert your plant into the hole.</p>
+                    </div>
+                    <figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/aae11d4d-a4aa-4251-a4d9-41023ebf6d84.jpg" alt="Orchid in foam oasis" data-credit="Lucy Akins"/> </figure>
+                    <figcaption> Orchid in foam oasis (Lucy Akins) </figcaption>
+                </div>
+            <div>
+                    <div> <p><span>Step 3</span></p><p>You can add various plants if you wish.</p>
+                    </div>
+                    <figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/7afdfa1e-da74-44b5-b89c-ca8123516272.jpg" alt="Various foliage" data-credit="Lucy Akins"/> </figure>
+                    <figcaption> Various foliage (Lucy Akins) </figcaption>
+                </div>
+            <div>
+                    <div> <p><span>Step 4</span></p><p>Using floral pins, attach enough moss around the oasis to cover it.</p>
+                    </div>
+                    <figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/4f6612c0-316a-4c74-bb03-cb4e778f6d72.jpg" alt="Attach moss." data-credit="Lucy Akins"/> </figure>
+                    <figcaption> Attach moss. (Lucy Akins) </figcaption>
+                </div>
+            <div>
+                    <div> <p><span>Step 5</span></p><p>Gently place the cloche over the oasis. The glass may push some of the moss upward, exposing some of the foam.</p>
+                    </div>
+                    <figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/eeb1e0b4-e573-40a3-8db1-2c76f0b13b84.jpg" alt="Place cloche over oasis." data-credit="Lucy Akins"/> </figure>
+                    <figcaption> Place cloche over oasis. (Lucy Akins) </figcaption>
+                </div>
+            <div>
+                    <div> <p><span>Step 6</span></p><p>Simply pull down the moss with tweezers or insert more moss to fill in the empty spaces.</p>
+                    </div>
+                    <figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/812d4649-4152-4363-97c0-f181d02e709a.jpg" alt="Rearrange moss." data-credit="Lucy Akins"/> </figure>
+                    <figcaption> Rearrange moss. (Lucy Akins) </figcaption>
+                </div>
+            <div>
+                    <div> <p><span>Step 7</span></p><p>You can use any platform you wish. In this case, a small saucer was used.</p>
+                    </div>
+                    <figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/0cb3988c-9318-47d6-bc9c-c798da1ede72.jpg" alt="Place cloche on a platform to sit on." data-credit="Lucy Akins"/> </figure>
+                    <figcaption> Place cloche on a platform to sit on. (Lucy Akins) </figcaption>
+                </div>
+            <div>
+                    <div> <p><span>Step 8</span></p><p>This particular terrarium rests on a planter saucer and features a small white pumpkin.</p>
+                    </div>
+                    <figure> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/e3e18f0b-ab2c-4ffb-9988-a1ea63faef8b.jpg" alt="Cloche placed on a terracotta saucer" data-credit="Lucy Akins"/> </figure>
+                    <figcaption> Cloche placed on a terracotta saucer (Lucy Akins) </figcaption>
+                </div>
+            <div>
+                    <div> <p><span>Step 9</span></p><p>This particular terrarium was placed on a wood slice and a little toy squirrel was placed inside to add a little whimsy.</p>
+                    </div>
+                    <figure> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/2cd79f8d-0d16-4573-8861-e47fb74b0638.jpg" alt="Placed on a wooden slice" data-credit="Lucy Akins"/> </figure>
+                    <figcaption> Placed on a wooden slice (Lucy Akins) </figcaption>
+                </div>
+            <div>
+                    <div> <p><span>Finished Terrarium</span></p><p>Displayed alone or in a group, these pretty arrangements allow you to add a little nature to your decor or tablescape.</p>
+                    </div>
+                    <figure> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/78670312-8636-4c42-a75c-3029f7aa6c73.jpg" alt="Cloche terrarium" data-credit="Lucy Akins"/> </figure>
+                    <figcaption> Cloche terrarium (Lucy Akins) </figcaption>
+                </div>
+            
+            
+            
+            
+            
+            
+            <section id="FeaturedTombstone" data-module="rcp_tombstone">
+                <h2>Featured</h2>
+                
+            </section>
+        </div></article>

article_scraper/resources/tests/readability/ehow-1/source.html

@@ -0,0 +1,934 @@
+<!DOCTYPE html>
+<!--[if IE]><![endif]-->
+<html class="Crafts en-US" lang="en-US" xmlns:fb="http://www.facebook.com/2008/fbml" xmlns:og="http://ogp.me/ns#" data-channel="crafts" itemscope="" itemtype="http://schema.org/Article">
+
+<head>
+    <meta charset="utf-8"/>
+    <meta name="description" content="How to Build a Terrarium. Glass cloche terrariums are not only appealing to the eye, but they also preserve a bit of nature in your home and serve as a simple, yet beautiful, piece of art. Closed terrariums are easy to care for, as they retain much of their own moisture and provide a warm environment with a consistent level of humidity. You..."/>
+    <meta name="format-detection" content="telephone=no"/>
+    <meta name="y_key" content="4d4cb4ef1ad334cb"/>
+    <meta name="alexaVerifyID" content="xlJch1oUHrXNtOQBpiuUCV0-rlc"/>
+    <meta name="cachetime" content="201504281431"/>
+    <meta name="p:domain_verify" content="4d1387210be25228f0c4692cc47db05b"/>
+    <meta name="cme_last_updated" content="2015-04-28T11:54:11.401000"/>
+    <meta name="publishdate" content="05/29/2007 09:49:00"/>
+    <meta name="last_rcp_import" content="2015-04-27T19:34:40.179876"/>
+    <meta name="articleimage" content="http://s3.amazonaws.com/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg"/>
+    <link href="http://www.ehow.com/how_2042752_build-terrarium.html" rel="canonical"/>
+    <link type="application/rss+xml" href="http://www.ehow.com/feed/ehow-crafts.rss" rel="alternate" title="Subscribe to our RSS Feed"/>
+    <meta name="studioid" scheme="DMINSTR2" content="f941e4cd-a49e-4948-b244-677c15ff8b7f"/>
+    <meta name="contentid" scheme="DMINSTR2" content="2042752"/>
+    <meta name="category" scheme="DMINSTR2" content="Crafts"/>
+    <meta name="subcategory" scheme="DMINSTR2" content="Other DIY Crafts"/>
+    <meta name="subsubcat" scheme="DMINSTR2" content="Other DIY Projects"/>
+    <meta name="subpagetype" scheme="DMINSTR2" content="topic_view"/>
+    <meta name="exp_page" scheme="DMINSTR2" content=""/>
+    <meta name="wa_pchn" scheme="DMINSTR2" content="Crafts"/>
+    <meta name="pagetype" scheme="DMINSTR2" content="inline"/>
+    <meta name="wa_lr" scheme="DMINSTR2" content="en-US"/>
+    <meta name="siteid" scheme="DMINSTR2" content="EHWC"/>
+    <meta name="exp_name" scheme="DMINSTR2" content="desktop:default"/>
+    <meta name="wa_un" scheme="DMINSTR2" content=""/>
+    <meta name="wa_isreg" scheme="DMINSTR2" content=""/>
+    <meta name="wa_lgdin" scheme="DMINSTR2" content=""/>
+    <meta name="wa_clvl" scheme="DMINSTR2" content="1"/>
+    <meta name="wa_lgsrc" scheme="DMINSTR2" content="1"/>
+    <meta name="author" scheme="DMINSTR2" content="Lucy Akins"/>
+    <meta name="author_image" content=""/>
+    <meta name="twitter:card" content="summary"/>
+    <meta name="twitter:site" content="@ehow"/>
+    <meta property="og:image" content="http://img-aws.ehowcdn.com/200x200/cme/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg"/>
+    <meta property="og:url" content="http://www.ehow.com/how_2042752_build-terrarium.html"/>
+    <meta property="og:type" content="article"/>
+    <meta property="og:title" content="How to Build a Terrarium | eHow"/>
+    <meta property="og:description" content="Glass cloche terrariums are not only appealing to the eye, but they also preserve a bit of nature in your home and serve as a simple, yet beautiful, piece of art. Closed terrariums are easy to care for, as they retain much of their own moisture and provide a warm environment with a consistent level of humidity. You won’t have to water the..."/>
+    <meta property="fb:app_id" content="63203377906"/>
+    <meta property="fb:ttl" content="604800"/>
+    <meta property="og:site_name" content="eHow"/>
+    <title>How to Build a Terrarium (with Pictures) | eHow</title>
+    <link type="image/x-icon" href="http://v5-static.ehowcdn.com/favicon.ico" rel="shortcut icon apple-touch-icon"/>
+    <link rel="stylesheet" type="text/css" href="http://dynamic02.ehowcdn.com/services/modules/css/common_header,corporate-skin,inline/fa923205/"/>
+    <!--[if lt IE 9]><link href="http://v5-static.ehowcdn.com/ui/stylesheets/layout/ie.css" rel="stylesheet" /><script src="http://v5-static.ehowcdn.com/ui/scripts/ie_hacks/header.js"></script><![endif]-->
+    <!--[if IE 9]><link href="http://v5-static.ehowcdn.com/ui/stylesheets/layout/ie9.css" rel="stylesheet" /><![endif]-->
+    <script>
+        window._omnitureLinkSetDomain = 'extended.dmtracker.com';
+    </script>
+    <script>
+        var dmjs = dmjs || {};
+        dmjs.global = dmjs.global || {};
+        dmjs.global.glbBaseUrl = "http:\/\/www.ehow.com";
+        dmjs.global.facebook = {
+            "app_id": "63203377906",
+            "api_version": "v2.0",
+            "login_service": "http:\/\/www.ehow.com\/services\/fb\/login\/",
+            "comment_service": "http:\/\/www.ehow.com\/services\/comment\/refresh\/"
+        };
+        dmjs.urls = {
+            "cdn": "http:\/\/v5-static.ehowcdn.com",
+            "sitelife": "http:\/\/sitelife.ehow.com\/ver1.0\/",
+            "dmtracker": "vs.dmtracker.com"
+        };
+        dmjs.global.IN_PRODUCTION = true;
+        dmjs.global.darkhorse = true;
+        dmjs.global.pool = "default";
+        dmjs.global.channel = "crafts";
+        dmjs.global.skin = "crafts";
+        dmjs.global.facebook.xid = "aG93XzIwNDI3NTJfYnVpbGQtdGVycmFyaXVtLmh0bWw";
+        dmjs.global.facebook.associate_on_comment = false;
+        dmjs.global._adChannel = "craft_other_diy,text,suggested7";
+        dmjs.global.IsSlideStep = false;
+        dmjs.global.fixed_category = [{
+            "id": 5352,
+            "title": "Crafts",
+            "url": "\/crafts\/"
+        }, {
+            "id": 5426,
+            "title": "Other DIY Crafts",
+            "url": "\/diy-crafts\/"
+        }, {
+            "id": 5429,
+            "title": "Other DIY Projects",
+            "url": "\/diy-projects\/"
+        }];
+        dmjs.global.SlideShowTest = {
+            "_id": "www.ehow.com\/how_16357_make-paper-sunflowers.html",
+            "dimensions": "none",
+            "image": {
+                "url": "http:\/\/s3.amazonaws.com\/uploadedimages.demandmedia\/sunflowers-1.jpg"
+            },
+            "subtitle": "",
+            "title": "How to Make Paper Sunflowers",
+            "type": "Article",
+            "url": "\/how_16357_make-paper-sunflowers.html"
+        };
+        dmjs.global.contributor = "www.demandstudios.com\/contributor\/3959b24c-24bb-447f-a676-55ce0b6e5519";
+        dmjs.global.articleType = "Article";
+        dmjs.global.articleID = 2042752;
+        /* Add xdomain first event helper to be added before ad event listeners are added */
+        dmjs.xdomainFirstEventListener = true;
+        if (window.addEventListener) {
+            window.addEventListener("message", function(event) {
+                if (typeof xdomain != 'undefined' && typeof xdomain.dmjsFirstEventListener != 'undefined') {
+                    xdomain.dmjsFirstEventListener(event);
+                }
+            }, false);
+        } else {
+            window.attachEvent("message", function(event) {
+                if (typeof xdomain != 'undefined' && typeof xdomain.dmjsFirstEventListener != 'undefined') {
+                    xdomain.dmjsFirstEventListener(event);
+                }
+            });
+        }
+    </script>
+    <!--[if lt IE 9]>
+<script src="http://v5-static.ehowcdn.com/content/compressed/common_header-646656f9.js"  ></script>
+<![endif]-->
+    <script src="http://dynamic01.ehowcdn.com/services/modules/js/gpt,common_header_slim/4eb05a96/"></script>
+    <script class="kxct" data-id="JWC_M4Hq" data-timing="async" data-version="1.9" type="text/javascript">
+        window.Krux || ((Krux = function() {
+            Krux.q.push(arguments)
+        }).q = []);
+        (function() {
+            var k = document.createElement('script');
+            k.type = 'text/javascript';
+            k.async = true;
+            var m, src = (m = location.href.match(/\bkxsrc=([^&]+)/)) && decodeURIComponent(m[1]);
+            k.src = /^https?:\/\/([a-z0-9_\-\.]+\.)?krxd\.net(:\d{1,5})?\//i.test(src) ? src : src === "disable" ? "" : (location.protocol === "https:" ? "https:" : "http:") + "//cdn.krxd.net/controltag?confid=JWC_M4Hq";
+            var s = document.getElementsByTagName('script')[0];
+            s.parentNode.insertBefore(k, s);
+        }());
+    </script>
+    <script>
+        window.Krux || ((Krux = function() {
+            Krux.q.push(arguments);
+        }).q = []);
+        (function() {
+            function retrieve(n) {
+                var m, k = 'kx' + n;
+                if (window.localStorage) {
+                    return window.localStorage[k] || "";
+                } else if (navigator.cookieEnabled) {
+                    m = document.cookie.match(k + '=([^;]*)');
+                    return (m && unescape(m[1])) || "";
+                } else {
+                    return '';
+                }
+            }
+            Krux.user = retrieve('user');
+            Krux.segments = retrieve('segs') && retrieve('segs').split(',') || [];
+        })();
+    </script>
+    <script type="text/javascript">
+        var GPT = [];
+        googletag.cmd.push(function() {
+            window.slot_define_string = "6117/dmd.ehow/crafts/otherdiycrafts/otherdiyprojects";
+            GPT['300x250_300x600'] = googletag.defineSlot('/6117/dmd.ehow/crafts/otherdiycrafts/otherdiyprojects', [
+                [300, 250],
+                [300, 600]
+            ], '300x250_300x600').addService(googletag.pubads());
+            GPT['728x90_990x90_970x250'] = googletag.defineSlot('/6117/dmd.ehow/crafts/otherdiycrafts/otherdiyprojects', [
+                [728, 90],
+                [990, 90],
+                [970, 250]
+            ], '728x90_990x90_970x250').addService(googletag.pubads());
+            GPT['outofpage'] = googletag.defineOutOfPageSlot('/6117/dmd.ehow/crafts/otherdiycrafts/otherdiyprojects', 'outofpage').addService(googletag.pubads()).setCollapseEmptyDiv(true, true);
+            googletag.pubads().setTargeting('cat', 'crafts');
+            googletag.pubads().setTargeting('scat', 'otherdiycrafts');
+            googletag.pubads().setTargeting('sscat', 'otherdiyprojects');
+            googletag.pubads().setTargeting('x', '7');
+            googletag.pubads().setTargeting('art', '2042752');
+            googletag.pubads().setTargeting('ctype', 'step_articles');
+            googletag.pubads().setTargeting('vid', '0');
+            googletag.pubads().setTargeting('blog', '0');
+            googletag.pubads().setTargeting('ad_chan', 'craft_other_diy');
+            window.gpt_targeting_vars = {
+                "cat": "crafts",
+                "scat": "otherdiycrafts",
+                "sscat": "otherdiyprojects",
+                "x": "7",
+                "art": 2042752,
+                "ctype": "step_articles",
+                "vid": 0,
+                "blog": 0,
+                "ad_chan": "craft_other_diy"
+            };
+            googletag.pubads().setTargeting('dc_ref', top.window.location.href.replace('#', '-'));
+            googletag.pubads().setTargeting("envr", ""); //for production. set envr to empty string
+            googletag.pubads().setTargeting("ksg", Krux.segments);
+            googletag.pubads().setTargeting("kuid", Krux.user);
+            googletag.pubads().collapseEmptyDivs();
+            googletag.pubads().enableSingleRequest();
+            googletag.enableServices();
+        });
+    </script>
+    <script>
+        dataLayer = [{
+            'content_category': 'Crafts',
+            'content_subCategory': 'Other DIY Crafts',
+            'content_subSubCategory': 'Other DIY Projects',
+            'content_pageType': 'inline',
+            'content_subPageType': 'topic_view',
+            'content_channel': 'crafts',
+            'content_author': 'Lucy Akins',
+            'content_publishDate': '05/29/2007 09:49:00',
+            'content_name': 'How to Build a Terrarium (with Pictures) | eHow',
+            'content_hasImages': 'True',
+            'content_experience': 'desktop:default',
+            'httpStatusCode': '200'
+        }];
+    </script>
+    <!--
+<PageMap>
+	<DataObject type="meta">
+		<Attribute name="articlethumbnail"><![CDATA[]]></Attribute>
+		<Attribute name="articletype"><![CDATA[professional]]></Attribute>
+	</DataObject>
+</PageMap>
+-->
+</head>
+
+<body class="Crafts TopicView " data-skin="crafts">
+    <!--[if IE 7]>
+			<script src="http://v5-static.ehowcdn.com/ui/scripts/ie7_notification.js"  ></script>		<![endif]-->
+    <noscript>
+        <iframe src="//www.googletagmanager.com/ns.html?id=GTM-K7RB82" height="0" width="0" style="display:none;visibility:hidden"></iframe>
+    </noscript>
+    <script>
+        (function(w, d, s, l, i) {
+            w[l] = w[l] || [];
+            w[l].push({
+                'gtm.start': new Date().getTime(),
+                event: 'gtm.js'
+            });
+            var f = d.getElementsByTagName(s)[0],
+                j = d.createElement(s),
+                dl = l != 'dataLayer' ? '&l=' + l : '';
+            j.async = true;
+            j.src = '//www.googletagmanager.com/gtm.js?id=' + i + dl;
+            f.parentNode.insertBefore(j, f);
+        })(window, document, 'script', 'dataLayer', 'GTM-K7RB82');
+    </script>
+    <header id="Header" data-section="header">
+        <section class="topContainer inner-container page-header clearfix">
+            <a href="http://www.ehow.com/" class="image corp-logo fl"><img src="http://v5-static.ehowcdn.com/media/images/logos/logo-stamp.png" width="134" height="38" alt="eHow Logo"/></a>
+            <form method="get" action="http://www.ehow.com/search.html" id="searchHeader" class="Search ">
+                <input type="text" name="s" data-type="searchinput" class="input fl headline4"/>
+                <input name="skin" type="hidden" value="corporate"/>
+                <input name="t" type="hidden" value="all"/>
+                <input type="submit" value="Search" class="bttn submit headline5 head"/> </form>
+        </section>
+        <nav id="primaryNav" role="navigation">
+            <ul class="inner-container channelListing clearfix skin-crafts">
+                <li class="list js-list"><a href="http://www.ehow.com/ehow-mom/" class="mom head title channel gtm_headerNavmom" data-channel="mom">mom</a></li>
+                <li class="list js-list"><a href="http://www.ehow.com/ehow-style/" class="style head title channel gtm_headerNavstyle" data-channel="style">style</a></li>
+                <li class="list js-list"><a href="http://www.ehow.com/ehow-food/" class="food head title channel gtm_headerNavfood" data-channel="food">food</a></li>
+                <li class="list js-list"><a href="http://www.ehow.com/ehow-tech/" class="tech head title channel gtm_headerNavtech" data-channel="tech">tech</a></li>
+                <li class="list js-list"><a href="http://www.ehow.com/ehow-home/" class="home head title channel gtm_headerNavhome" data-channel="home">home</a></li>
+                <li class="list js-list"><a href="http://www.ehow.com/ehow-money/" class="money head title channel gtm_headerNavmoney" data-channel="money">money</a></li>
+                <li class="list js-list"><a href="http://www.ehow.com/ehow-crafts/" class="crafts head title channel gtm_headerNavcrafts" data-channel="crafts">crafts</a></li>
+                <li class="list js-list more"><a class="head title more js-more" data-channel="more" href="#">More<span class="js-icon more-icon elegant-icons">3</span></a>
+                    <div class="more_dropdown">
+                        <div class="moreLinks">
+                            <div class="headline2 head mg-4">Browse Articles &amp; Videos By Category</div>
+                            <ul class="list list-0">
+                                <li class="item"><a href="http://www.ehow.com/arts/" class="headline5 title target">Arts &amp; Entertainment</a></li>
+                                <li class="item"><a href="http://www.ehow.com/business/" class="headline5 title target">Business</a></li>
+                                <li class="item"><a href="http://www.ehow.com/careers/" class="headline5 title target">Careers &amp; Work</a></li>
+                                <li class="item"><a href="http://www.ehow.com/cars/" class="headline5 title target">Cars</a></li>
+                                <li class="item"><a href="http://www.ehow.com/computers/" class="headline5 title target">Computers</a></li>
+                                <li class="item"><a href="http://www.ehow.com/crafts/" class="headline5 title target">Crafts</a></li>
+                                <li class="item"><a href="http://www.ehow.com/cultures/" class="headline5 title target">Culture &amp; Society</a></li>
+                            </ul>
+                            <ul class="list list-1">
+                                <li class="item"><a href="http://www.ehow.com/education/" class="headline5 title target">Education</a></li>
+                                <li class="item"><a href="http://www.ehow.com/electronics/" class="headline5 title target">Electronics</a></li>
+                                <li class="item"><a href="http://www.ehow.com/fashion/" class="headline5 title target">Fashion, Style &amp; Personal Care</a></li>
+                                <li class="item"><a href="http://www.ehow.com/food/" class="headline5 title target">Food &amp; Drink</a></li>
+                                <li class="item"><a href="http://www.ehow.com/health/" class="headline5 title target">Health</a></li>
+                                <li class="item"><a href="http://www.ehow.com/hobbies-games/" class="headline5 title target">Hobbies, Games &amp; Toys</a></li>
+                                <li class="item"><a href="http://www.ehow.com/holidays/" class="headline5 title target">Holidays &amp; Celebrations</a></li>
+                            </ul>
+                            <ul class="list list-2">
+                                <li class="item"><a href="http://www.ehow.com/home/" class="headline5 title target">Home &amp; Garden</a></li>
+                                <li class="item"><a href="http://www.ehow.com/internet/" class="headline5 title target">Internet</a></li>
+                                <li class="item"><a href="http://www.ehow.com/legal/" class="headline5 title target">Legal</a></li>
+                                <li class="item"><a href="http://www.ehow.com/parenting/" class="headline5 title target">Parenting</a></li>
+                                <li class="item"><a href="http://www.ehow.com/parties/" class="headline5 title target">Parties &amp; Entertaining</a></li>
+                                <li class="item"><a href="http://www.ehow.com/personal-finance/" class="headline5 title target">Personal Finance</a></li>
+                                <li class="item"><a href="http://www.ehow.com/pets/" class="headline5 title target">Pets</a></li>
+                            </ul>
+                            <ul class="list list-3">
+                                <li class="item"><a href="http://www.ehow.com/relationships/" class="headline5 title target">Relationships &amp; Family</a></li>
+                                <li class="item"><a href="http://www.ehow.com/sports/" class="headline5 title target">Sports &amp; Fitness</a></li>
+                                <li class="item"><a href="http://www.ehow.com/travel/" class="headline5 title target">Travel</a></li>
+                                <li class="item"><a href="http://www.ehow.com/weddings/" class="headline5 title target">Weddings</a></li>
+                            </ul>
+                        </div>
+                        <div class="sponsored_programs">
+                            <ul class="unstyled">
+                                <li class="item">
+                                    <a href="http://www.ehow.com/ehow-indoor/" class="diyhome"></a>
+                                </li>
+                                <li class="item">
+                                    <a href="http://www.ehow.com/ehow-outdoor/" class="diyoutdoor"></a>
+                                </li>
+                            </ul>
+                        </div>
+                    </div>
+                </li>
+            </ul>
+        </nav>
+    </header>
+    <div class="inner-container">
+        <!-- gpt slot 728x90_990x90_970x250 -->
+        <div class="bannerAd mod" data-module="gpt-ad-728x90_990x90_970x250" id="728x90_990x90_970x250">
+            <script type="text/javascript">
+                googletag.cmd.push(function() {
+                    googletag.display('728x90_990x90_970x250');
+                });
+            </script>
+        </div>
+        <!-- gpt slot outofpage -->
+        <div class="AdUnitOutOfPage mod" data-module="gpt-ad-outofpage" id="outofpage">
+            <script type="text/javascript">
+                googletag.cmd.push(function() {
+                    googletag.display('outofpage');
+                });
+            </script>
+        </div>
+    </div>
+    <section id="Body" class="InlineTemplate FLC">
+        <script id="_ehow-com-hnh"></script>
+        <ol class="breadcrumbs unstyled inline" itemscope="" itemtype="http://data-vocabulary.org/Breadcrumb" data-module="breadcrumb">
+            <li class="list">
+                <a href="http://www.ehow.com/" rel="home" itemprop="url" class="title"> <span itemprop="title">eHow</span> </a>
+            </li>
+            <li class="list" itemprop="child" itemscope="" itemtype="http://data-vocabulary.org/Breadcrumb">
+                <a href="http://www.ehow.com/crafts/" rel="directory" itemprop="url" class="title"> <span itemprop="title">Crafts</span> </a>
+            </li>
+            <li class="list" itemprop="child" itemscope="" itemtype="http://data-vocabulary.org/Breadcrumb">
+                <a href="http://www.ehow.com/crafts/diy-crafts/" rel="directory" itemprop="url" class="title"> <span itemprop="title">Other DIY Crafts</span> </a>
+            </li>
+            <li class="list" itemprop="child" itemscope="" itemtype="http://data-vocabulary.org/Breadcrumb">
+                <a href="http://www.ehow.com/crafts/diy-crafts/diy-projects/" rel="directory" itemprop="url" class="title"> <span itemprop="title">Other DIY Projects</span> </a>
+            </li>
+            <li class="list" itemprop="child" itemscope="" itemtype="http://data-vocabulary.org/Breadcrumb">
+                <a href="http://www.ehow.com/how_2042752_build-terrarium.html" rel="directory" itemprop="url" class="title"> <span itemprop="title">How to Build a Terrarium</span> </a>
+            </li>
+        </ol>
+        <!-- gpt slot 990x50 -->
+        <div class="col-main">
+            <header class="page-head bordered">
+                <h1 class="headline1 head mg-2" itemprop="headline">
+How to Build a Terrarium	</h1>
+                <div data-type="AuthorProfile" itemprop="author" itemscope="" itemtype="http://www.schema.org/Person">
+                    <script type="text/html" data-element="template">
+                        <footer class="AuthorProfileWrap footer">
+                            <div class="AuthorProfileModal" data-element="modal">
+                                <div class="wrapper">
+                                    <div class="ModalClose close" data-action="hideModal" data-element="closeButton">X</div> <img src="http://img-aws.ehowcdn.com/100x100/studio-image/ver1.0/Content/images/store/3/11/a3407c7f-fdf9-4ae6-8bac-48007b0109ea.Medium.jpg" onerror="this.className = &#039;hide&#039;" />
+                                    <div class="headline3 head mg-2" itemprop="name"> Lucy Akins </div>
+                                    <p class="bio"> Lucy Akins is an artist/photographer/blogger living in Toronto, Ontario. Her blog, Craftberry Bush, has blossomed into a business, having gained domestic and international recognition. Her work has been published in several magazines, including: Cottages and Bungalows, American Farmlife Style, National Geographic Kids, Artful Blogger, Somerset Life, and most recently graced the cover of Somerset Home. </p>
+                                    <p class="portfolio"> <a class="profile" itemprop="url" rel="author" href="http://www.demandstudios.com/profile-3959b24c-Lucy-Akins/" target="_blank">View my portfolio</a> </p>
+                                </div>
+                            </div>
+                        </footer>
+                    </script>
+                    <div class="post-meta clearfix headline6 mg-2 ">
+                        <div class="meta">
+                            <a class="byline trigger gtm_contributorByline head-alt nb-name" rel="author" itemprop="author" href="/contributor/lucy_akins/" target="_top"> <span itemprop="name">By Lucy Akins</span> </a>
+                            <div class="nb-byline">eHow Contributor</div>
+                        </div>
+                    </div>
+                    <div class="article-meta"> </div>
+                    <div data-element="container"></div>
+                </div>
+                <div class="clearfix">
+                    <ul class="social-icons js-social-icons inline flcb" data-url="http://www.ehow.com/how_2042752_build-terrarium.html">
+                        <li class="item btnPinterest">
+                            <a class="js-btn btn gtm_socialShare" data-social="pinterest" data-url="http://www.ehow.com/how_2042752_build-terrarium.html?p=d" data-title="How to Build a Terrarium" data-media="http://s3.amazonaws.com/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg"> <span class="social-icon elegant-icons"></span> <span class="action js-pins">Pin</span> </a>
+                        </li>
+                        <li class="item btnFacebook">
+                            <a class="js-btn btn gtm_socialShare" data-social="facebook" data-url="http://www.ehow.com/how_2042752_build-terrarium.html" data-title="How to Build a Terrarium" data-media="http://s3.amazonaws.com/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg" data-description="Glass cloche terrariums are not only appealing to the eye, but they also preserve a bit of nature in your home and serve as a simple, yet beautiful, piece of art. Closed terrariums are easy to care for, as they retain much of their own moisture and provide a warm environment with a consistent level of humidity. You won’t have to water the..."> <span class="social-icon elegant-icons"></span> <span class="action js-facebook_shares">Share</span> </a>
+                        </li>
+                        <li class="item btnTwitter">
+                            <a class="js-btn btn gtm_socialShare" data-social="twitter" data-url="http://www.ehow.com/how_2042752_build-terrarium.html" data-title="How to Build a Terrarium" data-media="http://s3.amazonaws.com/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg"> <span class="social-icon elegant-icons"></span> <span class="action js-tweets">Tweet</span> </a>
+                        </li>
+                        <li class="item btnGoogle">
+                            <a class="js-btn btn gtm_socialShare" data-social="google" data-url="http://www.ehow.com/how_2042752_build-terrarium.html" data-title="How to Build a Terrarium"> <span class="social-icon elegant-icons"></span> <span class="action js-plusones">Share</span> </a>
+                        </li>
+                    </ul>
+                    <div class="pagescore js-pagescore page-heading" data-score="true" data-url="http://www.ehow.com/how_2042752_build-terrarium.html"> <span class="headline3 num js-num"></span> <span class="icon"></span> <span class="msg">Found This Helpful</span> </div>
+                </div>
+            </header>
+            <div class="mod step">
+                <div class="stepContent mod">
+                    <div class="content lead">
+                        <p>Glass cloche terrariums are not only appealing to the eye, but they also preserve a bit of nature in your home and serve as a simple, yet beautiful, piece of art. Closed terrariums are easy to care for, as they retain much of their own moisture and provide a warm environment with a consistent level of humidity. You won’t have to water the terrariums unless you see that the walls are not misting up. Small growing plants that don’t require a lot of light work best such as succulents, ferns, moss, even orchids.</p>
+                    </div>
+                    <figure class="stepThumb"> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg" alt="Glass cloche terrariums" title="Glass cloche terrariums" class="photo" data-credit="Lucy Akins " longdesc="http://s3.amazonaws.com/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg"/> </figure>
+                    <figcaption class="small caption"> Glass cloche terrariums (Lucy Akins) </figcaption>
+                </div>
+            </div>
+            <div id="relatedContentUpper" class="RelatedContent Module" data-module="rcp_top">
+                <header class="">
+                    <h3 class="head headline2 mg-2">Other People Are Reading</h3> </header>
+                <ul class="unstyled inline box box-inner people-reading mod FLC" data-type="more-relatedArticles">
+                    <li class="fl item headline5 first">
+                        <a class="gtm_otherPeopleReading" href="http://www.ehow.com/video_12263138_plant-terrarium-ideas.html"> <img src="http://img-aws.ehowcdn.com/105x70/viper/media/2b3830ca-4eb6-4bfd-a1cd-d430b3e28d59/jpeg/a7edbb0f-86a7-4c73-9c2d-f85f87b49efa_0.jpg" class="thumb fl"/> </a> <a class="title gtm_otherPeopleReading headline5" href="http://www.ehow.com/video_12263138_plant-terrarium-ideas.html">Plant Terrarium Ideas</a> </li>
+                    <li class="fl item headline5 mid">
+                        <a class="gtm_otherPeopleReading" href="http://www.ehow.com/how_4885344_build-terrarium-succulent-plants.html"> <img src="http://img-aws.ehowcdn.com/105x70/cme/photography.prod.demandstudios.com/619daaaa-7991-4fc6-9b62-dfeab8a285b4.jpg" class="thumb fl"/> </a> <a class="title gtm_otherPeopleReading headline5" href="http://www.ehow.com/how_4885344_build-terrarium-succulent-plants.html">How to Build a Terrarium With Succulent Plants</a> </li>
+                </ul>
+            </div>
+            <div class="mod step">
+                <div class="stepContent">
+                    <div class="content"> <span class="headline2 head mg-1 block">What You'll Need:</span>
+                        <ul class="markdown-ul">
+                            <li>Cloche</li>
+                            <li>Planter saucer, small shallow dish or desired platform</li>
+                            <li>Floral foam oasis</li>
+                            <li>Ruler </li>
+                            <li>Spoon</li>
+                            <li>Floral wire pins or paper clips</li>
+                            <li>Small plants (from a florist or nursery)</li>
+                            <li>Moss</li>
+                            <li>Tweezers</li>
+                            <li>Other small decorative items (optional)</li>
+                        </ul>
+                    </div>
+                </div>
+            </div>
+            <div class="mod step">
+                <div class="stepContent">
+                    <div class="content"> <span class="headline2 head mg-1 block">Step 1</span>
+                        <p>Measure the circumference of your cloche and cut the foam oasis about 3/4 inch (2 cm) smaller. Place the foam oasis into a container full of water and allow to soak until it sinks to the bottom. Dig out a hole on the oasis large enough to fit your plant, being careful not to pierce all the way through to the bottom.</p>
+                    </div>
+                    <figure class="stepThumb"> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/fc249ef6-4d27-41b4-8c21-15f7a8512b50.jpg" alt="Dig a hole in the oasis." class="photo" data-credit="Lucy Akins"/> </figure>
+                    <figcaption class="small caption"> Dig a hole in the oasis. (Lucy Akins) </figcaption>
+                </div>
+            </div>
+            <div id="m1"></div>
+            <div class="mod Inline">
+                <div id="DMINSTR" type="adimpression" data-type="adTracking" name="&amp;adunit_id=ca-ehow_300x250&amp;ad_unit_type=CNT&amp;ad_unit_network=GAP&amp;revenue_basis=CPC&amp;ad_position=-&amp;revenue_tag=craft_other_diy">
+                    <div id="GoogleAdsense300x250"></div>
+                    <script>
+                        dmjs.revenueTags.push('ca-ehow_300x250;craft_other_diy,text,suggested7');
+                        googleAds.addAdUnit({
+                            priority: 3,
+                            adUnitId: 'GoogleAdsense300x250',
+                            google_ad_channel: 'craft_other_diy,text,suggested7',
+                            google_ad_client: 'ca-ehow_300x250',
+                            google_page_url: 'http://www.ehow.com/how_2042752_build-terrarium.html',
+                            google_max_num_ads: '1',
+                            google_ad_type: 'text',
+                            templates: {
+                                text: '&lt;div class="GoogleTextAd GoogleAd300x250 ad-300"> \
+								&lt;p>&lt;a class="header" href="{!= google_info.feedback_url !}">Sponsored Links&lt;/a>&lt;/p> \
+								&lt;ul> \
+								{! for (var i = 0; i &lt; ads.length; i++) { !} \
+									&lt;li class="Ad"> \
+										&lt;a rel="nofollow" href="{!= ads[i].url !}" target="_blank" title="go to {!= clean(ads[i].visible_url) !}" class="title"> \
+											{!= ads[i].line1 !} \
+										&lt;/a> \
+										&lt;p class="copy">{!= ads[i].line2 !} {!= ads[i].line3 !}&lt;/p> \
+										&lt;a rel="nofollow" href="{!= ads[i].url !}" target="_blank" title="go to {!= clean(ads[i].visible_url) !}" class="baseurl url"> \
+											{!= ads[i].visible_url !} \
+										&lt;/a> \
+									&lt;/li> \
+								{! } !} \
+								&lt;/ul> \
+							&lt;/div>',
+                                image: '&lt;div class="GoogleImageAd GoogleImageAdNew"> \
+								&lt;a class="header" href="{!= google_info.feedback_url !}">Sponsored Links&lt;/a> \
+								&lt;a href="{!= ads[0].url !}" title="go to {!= clean(ads[0].visible_url) !}" target="_blank" class="Ad" >\
+									&lt;img src="{!= ads[0].image_url !}" width="{!= ads[0].image_width !}" height="{!= ads[0].image_height !}" border="0" alt="" />\
+								&lt;/a> \
+							&lt;/div>',
+                                flash: '&lt;div class="GoogleFlashAd GoogleFlashAdNew">\
+								&lt;a class="header" href="{!= google_info.feedback_url !}">Sponsored Links&lt;/a> \
+								&lt;div class="Ad">\
+									&lt;object classid="clsid:D27CDB6E-AE6D-11cf-96B8-444553540000" codebase="http://download.macromedia.com/pub/shockwave/cabs/flash/swflash.cab#version=6,0,0,0" width="{!= ads[0].image_width !}" height="{!= ads[0].image_height !}">&lt;param name="movie" value="{!= ads[0].image_url !}">&lt;param name="quality" value="high">&lt;param name="AllowScriptAccess" value="never">&lt;embed src="{!= ads[0].image_url !}" width="{!= ads[0].image_width !}" height="{!= ads[0].image_height !}" type="application/x-shockwave-flash" AllowScriptAccess="never" pluginspage="http://www.macromedia.com/go/getflashplayer">&lt;/embed>&lt;/object>\
+								&lt;/div>\
+							&lt;/div>'
+                            },
+                            google_image_size: '300x250'
+                        });
+                    </script>
+                </div>
+            </div>
+            <div class="mod step">
+                <div class="stepContent">
+                    <div class="content"> <span class="headline2 head mg-1 block">Step 2</span>
+                        <p>Insert your plant into the hole.</p>
+                    </div>
+                    <figure class="stepThumb"> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/aae11d4d-a4aa-4251-a4d9-41023ebf6d84.jpg" alt="Orchid in foam oasis" class="photo" data-credit="Lucy Akins"/> </figure>
+                    <figcaption class="small caption"> Orchid in foam oasis (Lucy Akins) </figcaption>
+                </div>
+            </div>
+            <div class="mod step">
+                <div class="stepContent">
+                    <div class="content"> <span class="headline2 head mg-1 block">Step 3</span>
+                        <p>You can add various plants if you wish.</p>
+                    </div>
+                    <figure class="stepThumb"> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/7afdfa1e-da74-44b5-b89c-ca8123516272.jpg" alt="Various foliage" class="photo" data-credit="Lucy Akins"/> </figure>
+                    <figcaption class="small caption"> Various foliage (Lucy Akins) </figcaption>
+                </div>
+            </div>
+            <div class="mod step">
+                <div class="stepContent">
+                    <div class="content"> <span class="headline2 head mg-1 block">Step 4</span>
+                        <p>Using floral pins, attach enough moss around the oasis to cover it.</p>
+                    </div>
+                    <figure class="stepThumb"> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/4f6612c0-316a-4c74-bb03-cb4e778f6d72.jpg" alt="Attach moss." class="photo" data-credit="Lucy Akins"/> </figure>
+                    <figcaption class="small caption"> Attach moss. (Lucy Akins) </figcaption>
+                </div>
+            </div>
+            <div class="mod step">
+                <div class="stepContent">
+                    <div class="content"> <span class="headline2 head mg-1 block">Step 5</span>
+                        <p>Gently place the cloche over the oasis. The glass may push some of the moss upward, exposing some of the foam.</p>
+                    </div>
+                    <figure class="stepThumb"> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/eeb1e0b4-e573-40a3-8db1-2c76f0b13b84.jpg" alt="Place cloche over oasis." class="photo" data-credit="Lucy Akins"/> </figure>
+                    <figcaption class="small caption"> Place cloche over oasis. (Lucy Akins) </figcaption>
+                </div>
+            </div>
+            <div class="mod step">
+                <div class="stepContent">
+                    <div class="content"> <span class="headline2 head mg-1 block">Step 6</span>
+                        <p>Simply pull down the moss with tweezers or insert more moss to fill in the empty spaces.</p>
+                    </div>
+                    <figure class="stepThumb"> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/812d4649-4152-4363-97c0-f181d02e709a.jpg" alt="Rearrange moss." class="photo" data-credit="Lucy Akins"/> </figure>
+                    <figcaption class="small caption"> Rearrange moss. (Lucy Akins) </figcaption>
+                </div>
+            </div>
+            <div class="mod step">
+                <div class="stepContent">
+                    <div class="content"> <span class="headline2 head mg-1 block">Step 7</span>
+                        <p>You can use any platform you wish. In this case, a small saucer was used.</p>
+                    </div>
+                    <figure class="stepThumb"> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/0cb3988c-9318-47d6-bc9c-c798da1ede72.jpg" alt="Place cloche on a platform to sit on." class="photo" data-credit="Lucy Akins"/> </figure>
+                    <figcaption class="small caption"> Place cloche on a platform to sit on. (Lucy Akins) </figcaption>
+                </div>
+            </div>
+            <div class="mod step">
+                <div class="stepContent">
+                    <div class="content"> <span class="headline2 head mg-1 block">Step 8</span>
+                        <p>This particular terrarium rests on a planter saucer and features a small white pumpkin.</p>
+                    </div>
+                    <figure class="stepThumb"> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/e3e18f0b-ab2c-4ffb-9988-a1ea63faef8b.jpg" alt="Cloche placed on a terracotta saucer" class="photo" data-credit="Lucy Akins"/> </figure>
+                    <figcaption class="small caption"> Cloche placed on a terracotta saucer (Lucy Akins) </figcaption>
+                </div>
+            </div>
+            <div class="mod step">
+                <div class="stepContent">
+                    <div class="content"> <span class="headline2 head mg-1 block">Step 9</span>
+                        <p>This particular terrarium was placed on a wood slice and a little toy squirrel was placed inside to add a little whimsy.</p>
+                    </div>
+                    <figure class="stepThumb"> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/2cd79f8d-0d16-4573-8861-e47fb74b0638.jpg" alt="Placed on a wooden slice" class="photo" data-credit="Lucy Akins"/> </figure>
+                    <figcaption class="small caption"> Placed on a wooden slice (Lucy Akins) </figcaption>
+                </div>
+            </div>
+            <div class="mod step">
+                <div class="stepContent">
+                    <div class="content"> <span class="headline2 head mg-1 block">Finished Terrarium</span>
+                        <p>Displayed alone or in a group, these pretty arrangements allow you to add a little nature to your decor or tablescape.</p>
+                    </div>
+                    <figure class="stepThumb"> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/78670312-8636-4c42-a75c-3029f7aa6c73.jpg" alt="Cloche terrarium" class="photo" data-credit="Lucy Akins"/> </figure>
+                    <figcaption class="small caption"> Cloche terrarium (Lucy Akins) </figcaption>
+                </div>
+            </div>
+            <div id="m2"></div>
+            <div id="DMINSTR" type="adimpression" data-type="adTracking" name="&amp;adunit_id=ca-ehow_336x280&amp;ad_unit_type=CNT&amp;ad_unit_network=GAP&amp;revenue_basis=CPC&amp;ad_position=-&amp;revenue_tag=craft_other_diy" data-module="google-ad-336x280">
+                <div id="GoogleAdsense336x280" class="mod"></div>
+                <script>
+                    dmjs.revenueTags.push('ca-ehow_336x280;craft_other_diy,text,suggested7');
+                    googleAds.addAdUnit({
+                        priority: 2,
+                        adUnitId: 'GoogleAdsense336x280',
+                        google_ad_channel: 'craft_other_diy,text,suggested7',
+                        google_ad_client: 'ca-ehow_336x280',
+                        google_page_url: 'http://www.ehow.com/how_2042752_build-terrarium.html',
+                        google_max_num_ads: '4',
+                        google_ad_type: 'text',
+                        google_hints: '',
+                        templates: {
+                            text: '&lt;div class="GoogleTextAd GoogleTextAdNew">\
+						&lt;a class="header" href="{!= google_info.feedback_url !}">Sponsored Links&lt;/a>\
+						&lt;ul class="opNew336 evenColorItems">\
+						{! for (var i = 0; i &lt; ads.length; i++) { !}\
+						&lt;li class="item">\
+						&lt;div class="term">\
+							&lt;a rel="nofollow" target="_blank" href="{!= ads[i].url !}" title="go to {!= clean(ads[i].visible_url) !}" class="title">\
+								&lt;span>{!= ads[i].line1 !}&lt;/span>\
+							&lt;/a>\
+						&lt;/div>\
+						&lt;div class="Ad">\
+								&lt;p class="copy">{!= ads[i].line2 !}&lt;/p>\
+								&lt;p class="copy">{!= ads[i].line3 !}&lt;/p>\
+								&lt;a rel="nofollow" target="_blank" href="{!= ads[i].url !}" title="go to {!= clean(ads[i].visible_url) !}" class="baseurl url">\
+									{!= ads[i].visible_url !}\
+								&lt;/a>\
+						&lt;/div>\
+						&lt;/li>\
+						{! } !}\
+						&lt;/ul>\
+					&lt;/div>',
+                            image: '&lt;div class="GoogleImageAd GoogleImageAdNew"> \
+							&lt;a class="header" href="{!= google_info.feedback_url !}">Sponsored Links&lt;/a> \
+							&lt;a href="{!= ads[0].url !}" title="go to {!= clean(ads[0].visible_url) !}" target="_blank" class="Ad" >\
+								&lt;img src="{!= ads[0].image_url !}" width="{!= ads[0].image_width !}" height="{!= ads[0].image_height !}" border="0" alt="" />\
+							&lt;/a> \
+						&lt;/div>',
+                            flash: '&lt;div class="GoogleFlashAd GoogleFlashAdNew">\
+							&lt;a class="header" href="{!= google_info.feedback_url !}">Sponsored Links&lt;/a> \
+							&lt;div class="Ad">\
+								&lt;object classid="clsid:D27CDB6E-AE6D-11cf-96B8-444553540000" codebase="http://download.macromedia.com/pub/shockwave/cabs/flash/swflash.cab#version=6,0,0,0" width="{!= ads[0].image_width !}" height="{!= ads[0].image_height !}">&lt;param name="movie" value="{!= ads[0].image_url !}">&lt;param name="quality" value="high">&lt;param name="AllowScriptAccess" value="never">&lt;embed src="{!= ads[0].image_url !}" width="{!= ads[0].image_width !}" height="{!= ads[0].image_height !}" type="application/x-shockwave-flash" AllowScriptAccess="never" pluginspage="http://www.macromedia.com/go/getflashplayer">&lt;/embed>&lt;/object>\
+							&lt;/div>\
+						&lt;/div>'
+                        },
+                        google_image_size: '336x280'
+                    });
+                    AdblockPlus.detect(function(usesABP) {
+                        if (usesABP) {
+                            googleAds.overrideAd('GoogleAdsense336x280', {
+                                google_ad_channel: 'craft_other_diy,text,suggested7,Art6'
+                            });
+                        }
+                    });
+                </script>
+            </div>
+            <div class="RelatedSearches FLC mod" id="RelatedSearches" data-module="related-searches">
+                <div class="headline2 head heading">Related Searches</div>
+            </div>
+            <div class="community mod" style="float:left;width: 610px;">
+                <div id="_ehow-com-injection" data-article-id="2042752" data-cat-id="5352" data-scat-id="5426" data-sscat-id="5429"></div>
+                <script id="_ehow-com-article" type="text/javascript">
+                    dmjs.domready(function() {
+                        if (!(dmjs.platform.isTablet())) {
+                            var ehowCom = document.createElement('script');
+                            ehowCom.type = 'text/javascript';
+                            ehowCom.async = true;
+                            ehowCom.src = 'https://community.ehow.com/widget/ehow/community.js';
+                            var s = document.getElementsByTagName('script')[0];
+                            s.parentNode.insertBefore(ehowCom, s);
+                        }
+                    });
+                </script>
+            </div>
+            <aside class="mod related clearfix" data-module="rcp_bottom">
+                <h3 class="head headline2 mg-2">More Like This</h3>
+                <ul class="unstyled">
+                    <li class="item fl ">
+                        <a href="http://www.ehow.com/how_4431062_convert-aquarium-terrarium.html"> <img src="http://img-aws.ehowcdn.com/200x133/cme/photography.prod.demandstudios.com/90f9f3d7-6e4f-4be8-b379-3d994ccbf69c.jpg" class="img" alt=""/>
+                            <p class="headline6 mg-0">How to Convert an Aquarium to a Terrarium</p>
+                        </a>
+                    </li>
+                    <li class="item fl even">
+                        <a href="http://www.ehow.com/video_12263138_plant-terrarium-ideas.html"> <img src="http://img-aws.ehowcdn.com/200x133/viper/media/2b3830ca-4eb6-4bfd-a1cd-d430b3e28d59/jpeg/a7edbb0f-86a7-4c73-9c2d-f85f87b49efa_0.jpg" class="img" alt=""/>
+                            <p class="headline6 mg-0">Plant Terrarium Ideas</p>
+                        </a>
+                    </li>
+                    <li class="item fl ">
+                        <a href="http://www.ehow.com/video_9374_make-terrarium.html"> <img src="http://img-aws.ehowcdn.com/200x133/ehow/images/a00/09/4u/make-terrarium-800x800.jpg" class="img" alt=""/>
+                            <p class="headline6 mg-0">How to Make a Terrarium</p>
+                        </a>
+                    </li>
+                </ul>
+            </aside>
+            <figure id="Comments" class="mod FLC" data-module="comments">
+                <div data-id="fb_comments"></div>
+            </figure>
+            <section id="FeaturedTombstone" class="mod" data-module="rcp_tombstone">
+                <h2 class="headline2 head mg-2">Featured</h2>
+                <div class="featured FLC">
+                    <section class="Alpha  item" data-cme-module="0">
+                        <a class="target link-accent gtm_featuredArticles" href="http://www.ehow.com/how_2056495_make-ballet-tutu.html"> <img src="http://img-aws.ehowcdn.com/200x133/cme/photography.prod.demandstudios.com/7f06be69-3650-4e94-88d4-2289c1a5c6a2.jpg" class="photo" alt=""/> <span class="subtitle">Read Article</span>
+                            <div class="title headline6">How to Make a Ballet Tutu</div>
+                        </a>
+                    </section>
+                    <section class=" item" data-cme-module="1">
+                        <a class="target link-accent gtm_featuredArticles" href="http://www.ehow.com/how_7687715_paint-tulip-watercolor.html"> <img src="http://img-aws.ehowcdn.com/200x133/cme/uploadedimages.demandmedia/tulips-1.jpg" class="photo" alt=""/> <span class="subtitle">Read Article</span>
+                            <div class="title headline6">How to Paint a Tulip in Watercolor</div>
+                        </a>
+                    </section>
+                    <section class="Omega item" data-cme-module="2">
+                        <a class="target link-accent gtm_featuredArticles" href="http://www.ehow.com/info_12340422_kids-kitchen-slipcover.html"> <img src="http://img-aws.ehowcdn.com/200x133/cme/uploadedimages.demandmedia/kitchen-1.jpg" class="photo" alt=""/> <span class="subtitle">Read Article</span>
+                            <div class="title headline6">Kids Kitchen Slipcover</div>
+                        </a>
+                    </section>
+                </div>
+            </section>
+        </div>
+        <div class="col-rail">
+            <!-- gpt slot 300x250_300x600 -->
+            <div class="AdUnit mod" data-module="gpt-ad-300x250_300x600" id="300x250_300x600">
+                <script type="text/javascript">
+                    googletag.cmd.push(function() {
+                        googletag.display('300x250_300x600');
+                    });
+                </script>
+            </div>
+            <!-- gpt slot 300x100 -->
+            <div class="mod hide" data-module="radlinks">
+                <h3 class="headline3 head mg-2">Related Ads</h3>
+                <div id="DMINSTR" type="rlimpression" data-type="adTracking" name="&amp;adunit_id=ca-ehow-radlinks_js&amp;ad_unit_type=LNK&amp;ad_unit_network=GAP&amp;revenue_basis=CPC&amp;ad_position=-&amp;revenue_tag=craft_other_diy">
+                    <div id="GoogleRelatedAds"></div>
+                    <script>
+                        dmjs.revenueTags.push('ca-ehow-radlinks_js;craft_other_diy,text,suggested7');
+                        googleAds.addAdUnit({
+                            priority: 1,
+                            adUnitId: 'GoogleRelatedAds',
+                            google_ad_channel: 'craft_other_diy,text,suggested7',
+                            google_ad_client: 'ca-ehow-radlinks_js',
+                            google_page_url: 'http://www.ehow.com/how_2042752_build-terrarium.html',
+                            google_max_num_ads: '0',
+                            google_num_radlinks: '6',
+                            google_max_radlink_len: '27',
+                            metaData: {
+                                "contentid": "2042752",
+                                "studioid": "f941e4cd-a49e-4948-b244-677c15ff8b7f",
+                                "category": "Crafts",
+                                "subcategory": "Other DIY Crafts",
+                                "skin": "crafts"
+                            }
+                        });
+                    </script>
+                </div>
+            </div>
+            <div class="RelatedSearches FLC mod" id="RelatedSearches" data-module="related-searches-right">
+                <div class="headline3 head mg-2">Related Searches</div>
+            </div>
+            <section class="mod box slideshow" data-module="rcp_slideshow_module">
+                <a href="http://www.ehow.com/how_16357_make-paper-sunflowers.html"> <img src="http://img-aws.ehowcdn.com/300x200/cme/uploadedimages.demandmedia/sunflowers-1.jpg" alt="" title="" class="photo" longdesc="http://s3.amazonaws.com/uploadedimages.demandmedia/sunflowers-1.jpg"/> <span class="arrow arrow-left" data-icon="4"></span> <span class="arrow arrow-right" data-icon="5"></span> <span class="subtitle">
+Read Article	</span> </a>
+                <a href="http://www.ehow.com/how_16357_make-paper-sunflowers.html">
+                    <h3 class="headline4 title">
+How to Make Paper Sunflowers	</h3> </a>
+            </section>
+            <ul class="media-list mod unstyled " data-module="rcp_right_rail">
+                <h2 class="headline3 head mg-2 title">You May Like</h2>
+                <li class="media headline6">
+                    <a href="http://www.ehow.com/how_4431062_convert-aquarium-terrarium.html" class="fl gtm_youMayLike"> <img src="http://img-aws.ehowcdn.com/150X100/cme/photography.prod.demandstudios.com/90f9f3d7-6e4f-4be8-b379-3d994ccbf69c.jpg" alt="" class="media-object"/>
+                        <div class="pagescore js-pagescore small" data-score="true" data-url="http://www.ehow.com/how_4431062_convert-aquarium-terrarium.html"> <span class=" num js-num"></span> <span class="icon"></span> </div>
+                    </a>
+                    <div class="media-body"> <a href="http://www.ehow.com/how_4431062_convert-aquarium-terrarium.html" class="gtm_youMayLike">How to Convert an Aquarium to a Terrarium</a> </div>
+                </li>
+                <li class="media headline6">
+                    <a href="http://www.ehow.com/video_12263137_build-orchid-terrarium.html" class="fl gtm_youMayLike"> <img src="http://img-aws.ehowcdn.com/150X100/viper/media/2b3830ca-4eb6-4bfd-a1cd-d430b3e28d59/jpeg/47b02568-2cde-44f4-a39c-494294e50ccd_1.jpg" alt="" class="media-object"/>
+                        <div class="pagescore js-pagescore small" data-score="true" data-url="http://www.ehow.com/video_12263137_build-orchid-terrarium.html"> <span class=" num js-num"></span> <span class="icon"></span> </div>
+                    </a>
+                    <div class="media-body"> <a href="http://www.ehow.com/video_12263137_build-orchid-terrarium.html" class="gtm_youMayLike">How to Build an Orchid Terrarium</a> </div>
+                </li>
+                <li class="media headline6">
+                    <a href="http://www.ehow.com/video_9374_make-terrarium.html" class="fl gtm_youMayLike"> <img src="http://img-aws.ehowcdn.com/150X100/ehow/images/a00/09/4u/make-terrarium-800x800.jpg" alt="" class="media-object"/>
+                        <div class="pagescore js-pagescore small" data-score="true" data-url="http://www.ehow.com/video_9374_make-terrarium.html"> <span class=" num js-num"></span> <span class="icon"></span> </div>
+                    </a>
+                    <div class="media-body"> <a href="http://www.ehow.com/video_9374_make-terrarium.html" class="gtm_youMayLike">How to Make a Terrarium</a> </div>
+                </li>
+                <li class="media headline6">
+                    <a href="http://www.ehow.com/video_12263133_group-plants-together-terrarium.html" class="fl gtm_youMayLike"> <img src="http://img-aws.ehowcdn.com/150X100/viper/media/2b3830ca-4eb6-4bfd-a1cd-d430b3e28d59/jpeg/dc66dd37-78b6-4494-a8db-7bbe697cedda_0.jpg" alt="" class="media-object"/>
+                        <div class="pagescore js-pagescore small" data-score="true" data-url="http://www.ehow.com/video_12263133_group-plants-together-terrarium.html"> <span class=" num js-num"></span> <span class="icon"></span> </div>
+                    </a>
+                    <div class="media-body"> <a href="http://www.ehow.com/video_12263133_group-plants-together-terrarium.html" class="gtm_youMayLike">How to Group Plants Together in a Terrarium</a> </div>
+                </li>
+                <li class="media headline6">
+                    <a href="http://www.ehow.com/video_12263135_build-tropical-terrarium.html" class="fl gtm_youMayLike"> <img src="http://img-aws.ehowcdn.com/150X100/viper/media/2b3830ca-4eb6-4bfd-a1cd-d430b3e28d59/jpeg/70ebb809-634d-41df-9fff-9b183f81e9e0_1.jpg" alt="" class="media-object"/>
+                        <div class="pagescore js-pagescore small" data-score="true" data-url="http://www.ehow.com/video_12263135_build-tropical-terrarium.html"> <span class=" num js-num"></span> <span class="icon"></span> </div>
+                    </a>
+                    <div class="media-body"> <a href="http://www.ehow.com/video_12263135_build-tropical-terrarium.html" class="gtm_youMayLike">How to Build a Tropical Terrarium</a> </div>
+                </li>
+                <li class="media headline6">
+                    <a href="http://www.ehow.com/how_7853040_diy-own-living-room-furniture.html" class="fl gtm_youMayLike"> <img src="http://img-aws.ehowcdn.com/150X100/cme/photography.prod.demandstudios.com/2ecfca5a-1c35-4fbe-9ff5-8f5bd8129d97.jpg" alt="" class="media-object"/>
+                        <div class="pagescore js-pagescore small" data-score="true" data-url="http://www.ehow.com/how_7853040_diy-own-living-room-furniture.html"> <span class=" num js-num"></span> <span class="icon"></span> </div>
+                    </a>
+                    <div class="media-body"> <a href="http://www.ehow.com/how_7853040_diy-own-living-room-furniture.html" class="gtm_youMayLike">DIY Build Your Own Living Room Furniture</a> </div>
+                </li>
+                <li class="media headline6">
+                    <a href="http://www.ehow.com/slideshow_12333243_edible-arrangements-make-own-easter-candy-terrariums.html" class="fl gtm_youMayLike"> <img src="http://img-aws.ehowcdn.com/150X100/ds-cdn-write/upload/image/39/4D/34B73D38-EC9F-4415-A740-23DCF2F94D39/34B73D38-EC9F-4415-A740-23DCF2F94D39.jpg" alt="" class="media-object"/>
+                        <div class="pagescore js-pagescore small" data-score="true" data-url="http://www.ehow.com/slideshow_12333243_edible-arrangements-make-own-easter-candy-terrariums.html"> <span class=" num js-num"></span> <span class="icon"></span> </div>
+                    </a>
+                    <div class="media-body"> <a href="http://www.ehow.com/slideshow_12333243_edible-arrangements-make-own-easter-candy-terrariums.html" class="gtm_youMayLike">Edible Arrangements: Make Your Own Easter Candy Terrariums</a> </div>
+                </li>
+                <li class="media headline6">
+                    <a href="http://www.ehow.com/ehow-mom/blog/3-ingredient-easter-bark/" class="fl gtm_youMayLike"> <img src="http://img-aws.ehowcdn.com/150X100/ehow-mom-blog-us/files/2015/03/easterbark0.jpg" alt="" class="media-object"/>
+                        <div class="pagescore js-pagescore small" data-score="true" data-url="http://www.ehow.com/ehow-mom/blog/3-ingredient-easter-bark/"> <span class=" num js-num"></span> <span class="icon"></span> </div>
+                    </a>
+                    <div class="media-body"> <a href="http://www.ehow.com/ehow-mom/blog/3-ingredient-easter-bark/" class="gtm_youMayLike">3-Ingredient Easter Bark</a> </div>
+                </li>
+            </ul>
+        </div>
+    </section>
+    <script src="http://v5-static.ehowcdn.com/content/compressed/module-jquery-764eded0.js"></script>
+    <footer id="Footer" data-section="footer">
+        <div class="inner-container">
+            <div class="details clearfix">
+                <div class="identity"> <a href="http://www.demandmedia.com/" class="logo">demandmedia.com</a>
+                    <p class="copyright">© 1999-2015 Demand Media, Inc.</p>
+                </div>
+                <nav xmlns="http://www.w3.org/1999/html">
+                    <ul class="listing">
+                        <li>
+                            <ul>
+                                <li class="item row1"> <a href="http://www.ehow.com/about-us.html" class="title">About eHow</a> </li>
+                                <li class="item row1"> <a href="http://www.ehow.co.uk/" class="title">eHow UK</a> </li>
+                                <li class="item row1"> <a href="http://www.ehowenespanol.com/" class="title">eHow en Español</a> </li>
+                                <li class="item row1"> <a href="http://www.ehow.com.br/" class="title">eHow Brasil</a> </li>
+                                <li class="item row1"> <a href="http://www.ehow.de/" class="title">eHow Deutschland</a> </li>
+                                <li class="item row1"> <a href="http://www.ehowhelp.com/hc/en-us" class="title">Contact Us</a> </li>
+                            </ul>
+                        </li>
+                        <li>
+                            <ul>
+                                <li class="item row2"> <a href="http://www.ehow.com/blog/" class="title">eHow Blog</a> </li>
+                                <li class="item row2"> <a href="http://www.ehow.com/how-to.html" class="title">How to by Topic</a> </li>
+                                <li class="item row2"> <a href="http://www.ehow.com/videos.html" class="title">How to Videos</a> </li>
+                            </ul>
+                        </li>
+                        <li>
+                            <ul>
+                                <li class="item row3"> <a href="http://www.ehow.com/terms-of-use.html" class="title">Terms of Use</a> </li>
+                                <li class="item row3"> <a href="http://www.ehow.com/privacy.html" class="title">Privacy Policy</a> </li>
+                                <li class="item row3"> <a href="http://www.ehow.com/terms-of-use.html#copyright/" class="title">Report Copyright</a> </li>
+                                <li class="item row3">
+                                    <a id="_bapw-link" class="ad-choice" style="cursor:pointer !important"><img id="_bapw-icon" style="display:inline !important;vertical-align:middle !important"/> <span style="vertical-align:middle !important">Ad Choices</span></a> <abbr>en-US</abbr> </li>
+                            </ul>
+                        </li>
+                    </ul>
+                </nav>
+                <div class="socialConnect fr">
+                    <p class="headline4 heading head">Connect with us:</p>
+                    <ul>
+                        <li class="source"><a class="icon elegant-icons gtm_social" href="http://www.pinterest.com/ehow/" target="_blank"></a></li>
+                        <li class="source"><a class="icon elegant-icons gtm_social" href="https://www.facebook.com/eHow" target="_blank"></a></li>
+                        <li class="source"><a class="icon elegant-icons gtm_social" href="https://twitter.com/ehow" target="_blank"></a></li>
+                        <li class="source"><a class="icon elegant-icons gtm_social" href="https://plus.google.com/+howto/posts" target="_blank"></a></li>
+                        <li class="source last"><a class="icon elegant-icons gtm_social" href="http://instagram.com/ehow_team" target="_blank"></a></li>
+                    </ul>
+                </div>
+            </div>
+        </div>
+        <div class="extras">
+            <div class="comscore comscore-eh">
+                <div class="category5352 eh-cat"></div>
+            </div>
+        </div>
+    </footer>
+    <!-- BEGIN dmtracker -->
+    <noscript><img src="http://vs.dmtracker.com/images/zig.gif?Log=1" alt="zig" /></noscript>
+    <div id="dmOmni"></div>
+    <!-- END dmtracker -->
+    <noscript><img src="http://b.scorecardresearch.com/b?c1=2&c2=6036385&c3=&c7=http%3A%2F%2Fwww.ehow.com%2Fhow_2042752_build-terrarium.html%3Fcomscorekw%3DeHow_Hobbies_Games_and_Toys&cv=2.0&cj=1" /></noscript>
+    <script>
+        var dmjs = dmjs || {};
+        dmjs.global = dmjs.global || {};
+        dmjs.global.comscore = {
+            "c1": "2",
+            "c2": "6036385",
+            "c3": "",
+            "options": {
+                "url_append": "comscorekw=eHow_Hobbies_Games_and_Toys"
+            }
+        };
+        dmjs.global.dm_tracker = {
+            "domain": "vs.dmtracker.com",
+            "url": "bit.ehow.com",
+            "bit": "bit"
+        };
+        dmjs.global.fb_comment = {
+            "num_posts": 5,
+            "width": 620,
+            "mobile": false,
+            "href": "http:\/\/www.ehow.com\/how_2042752_build-terrarium.html",
+            "selector": "[data-id=\"fb_comments\"]"
+        };
+        dmjs.global.hnh = {
+            "url": "http:\/\/services.ehowcdn.com\/widgets\/generated\/hnh.min.js"
+        };
+    </script>
+    <script src="http://dynamic01.ehowcdn.com/services/modules/js/hnh,evidon,rev_tag,dm_track,comscore,common_deferred,cache_buster/ba9c0085/" defer="defer"></script>
+    <!--[if lt IE 9]>
+<script src="http://v5-static.ehowcdn.com/content/compressed/common_footer-266c5d77.js"  ></script>
+<![endif]-->
+    <script src="http://v5-static.ehowcdn.com/content/compressed/inline-81314781.js" defer="defer"></script>
+    <script>
+        dmjs.domready(function() {
+            new OmnitureLinkSet($('[data-module="rcp_top"]'), "TLS101", "default_IL");
+        });
+    </script>
+    <script>
+        dmjs.domready(function() {
+            new OmnitureLinkSet($('[data-module="rcp_bottom"]'), "TLS103", "default_B");
+        });
+    </script>
+    <script>
+        dmjs.domready(function() {
+            new OmnitureLinkSet($('[data-module="rcp_tombstone"]'), "LS103", "default_TOMB");
+        });
+    </script>
+    <script>
+        dmjs.domready(function() {
+            new OmnitureLinkSet($('[data-module="rcp_slideshow_module"]'), "TLS106", "default_SS");
+        });
+    </script>
+    <script>
+        dmjs.domready(function() {
+            new OmnitureLinkSet($('[data-module="rcp_right_rail"]'), "TLS102", "default_R3");
+        });
+    </script>
+    <script>
+        googleAds.defaults.google_adtest = 'off';
+        googleAds.defaults.google_page_url = 'http://www.ehow.com/how_2042752_build-terrarium.html';
+        googleAds.render();
+    </script>
+    <script defer="" src="//cdn.optimizely.com/js/9718688.js"></script>
+    <script>
+        dmjs.addEvent(window, 'load', function() {
+            dmjs.fireEvents();
+        });
+    </script>
+    <script defer="" src="http://static.ehownowcdn.com/widget/ehow/auth.js"></script>
+    <div id="fb-root"></div>
+    <div class="view-mobile"></div>
+</body>
+
+</html>

article_scraper/resources/tests/readability/ehow-2/expected.html

@@ -0,0 +1,143 @@
+<article><DIV id="readability-page-1"><div data-type="AuthorProfile">
+                <div>
+                    <p><a id="img-follow-tip" href="http://fakehost/contributor/gina_robertsgrey/" target="_blank">
+                        <img src="http://img-aws.ehowcdn.com/60x60/cme/cme_public_images/www_demandstudios_com/sitelife.studiod.com/ver1.0/Content/images/store/9/2/d9dd6f61-b183-4893-927f-5b540e45be91.Small.jpg" data-failover="//img-aws.ehowcdn.com/60x60/ehow-cdn-assets/test15/media/images/authors/missing-author-image.png" onerror="var failover = this.getAttribute('data-failover');&#10;                if (failover) failover = failover.replace(/^https?:/,'');&#10;                var src = this.src ? this.src.replace(/^https?:/,'') : '';&#10;                if (src != failover){&#10;                    this.src = failover;&#10;                }"/> </a></p>
+                </div>
+                
+                <p><time datetime="2016-09-14T07:07:00-04:00" itemprop="dateModified">Last updated September 14, 2016</time>
+                </p>
+                
+            </div><div>
+            <article data-type="article">
+                <div>
+                        <div>
+                            <p>Graduation parties are a great way to commemorate the years of hard work teens and college co-eds devote to education. They’re also costly for mom and dad.</p>
+                            <p>The average cost of a graduation party in 2013 was a whopping $1,200, according to Graduationparty.com; $700 of that was allocated for food. However that budget was based on Midwestern statistics, and parties in urban areas like New York City are thought to have a much higher price tag.</p>
+                            <p>Thankfully, there are plenty of creative ways to trim a little grad party fat without sacrificing any of the fun or celebratory spirit.</p>
+                        </div>
+                        <figure>
+                            <img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/2F/86/5547EF62-EAF5-4256-945D-0496F61C862F/5547EF62-EAF5-4256-945D-0496F61C862F.jpg" alt="Graduation" title="Graduation" data-credit="Mike Watson Images/Moodboard/Getty " longdesc="http://s3.amazonaws.com/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/2F/86/5547EF62-EAF5-4256-945D-0496F61C862F/5547EF62-EAF5-4256-945D-0496F61C862F.jpg" data-pin-ehow-hover="true" data-pin-no-hover="true"/>
+                        </figure>
+                        <figcaption>
+                            (Mike Watson Images/Moodboard/Getty)
+                        </figcaption>
+                    </div>
+                
+                
+                <span>
+<span>
+<div>
+<p><span><p>Parties hosted at restaurants, clubhouses and country clubs eliminate the need to spend hours cleaning up once party guests have gone home. But that convenience comes with a price tag. A country club may charge as much as $2,000 for room rental and restaurant food and beverage will almost always cost more than food prepped and served at home.</p></span> </p>
+        <figure>
+            <img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/FE/CB/121569D2-6984-4B2F-83C4-9D2D9A27CBFE/121569D2-6984-4B2F-83C4-9D2D9A27CBFE.jpg" alt="Save money hosting the party at home." data-credit="Thomas Jackson/Digital Vision/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true"/>
+        </figure>
+        <figcaption>
+            Thomas Jackson/Digital Vision/Getty Images </figcaption>
+        </div>
+        </span>
+        </span>
+        <span>
+<span>
+<div>
+<p><span><p>Instead of hiring a DJ, use your iPod or Smartphone to spin the tunes. Both easily hook up to most speakers or mp3 compatible docks to play music from your music library. Or download Pandora, the free online radio app, and play hours of music for free.</p>
+<p>Personalize the music with a playlist of the grad’s favorite songs or songs that were big hits during his or her years in school.</p></span> </p>
+        <figure>
+            <img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/DF/FC/A05B0252-BD73-4BC7-A09A-96F0A504FCDF/A05B0252-BD73-4BC7-A09A-96F0A504FCDF.jpg" alt="Online radio can take the place of a hired DJ." data-credit="Spencer Platt/Getty Images News/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true"/>
+        </figure>
+        <figcaption>
+            Spencer Platt/Getty Images News/Getty Images </figcaption>
+        </div>
+        </span>
+        </span>
+        <span>
+<span>
+<div>
+<p><span><p>Avoid canned drinks, which guests often open, but don't finish. Serve pitchers of tap water with lemon and cucumber slices or sliced strawberries for an interesting and refreshing flavor. Opt for punches and non-alcoholic drinks for high school graduates that allow guests to dole out the exact amount they want to drink.</p></span> </p>
+        <figure>
+            <img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/EB/DB/8A04CCA7-3255-4225-B59A-C41441F8DBEB/8A04CCA7-3255-4225-B59A-C41441F8DBEB.jpg" alt="Serve drinks in pitchers, not in cans." data-credit="evgenyb/iStock/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true"/>
+        </figure>
+        <figcaption>
+            evgenyb/iStock/Getty Images </figcaption>
+        </div>
+        
+        </span>
+        </span>
+        <span>
+<span>
+<div>
+<p><span><p>Instead of inviting everyone you – and the graduate – know or ever knew, scale back the guest list. Forgo inviting guests that you or your grad haven't seen for eons. There is no reason to provide provisions for people who are essentially out of your lives. Sticking to a small, but personal, guest list allows more time to mingle with loved ones during the party, too.</p></span> </p>
+        <figure>
+            <img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/94/10/08035476-0167-4A03-AADC-13A7E7AA1094/08035476-0167-4A03-AADC-13A7E7AA1094.jpg" alt="Limit guests to those close to the graduate." data-credit="Kane Skennar/Photodisc/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true"/>
+        </figure>
+        <figcaption>
+            Kane Skennar/Photodisc/Getty Images </figcaption>
+        </div>
+        </span>
+        </span>
+        <span>
+<span>
+<div>
+<p><span><p>See if your grad and his best friend, girlfriend or close family member would consider hosting a joint party. You can split some of the expenses, especially when the two graduates share mutual friends. You'll also have another parent to bounce ideas off of and to help you stick to your budget when you're tempted to splurge.</p></span> </p>
+        <figure>
+            <img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/06/49/4AD62696-FC95-4DA2-8351-42740C7B4906/4AD62696-FC95-4DA2-8351-42740C7B4906.jpg" alt="Throw a joint bash for big savings." data-credit="Mike Watson Images/Moodboard/Getty" data-pin-ehow-hover="true" data-pin-no-hover="true"/>
+        </figure>
+        <figcaption>
+            Mike Watson Images/Moodboard/Getty </figcaption>
+        </div>
+        </span>
+        </span>
+        <span>
+<span>
+<div>
+<p><span><p>Skip carving stations of prime rib and jumbo shrimp as appetizers, especially for high school graduation parties. Instead, serve some of the graduate's favorite side dishes that are cost effective, like a big pot of spaghetti with breadsticks. Opt for easy and simple food such as pizza, finger food and mini appetizers. </p>
+<p>Avoid pre-packaged foods and pre-made deli platters. These can be quite costly. Instead, make your own cheese and deli platters for less than half the cost of pre-made.</p></span> </p>
+        <figure>
+            <img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/D0/51/B6AED06C-5E19-4A26-9AAD-0E175F6251D0/B6AED06C-5E19-4A26-9AAD-0E175F6251D0.jpg" alt="Cost effective appetizers are just as satisfying as pre-made deli platters." data-credit="Mark Stout/iStock/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true"/>
+        </figure>
+        <figcaption>
+            Mark Stout/iStock/Getty Images </figcaption>
+        </div>
+        </span>
+        </span>
+        <span>
+<span>
+<div>
+<p><span><p>Instead of an evening dinner party, host a grad lunch or all appetizers party. Brunch and lunch fare or finger food costs less than dinner. Guests also tend to consume less alcohol in the middle of the day, which keeps cost down.</p></span> </p>
+        <figure>
+            <img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/35/B4/DD5FD05A-B631-4AFE-BC8F-FDACAD1EB435/DD5FD05A-B631-4AFE-BC8F-FDACAD1EB435.jpg" alt="A brunch gathering will cost less than a dinner party." data-credit="Mark Stout/iStock/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true"/>
+        </figure>
+        <figcaption>
+            Mark Stout/iStock/Getty Images </figcaption>
+        </div>
+        <div id="relatedContentUpper" data-module="rcp_top">
+            <header>
+                <h3>Other People Are Reading</h3>
+            </header>
+            
+        </div>
+        
+        </span>
+        </span>
+        <span>
+<span>
+<div>
+<p><span><p>Decorate your party in the graduate's current school colors or the colors of the school he or she will be headed to next. Décor that is not specifically graduation-themed may cost a bit less, and any leftovers can be re-used for future parties, picnics and events.</p></span> </p>
+        <figure>
+            <img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/A1/FA/2C368B34-8F6A-45F6-9DFC-0B0C4E33FAA1/2C368B34-8F6A-45F6-9DFC-0B0C4E33FAA1.jpg" alt="Theme the party by color without graduation-specific decor." data-credit="jethuynh/iStock/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true"/>
+        </figure>
+        <figcaption>
+            jethuynh/iStock/Getty Images </figcaption>
+        </div>
+        </span>
+        </span>
+        
+        
+        <h2>
+            <a target="_blank" href="https://www.google.com/adsense/support/bin/request.py?contact=abg_afc&amp;url=http://ehow.com/&amp;hl=en&amp;client=ehow&amp;gl=US">Related Searches</a>
+        </h2>
+        
+        
+        <p>Promoted By Zergnet</p>
+        
+        </article>
+        </div></DIV></article>

article_scraper/resources/tests/readability/embedded-videos/expected.html

@@ -0,0 +1,26 @@
+<article><DIV id="readability-page-1"><article>
+    <h2>Lorem</h2>
+    <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+      tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+    
+    <p>At root</p>
+    <iframe width="480" height="360" src="https://www.youtube.com/embed/LtOGa5M8AuU" frameborder="0" allowfullscreen="" aspect-ratio="auto"><empty></empty></iframe>
+    <iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/LtOGa5M8AuU" frameborder="0" allowfullscreen=""><empty></empty></iframe>
+    <iframe src="https://player.vimeo.com/video/32246206?color=ffffff+title=0+byline=0+portrait=0" width="500" height="281" frameborder="0" webkitallowfullscreen="" mozallowfullscreen="" allowfullscreen=""><empty></empty></iframe>
+    <p>In a paragraph</p>
+    <p><iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/LtOGa5M8AuU" frameborder="0" allowfullscreen=""><empty></empty></iframe></p>
+    <p>In a div</p>
+    <div><iframe width="480" height="360" src="https://www.youtube.com/embed/LtOGa5M8AuU" frameborder="0" allowfullscreen="" aspect-ratio="auto"><empty></empty></iframe></div>
+    <h2>Foo</h2>
+    <p>
+      Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </p>
+  </article></DIV></article>

article_scraper/resources/tests/readability/embedded-videos/source.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8"/>
+  <title>Embedded videos test</title>
+</head>
+<body>
+  <article>
+    <h1>Lorem</h1>
+    <div>
+      <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+      tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+    </div>
+    <h2>Videos</h2>
+    <p>At root</p>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/LtOGa5M8AuU"
+            frameborder="0" allowfullscreen=""></iframe>
+    <iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/LtOGa5M8AuU"
+            frameborder="0" allowfullscreen=""></iframe>
+    <iframe src="https://player.vimeo.com/video/32246206?color=ffffff+title=0+byline=0+portrait=0"
+            width="500" height="281" frameborder="0"
+            webkitallowfullscreen="" mozallowfullscreen="" allowfullscreen=""></iframe>
+    <p>In a paragraph</p>
+    <p><iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/LtOGa5M8AuU"
+               frameborder="0" allowfullscreen=""></iframe></p>
+    <p>In a div</p>
+    <div><iframe width="560" height="315" src="https://www.youtube.com/embed/LtOGa5M8AuU"
+                 frameborder="0" allowfullscreen=""></iframe></div>
+    <h2>Foo</h2>
+    <div>
+      Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+      quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+      consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </div>
+  </article>
+</body>
+</html>

article_scraper/resources/tests/readability/engadget/expected.html

@@ -0,0 +1,330 @@
+<article><DIV id="readability-page-1"><div>
+                                                        <p>The <a href="https://www.engadget.com/2017/06/13/the-xbox-one-x-is-aspirational-in-the-purest-sense-of-the-word/" target="_blank">Xbox
+                                                                One X</a> is the ultimate video game system. It sports
+                                                                more horsepower than any system ever. And it plays more
+                                                                titles in native 4K than <a href="https://www.engadget.com/2016/11/07/sony-playstation-4-pro-review/" target="_blank">Sony's
+                                                                    PlayStation 4 Pro</a>. It's just about everything
+                                                                you could want without investing in a gaming PC. The
+                                                                only problem? It's now been a year since the PS4 Pro
+                                                                launched, and the One X costs $500, while Sony's console
+                                                                launched at $400. That high price limits the Xbox One X
+                                                                to diehard Microsoft fans who don't mind paying a bit
+                                                                more to play the console's exclusive titles in 4K.
+                                                                Everyone else might be better off waiting, or opting for
+                                                                the $279 <a href="https://www.engadget.com/2016/08/02/xbox-one-s-review/" target="_blank">Xbox
+                                                                    One S</a>. </p>
+                                                    </div><section>
+                                                            <h4> Gallery: Xbox One
+                                                                X | 14 Photos </h4>
+                                                            <div data-behavior="lightbox_trigger" data-engadget-slideshow-id="803271" data-eng-bang="{&quot;gallery&quot;:803271,&quot;slide&quot;:7142088,&quot;index&quot;:0}" data-eng-mn="93511844"><p><a href="#" data-index="0" data-engadget-slide-id="7142088" data-eng-bang="{&quot;gallery&quot;:803271,&quot;slide&quot;:7142088,&quot;index&quot;:0}">
+                                                                <img src="https://o.aolcdn.com/images/dims?thumbnail=980%2C653&amp;quality=80&amp;image_uri=https%3A%2F%2Fs.blogcdn.com%2Fslideshows%2Fimages%2Fslides%2F714%2F208%2F8%2FS7142088%2Fslug%2Fl%2Fxbox-one-x-review-gallery-1-1.jpg&amp;client=cbc79c14efcebee57402&amp;signature=9bb08b52e12de8e4060f863a52c613489529818d"/>
+                                                            </a></p>
+                                                                
+                                                            </div>
+                                                        </section><div>
+                                                        
+                                                        
+                                                        <div>
+                                                                <div>
+                                                                    
+                                                                    <ul>
+                                                                        <li>Most
+                                                                            powerful hardware ever in a home console
+                                                                        </li>
+                                                                        <li>Solid
+                                                                            selection of enhanced titles
+                                                                        </li>
+                                                                        <li>4K Blu-ray
+                                                                            drive is great for movie fans
+                                                                        </li>
+                                                                    </ul>
+                                                                </div>
+                                                                <div>
+                                                                    
+                                                                    <ul>
+                                                                        <li>Expensive
+                                                                        </li>
+                                                                        <li>Not worth
+                                                                            it if you don’t have a 4K TV
+                                                                        </li>
+                                                                        <li>Still no VR
+                                                                            support
+                                                                        </li>
+                                                                    </ul>
+                                                                </div>
+                                                            </div>
+                                                        <div>
+                                                            
+                                                            <p>As promised, the Xbox One X is the
+                                                                most powerful game console ever. In practice, though, it
+                                                                really just puts Microsoft on equal footing with Sony’s
+                                                                PlayStation 4 Pro. 4K/HDR enhanced games look great, but
+                                                                it’s lack of VR is disappointing in 2017.</p>
+                                                        </div>
+                                                    </div><div>
+                                                
+                                                <div>
+                                                                <h3>Hardware</h3>
+                                                                <p><img data-credit="Devindra Hardawar/AOL" data-mep="2181678" src="https://o.aolcdn.com/images/dims?crop=1600%2C1067%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C1067&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F93beb86758ae1cf95721699e1e006e35%2F205826074%2FXbox%2BOne%2BX%2Breview%2Bgallery%2B7.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=c0f2d36259c2c1decfb60aae364527cda2560d4a" alt=""/></p>
+                                                                <p>Despite all the power inside, the One X is
+                                                                    Microsoft's smallest console to date. It looks
+                                                                    similar to the Xbox One S, except it has an entirely
+                                                                    matte black case and is slightly slimmer. It's also
+                                                                    surprisingly dense -- the console weighs 8.4 pounds,
+                                                                    but it feels far heavier than you'd expect for its
+                                                                    size, thanks to all of its new hardware. The One S,
+                                                                    in comparison, weighs two pounds less.</p>
+                                                                <p>The Xbox One X's real upgrades are under the hood. It
+                                                                    features an 8-core CPU running at 2.3Ghz, 12GB of
+                                                                    GDDR5 RAM, a 1 terabyte hard drive and an upgraded
+                                                                    AMD Polaris GPU with 6 teraflops of computing power.
+                                                                    The PS4 Pro has only 8GB of RAM and tops out at 4.2
+                                                                    teraflops. Microsoft's console is clearly faster.
+                                                                    That additional horsepower means the Xbox One X can
+                                                                    run more games in full native 4K than the Sony's
+                                                                    console.</p>
+                                                                <p><img data-credit="Devindra Hardawar/AOL" data-mep="2182489" src="https://o.aolcdn.com/images/dims?crop=1600%2C949%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C949&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F9ece7fdad1e7025dec06ac9bf98688d0%2F205826075%2FXbox%2BOne%2BX%2Breview%2Bgallery%2B5.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=9913883753141e7df322616bfe0bc41c6ecd80c8" alt=""/></p>
+                                                                <p>Along the front, there's the slot-loading 4K Blu-ray
+                                                                    drive, a physical power button, a single USB port
+                                                                    and a controller pairing button. And around back,
+                                                                    there are HDMI out and in ports, the latter of which
+                                                                    lets you plug in your cable box. Additionally, there
+                                                                    are two USB ports, connections for optical audio, IR
+                                                                    out, and gigabit Ethernet. If you've still got a
+                                                                    Kinect around, you'll need to use a USB adapter to
+                                                                    plug it in.</p>
+                                                            </div>
+                                                <div data-engadget-breakout-type="image">
+                                                            <figure><img src="https://o.aolcdn.com/images/dims?resize=980%2C640&amp;quality=100&amp;image_uri=https%3A%2F%2Fo.aolcdn.com%2Fimages%2Fdims%3Fcrop%3D1599%252C1043%252C0%252C0%26quality%3D85%26format%3Djpg%26resize%3D1600%252C1043%26image_uri%3Dhttp%253A%252F%252Fo.aolcdn.com%252Fhss%252Fstorage%252Fmidas%252F8b98ec8f6649158fe7448ac2f2695ac5%252F205826072%252FXbox%252BOne%252BX%252Breview%252Bgallery%252B6.jpg%26client%3Da1acac3e1b3290917d92%26signature%3D353dad1308f98c2c9dfc82c58a540a8b2f1fe63c&amp;client=cbc79c14efcebee57402&amp;signature=60b7c061460d0d45f5d367b8a9c62978af6b76ce"/>
+                                                                <figcaption><span>Devindra Hardawar/AOL</span>
+                                                                </figcaption>
+                                                            </figure>
+                                                        </div>
+                                                <div>
+                                                                <p>The console's controller hasn't changed since its
+                                                                    last mini-upgrade with the Xbox One S. That revision
+                                                                    rounded out its seams, improved bumper performance
+                                                                    and added a 3.5mm headphone jack. It's still a great
+                                                                    controller, though I'm annoyed Microsoft is sticking
+                                                                    with AA batteries as their default power source.
+                                                                    Sure, you could just pick up some renewable
+                                                                    batteries, or the Play and Charge kit, but that's an
+                                                                    extra expense. And manually swapping batteries feels
+                                                                    like a bad user experience when every other console
+                                                                    has rechargeable controllers.</p>
+                                                                <h3>In use</h3>
+                                                            </div>
+                                                <div data-engadget-breakout-type="image">
+                                                            <figure><img src="https://o.aolcdn.com/images/dims?resize=980%2C640&amp;quality=100&amp;image_uri=https%3A%2F%2Fo.aolcdn.com%2Fimages%2Fdims%3Fcrop%3D1600%252C900%252C0%252C0%26quality%3D85%26format%3Djpg%26resize%3D1600%252C900%26image_uri%3Dhttp%253A%252F%252Fo.aolcdn.com%252Fhss%252Fstorage%252Fmidas%252F1885534bd201fc37481b806645c1fc8b%252F205828119%252FXbox%252Bone%252BX%252Bscreenshot%252Bgallery%252B8.jpg%26client%3Da1acac3e1b3290917d92%26signature%3Df63cf67c88b37fd9424855984e45f6b950c8c11a&amp;client=cbc79c14efcebee57402&amp;signature=0adca80fc8ee26a7353be639082881450a5ad49f"/>
+                                                                <figcaption><span>Devindra Hardawar/AOL</span>
+                                                                </figcaption>
+                                                            </figure>
+                                                        </div>
+                                                <div>
+                                                                <p>You won't find any major differences between the One
+                                                                    X and the last Xbox at first — aside from a more
+                                                                    dramatic startup sequence. Navigating the Xbox
+                                                                    interface is fast and zippy, but mostly that's due
+                                                                    to a recent OS upgrade. If you're moving over from
+                                                                    an older Xbox One, you can use the backup tool to
+                                                                    transfer your games and settings to an external hard
+                                                                    drive. Just plug that into the new console during
+                                                                    setup and it'll make it feel just like your old
+                                                                    machine. It's also a lot faster than waiting for
+                                                                    everything to download from Xbox Live.</p>
+                                                                <p>You'll still have to set aside some time if you want
+                                                                    to play an Xbox One X-enhanced title, though. Those
+                                                                    4K textures will make games significantly larger,
+                                                                    but Microsoft says it's come up with a few ways to
+                                                                    help developers make downloading them more
+                                                                    efficient. For example, language packs and other
+                                                                    optional content won't get installed by default.</p>
+                                                                <p>We only had a few enhanced titles to test out during
+                                                                    our review: <em>Gears of War 4</em>, <em>Killer
+                                                                        Instinct</em> and <em>Super Lucky's Tale</em>.
+                                                                    They each took advantage of the console in different
+                                                                    ways. <em>Gears of War 4</em> runs natively in 4K at
+                                                                    30 FPS with Dolby Atmos and HDR (high dynamic range
+                                                                    lighting) support. It looked great -- especially
+                                                                    with HDR, which highlighted bright elements like
+                                                                    lightning strikes -- but I noticed the frame rate
+                                                                    dip occasionally. I was also surprised that load
+                                                                    times were on-par with what I've seen with the game
+                                                                    on the Xbox One S.</p>
+                                                            </div>
+                                                <div data-engadget-breakout-type="e2ehero">
+                                                                            
+                                                                            <figure><img src="https://o.aolcdn.com/images/dims?crop=1600%2C900%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C900&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F8352a8a14e88e2ca2ba5be4d8381a055%2F205828115%2FXbox%2Bone%2BX%2Bscreenshot%2Bgallery%2B1.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=d2ccb22e0eaabeb05bfe46e83dbe26fd07f01da8"/>
+                                                                                
+                                                                            </figure>
+                                                                        </div>
+                                                <div>
+                                                                <p>You can also play in Performance mode, which bumps
+                                                                    the frame rate up to 60FPS and uses higher quality
+                                                                    graphical effects, while rendering it lower in
+                                                                    1080p. Personally, I preferred this, since it makes
+                                                                    the game much smoother -- as if you're playing it on
+                                                                    a high-end gaming PC, not a console. Some
+                                                                    PlayStation 4 Pro games also let you choose how you
+                                                                    wanted to distribute its power, so in some ways
+                                                                    Microsoft is just following in its footsteps.</p>
+                                                                <p>I've been playing <em>Gears of War 4</em> on my
+                                                                    gaming PC (which is connected to my home theater)
+                                                                    over the past year, and I was impressed that the
+                                                                    Xbox One X is able to deliver a similar experience.
+                                                                    It didn't quite match my rig though, which is
+                                                                    powered by Intel Core i7 4790k CPU running at 4GHz,
+                                                                    16GB DDR3 RAM and an NVIDIA GTX 1080 GPU. Typically,
+                                                                    I play at 1,440p (2,560 by 1,440 pixels) with HDR
+                                                                    and all of the graphical settings set to their
+                                                                    highest level, and I can easily maintain a 60FPS
+                                                                    frame rate. The One X felt just as solid at 1080p,
+                                                                    but there were clearly plenty of graphics settings
+                                                                    it couldn't take advantage of, in particular higher
+                                                                    levels of bloom lighting and shadow detail.</p>
+                                                                
+                                                            </div>
+                                                
+                                                <section data-engadget-breakout-type="gallery">
+                                                                <h3> Gallery: Xbox
+                                                                    One X screenshots | 9 Photos </h3>
+                                                                <div data-behavior="lightbox_trigger" data-engadget-slideshow-id="803330" data-eng-bang="{&quot;gallery&quot;:803330,&quot;slide&quot;:7142924}" data-eng-mn="93511844"><p><a href="#" data-index="0" data-engadget-slide-id="7142924" data-eng-bang="{&quot;gallery&quot;:803330,&quot;slide&quot;:7142924}">
+                                                                    <img src="https://o.aolcdn.com/images/dims?thumbnail=980%2C653&amp;quality=80&amp;image_uri=https%3A%2F%2Fs.blogcdn.com%2Fslideshows%2Fimages%2Fslides%2F714%2F292%2F4%2FS7142924%2Fslug%2Fl%2Fxbox-one-x-screenshot-gallery-2-1.jpg&amp;client=cbc79c14efcebee57402&amp;signature=38c95635c7aad58a8a48038e05589f5cf35b1e28"/>
+                                                                </a></p>
+                                                                    
+                                                                </div>
+                                                            </section>
+                                                <div>
+                                                                <p><em>Killer Instinct</em> and <em>Super Lucky's
+                                                                    Tale</em> run in 4K at a smooth 60FPS. They both
+                                                                    looked and played better than their standard
+                                                                    versions, though I was surprised they didn't take
+                                                                    advantage of HDR. As usual, I noticed the
+                                                                    improvement in frame rates more than the higher
+                                                                    resolution. Unless you're sitting very close to a TV
+                                                                    above 50-inches, you'd likely have a hard time
+                                                                    telling between 4K and 1080p.</p>
+                                                                <p>That poses a problem for Microsoft: It's betting that
+                                                                    gamers will actually want true 4K rendering. In
+                                                                    practice, though, PlayStation 4 Pro titles running
+                                                                    in HDR and resolutions between 1080p and 4K often
+                                                                    look just as good to the naked eye. The Xbox One X's
+                                                                    big advantage is that its hardware could let more
+                                                                    games reach 60FPS compared to Sony's console.</p>
+                                                                <p>Microsoft says over 130 Xbox One X-enhanced titles
+                                                                    are in the works. That includes already-released
+                                                                    games like <em>Forza Motorsport 7</em> and <em>Assassin's
+                                                                        Creed Origins</em>, as well as upcoming titles
+                                                                    like <em>Call of Duty: WW2</em>. You'll be able to
+                                                                    find them easily in a special section in the Xbox
+                                                                    store. There is also a handful of Xbox 360 games
+                                                                    that'll get enhanced eventually, including <em>Halo
+                                                                        3</em> and <em>Fallout 3</em>. Some of those
+                                                                    titles will get bumped up to a higher resolution,
+                                                                    while others will get HDR support. Microsoft
+                                                                    describes these upgrades as a bonus for developers
+                                                                    who were prescient about how they built their games.
+                                                                    Basically, don't expect your entire 360 library to
+                                                                    get enhanced.</p>
+                                                            </div>
+                                                <div data-engadget-breakout-type="e2ehero">
+                                                                            
+                                                                            <figure><img src="https://o.aolcdn.com/images/dims?crop=1600%2C900%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C900&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2Facb08903fbe26ad77b80db8c8e7e8fb1%2F205828118%2FXbox%2Bone%2BX%2Bscreenshot%2Bgallery%2B7.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=21630fa5ec6d8fdce2c35f7e1f652636a2d8efe7"/>
+                                                                                
+                                                                            </figure>
+                                                                        </div>
+                                                <div>
+                                                                <p>Even if a game isn't specifically tuned for the new
+                                                                    console, Microsoft says you might still see some
+                                                                    performance improvements. The PlayStation 4 Pro,
+                                                                    meanwhile, has over one hundred games built for its
+                                                                    hardware, and its boost mode can speed up some older
+                                                                    games.</p>
+                                                                <p>Microsoft is still pushing the Xbox as more than just
+                                                                    a game console, though. 4K Blu-rays loaded up
+                                                                    quickly, and I didn't notice many delays as I
+                                                                    skipped around films. <em>Planet Earth II</em>, in
+                                                                    particular, looked fantastic thanks to its brilliant
+                                                                    use of HDR. Unfortunately, the One X doesn't support
+                                                                    Dolby Vision, so you're stuck with the slightly less
+                                                                    capable HDR 10 standard. That makes sense since it's
+                                                                    more widely supported, but it would have been nice
+                                                                    to see Dolby's, too.</p>
+                                                                <p>
+                                                                    <iframe allowfullscreen="" frameborder="0" gesture="media" height="360" src="https://www.youtube.com/embed/c8aFcHFu8QM" width="480" aspect-ratio="auto"><empty></empty></iframe>
+                                                                </p>
+                                                                <p>And speaking of Dolby technology, Microsoft is also
+                                                                    highlighting Atmos support on the One X, just like
+                                                                    it did with the One S. The company's app lets you
+                                                                    configure the console to pass audio Atmos signals to
+                                                                    your audio receiver. You can also shell out $15 to
+                                                                    get Atmos support for headphones, which simulates
+                                                                    immersive surround sound. It's strange to pay money
+                                                                    to unlock Dolby features, but it's worth it since
+                                                                    it's significantly better than Microsoft's audio
+                                                                    virtualization technology. The Netflix app also
+                                                                    supports Atmos for a handful of films (something
+                                                                    that the Xbox One S and PlayStation 4 offer, as
+                                                                    well).</p>
+                                                                <p>One thing you won't find in the new Xbox is VR
+                                                                    support. Microsoft has mentioned that the console
+                                                                    will offer some sort of mixed reality, but it hasn't
+                                                                    offered up any details yet. It's technically
+                                                                    powerful enough to work with any of the Windows
+                                                                    Mixed Reality headsets launching this fall. It's a
+                                                                    shame that Microsoft is being so wishy-washy because
+                                                                    Sony has had a very successful head start with the
+                                                                    PlayStation VR.</p>
+                                                                <h3>Pricing and the competition</h3>
+                                                            </div>
+                                                <div data-engadget-breakout-type="image">
+                                                            <figure><img src="https://o.aolcdn.com/images/dims?resize=980%2C640&amp;quality=100&amp;image_uri=https%3A%2F%2Fo.aolcdn.com%2Fimages%2Fdims%3Fcrop%3D1600%252C1027%252C0%252C0%26quality%3D85%26format%3Djpg%26resize%3D1600%252C1027%26image_uri%3Dhttp%253A%252F%252Fo.aolcdn.com%252Fhss%252Fstorage%252Fmidas%252Fa2c8ba1caccdbb9e0559797e5141eafd%252F205826078%252FXbox%252BOne%252BX%252Breview%252Bgallery%252B11.jpg%26client%3Da1acac3e1b3290917d92%26signature%3Da11bcddced805c6e3698f8ce0494102aef057265&amp;client=cbc79c14efcebee57402&amp;signature=1e9bd192add2772bc842a34e67b7572cfd1b265a"/>
+                                                                <figcaption><span>Devindra Hardawar/AOL</span>
+                                                                </figcaption>
+                                                            </figure>
+                                                        </div>
+                                                <div>
+                                                                <p>The biggest knock against the Xbox One X is its $500
+                                                                    price. The PS4 Pro launched at $400 last year, and
+                                                                    there's a good chance we'll see plenty of deals
+                                                                    around the holidays. If your friends are on Xbox
+                                                                    Live, or you're a devotee of Microsoft's first party
+                                                                    franchises, then the X makes more sense. If you just
+                                                                    want to play third-party titles that come to both
+                                                                    platforms, though, the PS4 Pro is clearly the better
+                                                                    deal.</p>
+                                                                <p>If you're looking to upgrade from an original Xbox
+                                                                    One, and you have a new TV, the One X might be more
+                                                                    compelling. It's faster and offers more features
+                                                                    than the One S, and more importantly, it'll last you
+                                                                    much longer without needing an upgrade. There's also
+                                                                    plenty of wisdom in simply waiting a while before
+                                                                    you buy the One X, especially if you haven't moved
+                                                                    to a 4K TV yet. The new console can make games look
+                                                                    better on 1080p sets, since it'll supersample
+                                                                    high-res textures and have more graphical effects,
+                                                                    but it's simply not worth the upgrade since those
+                                                                    TVs don't support HDR.</p>
+                                                                <p>If price isn't a huge concern for you, it's worth
+                                                                    considering investing in a gaming PC. A decent one
+                                                                    costs between $600 and $800, plus the price of a
+                                                                    monitor, but it'll easily be more powerful than the
+                                                                    One X. And you have the added benefit of upgrading
+                                                                    components down the line. Now that Microsoft and
+                                                                    game publishers are offering most major titles on
+                                                                    PC, you won't be missing out on much by ditching
+                                                                    consoles.</p>
+                                                                <h3>Wrap-up</h3>
+                                                                <p><img data-credit="Devindra Hardawar/AOL" data-mep="2181681" src="https://o.aolcdn.com/images/dims?crop=1600%2C1028%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C1028&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F5396460ef8b6bde7fb7272d9e66a7701%2F205826076%2FXbox%2BOne%2BX%2Breview%2Bgallery%2B9.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=f5b5b4b986c2f8b5031a4469ae0ecec82aff65b0" alt=""/></p>
+                                                                <p>Ultimately, the Xbox One X offers some major
+                                                                    performance upgrades that gamers will notice --
+                                                                    especially if you're coming from an original Xbox
+                                                                    One. But it's also a bit disappointing since it's
+                                                                    coming a year after the PS4 Pro, and it doesn't
+                                                                    offer VR yet. For Microsoft fans, though, none of
+                                                                    that will matter. It's exactly what the company
+                                                                    promised: the fastest game console ever made.</p>
+                                                            </div>
+                                                
+                                                
+                                            </div></DIV></article>

article_scraper/resources/tests/readability/firefox-nightly-blog/expected.html

@@ -0,0 +1,292 @@
+<article><div id="readability-page-1">
+                    <article id="post-997">
+                        
+                        <div>
+                            <h3>
+                                Highlights
+                            </h3>
+                            <ul>
+                                <li>
+                                    <a href="https://blog.mozilla.org/blog/2020/12/15/our-year-in-review-how-weve-kept-firefox-working-for-you-in-2020/" target="_blank">Here’s our Firefox Year in Review!</a>
+                                </li>
+                                <li>
+                                    <a href="https://blog.mozilla.org/performance/2020/12/15/2020-year-in-review/" target="_blank">Here’s our Performance Year in Review!</a>
+                                </li>
+                                <li>We’ve just landed <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1553982" target="_blank">Bug 1553982</a>, which aims to prevent starting an update while another Firefox instance is running (the cause of that about:restartrequired error page you may have seen).
+                                    <ul>
+                                        <li>
+                                            <div id="attachment_994">
+                                                <p><a href="https://3sgkpvh31s44756j71xlti9b-wpengine.netdna-ssl.com/files/2020/12/headlines85_0.png" target="_blank"><img aria-describedby="caption-attachment-994" src="https://3sgkpvh31s44756j71xlti9b-wpengine.netdna-ssl.com/files/2020/12/headlines85_0.png" alt="The about:restartrequired error page, saying &quot;Sorry. We just need to do one small thing to keep going. Nightly has just been updated in the background. Click Restart Nightly to complete the update. We will restore all your pages, windows and tabs afterwards, so you can be on your way quickly.&quot;, followed by a button to restart Nightly." width="1600" height="805" srcset="https://blog.nightly.mozilla.org/files/2020/12/headlines85_0.png 1600w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_0-300x151.png 300w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_0-600x302.png 600w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_0-768x386.png 768w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_0-1536x773.png 1536w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_0-1000x503.png 1000w" sizes="(max-width: 1600px) 100vw, 1600px"/></a></p><p id="caption-attachment-994">
+                                                    Users who run multiple user profiles concurrently will probably see this less!
+                                                </p>
+                                            </div>
+                                        </li>
+                                    </ul>
+                                </li>
+                                <li>Also just about to land is <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=353804" target="_blank">Bug 353804</a>, which provides some support for downloading new updates when we already have an update downloaded but haven’t installed it yet. That should prevent many cases of restarting to finish an update and then immediately being notified about another one.
+                                </li>
+                                <li>Thanks to evilpie, users can now <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1650645" target="_blank">import logins from Keepass(XC) into Firefox</a>
+                                </li>
+                                <li>From Firefox 85 it’s now possible to disable tab-to-search on a per-engine basis, by unchecking a search engine in <i>Search Preferences</i>. That will both hide the shortcut button and disable tab-to-search for the engine. (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1681512" target="_blank">Bug 1681512</a>)
+                                </li>
+                                <li>From Firefox 85 it’s also possible to disable tab-to-search globally by unchecking the <i>Search Engines</i> checkbox in the <i>Address Bar Preferences</i>, under <i>Privacy &amp; Security</i>.
+                                </li>
+                                <li>Firefox now supports printing non-contiguous page ranges (e.g. 1-3, 6, 7) – <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=499640" target="_blank">Bug 499640</a>
+                                </li>
+                                <li>DevTools and Marionette are now fully Fission compatible! Congratulations to those teams!
+                                    <ul>
+                                        <li>Reminder: Nightly users can help us test Fission by enabling it in about:preferences#experimental, and <a href="https://bugzilla.mozilla.org/enter_bug.cgi?assigned_to=nobody%40mozilla.org&amp;blocked=1561396&amp;bug_ignored=0&amp;bug_severity=--&amp;bug_status=NEW&amp;bug_type=defect&amp;cf_a11y_review_project_flag=---&amp;cf_fission_milestone=---&amp;cf_fx_iteration=---&amp;cf_fx_points=---&amp;cf_root_cause=---&amp;cf_status_firefox83=---&amp;cf_status_firefox84=---&amp;cf_status_firefox85=---&amp;cf_status_firefox86=---&amp;cf_status_firefox_esr78=---&amp;cf_status_thunderbird_esr78=---&amp;cf_tracking_firefox84=---&amp;cf_tracking_firefox85=---&amp;cf_tracking_firefox86=---&amp;cf_tracking_firefox_esr78=---&amp;cf_tracking_firefox_relnote=---&amp;cf_tracking_firefox_sumo=---&amp;cf_tracking_thunderbird_esr78=---&amp;cf_webcompat_priority=---&amp;component=DOM%3A%20Navigation&amp;contenttypemethod=list&amp;contenttypeselection=text%2Fplain&amp;defined_groups=1&amp;filed_via=standard_form&amp;flag_type-203=X&amp;flag_type-37=X&amp;flag_type-41=X&amp;flag_type-607=X&amp;flag_type-721=X&amp;flag_type-737=X&amp;flag_type-787=X&amp;flag_type-799=X&amp;flag_type-800=X&amp;flag_type-803=X&amp;flag_type-846=X&amp;flag_type-855=X&amp;flag_type-863=X&amp;flag_type-864=X&amp;flag_type-930=X&amp;flag_type-936=X&amp;flag_type-937=X&amp;flag_type-945=X&amp;form_name=enter_bug&amp;maketemplate=Remember%20values%20as%20bookmarkable%20template&amp;op_sys=Unspecified&amp;priority=--&amp;product=Core&amp;rep_platform=Unspecified&amp;target_milestone=---&amp;version=unspecified" target="_blank">filing bugs here</a>
+                                        </li>
+                                    </ul>
+                                </li>
+                            </ul>
+                            <h3>
+                                Friends of the Firefox team
+                            </h3>
+                            <h4>
+                                Introductions/Shout-Outs
+                            </h4>
+                            <ul>
+                                <li>[harry] Amy Churchwell joins the Search &amp; Navigation team today. She transferred internally from Marketing Engineering. Welcome Amy!
+                                </li>
+                            </ul>
+                            <h4>
+                                <a href="https://bugzilla.mozilla.org/buglist.cgi?title=Resolved%20bugs%20(excluding%20employees)&amp;quicksearch=1647931%2C1649618%2C1650645%2C1652412%2C1654217%2C1664768%2C1666831%2C1667061%2C1671579%2C1674806%2C1678173%2C1678372%2C1678616%2C1678865%2C1678866%2C1679252%2C1679412%2C1680931%2C1681213%2C1681554%2C1681642%2C1681948" target="_blank">Resolved bugs (excluding employees)</a>
+                            </h4>
+                            <h4>
+                                Fixed more than one bug
+                            </h4>
+                            <ul>
+                                <li>Masatoshi Kimura [:emk]
+                                </li>
+                                <li>Michelle Goossens [:masterwayz]
+                                </li>
+                                <li>Sonia
+                                </li>
+                                <li>Tim Nguyen :ntim
+                                </li>
+                            </ul>
+                            <h4>
+                                New contributors (🌟 = first patch)
+                            </h4>
+                            <ul>
+                                <li>🌟 Ankush Dua <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1671579" target="_blank">fixed an issue with revoked devtools_page permissions for WebExtensions</a>
+                                </li>
+                                <li>🌟 gero <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1674806" target="_blank">removed the windowtype attribute</a> from dialogs where we didn’t need it anymore
+                                </li>
+                                <li>manekenpix <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1664768" target="_blank">converted some DevTools code</a> to use DOM Promises instead of the defer library
+                                </li>
+                            </ul>
+                            <h3>
+                                Project Updates
+                            </h3>
+                            <h4>
+                                Add-ons / Web Extensions
+                            </h4>
+                            <h5>
+                                Addon Manager &amp; about:addons
+                            </h5>
+                            <ul>
+                                <li>Starting from Firefox 85, Mozilla-signed privileged addons can be installed from a third party website without triggering the “third party addon install doorhanger” (and without having to add new “install” site permission for those hosts, <a href="https://searchfox.org/mozilla-central/rev/23c25cd32a1e87095301273937b4ee162f41e860/browser/app/permissions#24-25" target="_blank">e.g. as we had to do for fpn.firefox.com</a>) – <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1681331" target="_blank">Bug 1681331</a>
+                                </li>
+                            </ul>
+                            <ul>
+                                <li>Fixed addon startup issue when an extension sideloaded in the profile is updated on disk (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1664144" target="_blank">Bug 1664144</a>)
+                                </li>
+                                <li>Some more small about:addons cleanup from ntim (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1678173" target="_blank">Bug 1678173</a>, <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1678865" target="_blank">Bug 1678865</a>, <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1678866" target="_blank">Bug 1678866</a>). Thanks a lot, ntim!
+                                </li>
+                            </ul>
+                            
+                            <h5>
+                                WebExtensions Framework
+                            </h5>
+                            <ul>
+                                <li>
+                                    <b>Ankush Dua</b> <i>contributed a fix for the devtools optional_permission</i> (the devtools optional_permission can be used by extension, like ABP, that provides a devtools panel as a secondary feature of the addon) – <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1671579" target="_blank">Bug 1671579</a>
+                                </li>
+                                <li>Fixed content scripts applied to webpages loaded as subframes of an extension browserAction/pageAction popup when Fission is enabled – <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1680877" target="_blank">Bug 1680877</a>
+                                </li>
+                                <li>Fixed addon startup issue when webRequest is moved from permissions to optional_permissions in an addon update (regression from <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1624235" target="_blank">Bug 1624235</a>) – <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1637059" target="_blank">Bug 1637059</a>
+                                </li>
+                            </ul>
+                            <h4>
+                                Developer Tools
+                            </h4>
+                            <ul>
+                                <li>
+                                    <b>DevTools Fission</b> <b>M2</b> – Making DevTools Fission compatible DONE.
+                                    <ul>
+                                        <li>
+                                            <div id="attachment_995">
+                                                <p><a href="https://3sgkpvh31s44756j71xlti9b-wpengine.netdna-ssl.com/files/2020/12/headlines85_1.png" target="_blank"><img aria-describedby="caption-attachment-995" src="https://3sgkpvh31s44756j71xlti9b-wpengine.netdna-ssl.com/files/2020/12/headlines85_1.png" alt="A table showing the total number of remaining bugs for the MVP to make the DevTools Fission-compatible." width="1600" height="192" srcset="https://blog.nightly.mozilla.org/files/2020/12/headlines85_1.png 1600w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_1-300x36.png 300w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_1-600x72.png 600w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_1-768x92.png 768w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_1-1536x184.png 1536w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_1-1000x120.png 1000w" sizes="(max-width: 1600px) 100vw, 1600px"/></a></p><p id="caption-attachment-995">
+                                                    Our DevTools are ready for Fission (out-of-process iframes)!
+                                                </p>
+                                            </div>
+                                        </li>
+                                    </ul>
+                                </li>
+                                <li>
+                                    <b>Marionette Fission</b> – Making Marionette Fission compatible DONE
+                                    <ul>
+                                        <li>
+                                            <div id="attachment_996">
+                                                <p><a href="https://3sgkpvh31s44756j71xlti9b-wpengine.netdna-ssl.com/files/2020/12/headlines85_2.png" target="_blank"><img aria-describedby="caption-attachment-996" src="https://3sgkpvh31s44756j71xlti9b-wpengine.netdna-ssl.com/files/2020/12/headlines85_2.png" alt="A table showing the total number of remaining bugs for the MVP to make Marionette Fission-compatible." width="1600" height="189" srcset="https://blog.nightly.mozilla.org/files/2020/12/headlines85_2.png 1600w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_2-300x35.png 300w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_2-600x71.png 600w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_2-768x91.png 768w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_2-1536x181.png 1536w, https://blog.nightly.mozilla.org/files/2020/12/headlines85_2-1000x118.png 1000w" sizes="(max-width: 1600px) 100vw, 1600px"/></a></p><p id="caption-attachment-996">
+                                                    Marionette, the framework that allows Firefox to be tested with automation, is now Fission compatible too!
+                                                </p>
+                                            </div>
+                                        </li>
+                                    </ul>
+                                </li>
+                            </ul>
+                            <h4>
+                                Fission
+                            </h4>
+                            <ul>
+                                <li>Neil has patches up to <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1682442" target="_blank">improve the behaviour of the tab unloader</a>, and <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1644911" target="_blank">show UI when subframes crash</a>
+                                </li>
+                            </ul>
+                            <h4>
+                                Installer &amp; Updater
+                            </h4>
+                            <ul>
+                                <li>Background updater work is also proceeding, with <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1676296" target="_blank">Bug 1676296</a> landing last week to support managing scheduled tasks in Gecko, and more development still also happening on the background task framework.
+                                </li>
+                            </ul>
+                            <h4>
+                                New Tab Page and Pocket
+                            </h4>
+                            <ul>
+                                <li>We’re running three experiments:
+                                    <ul>
+                                        <li>Newtab Pocket stories in AU and NZ
+                                        </li>
+                                        <li>New signup/login call-to-action in the Pocket doorhanger
+                                        </li>
+                                        <li>We’re testing some changes to newtab story personalization
+                                        </li>
+                                    </ul>
+                                </li>
+                            </ul>
+                            <h4>
+                                <a href="https://wiki.mozilla.org/Toolkit:Password_Manager" target="_blank">Password Manager</a>
+                            </h4>
+                            <ul>
+                                <li>Dimi fixed <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1677710" target="_blank">Bug 1677710</a> The password manager code triggers main thread sqlite disk I/O off of the gather-telemetry notification
+                                </li>
+                                <li>And <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1678200" target="_blank">Bug 1678200</a> Remove or update probes expiring in Firefox 86: pwmgr.doorhanger_submitted#doorhanger_submitted
+                                </li>
+                                <li>Thanks for Kenrick95 for fixing <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1678616" target="_blank">Bug 1678616</a> about:logins menu problem
+                                </li>
+                                <li>2021 Planning underway
+                                </li>
+                            </ul>
+                            <h4>
+                                PDFs &amp; Printing
+                            </h4>
+                            <ul>
+                                <li>mstriemer put a Printing… message in the dialog and hid the popup dialog which showed progress, the cancel button on that dialog caused problems and it looked dated <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1679133" target="_blank">Bug 1679133</a>
+                                </li>
+                                <li>mstriemer hid the print setting that don’t relate to PDFs when a PDF is being printed <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1669725" target="_blank">Bug 1669725</a>
+                                </li>
+                                <li>mstriemer updated the form to be disabled when loading a printer’s settings. Sometimes loading a physical printer’s settings can take a few settings and changes could be lost in this time <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1676388" target="_blank">Bug 1676388</a>
+                                </li>
+                                <li>emalysz made a change to avoid updating the preview for some settings that can’t change the preview output <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1676199" target="_blank">Bug 1676199</a>
+                                </li>
+                                <li>sfoster added a paginator to the preview when it’s hovered to show current page, next/prev/first/last buttons <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1654684" target="_blank">Bug 1654684</a>
+                                </li>
+                                <li>emalysz added support for non-contiguous page ranges (ex: 1-3, 6, 7) <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=499640" target="_blank">Bug 499640</a>
+                                </li>
+                                <li>emalysz fixed an issue where the form could get disabled with custom margins interactions <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1674106" target="_blank">Bug 1674106</a>
+                                </li>
+                            </ul>
+                            <h4>
+                                Performance
+                            </h4>
+                            <ul>
+                                <li>emalysz continues to make progress leading the charge migrating us from OS.File to IOUtils
+                                    <ul>
+                                        <li>Shout out to barret for landing necessary changes to IOUtils to support the migration!
+                                        </li>
+                                        <li>Currently investigating <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1679252" target="_blank">a bizarre ts_paint_webext regression</a> caused by one of these conversions
+                                        </li>
+                                    </ul>
+                                </li>
+                                <li>emalysz <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1664444" target="_blank">fixed a performance issue with the Screenshots feature</a>, and made it more compatible with Fission
+                                </li>
+                                <li>bigiri has <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1649610" target="_blank">a patch to move SharedDataMap.jsm off of OSFile</a>
+                                </li>
+                                <li>florian’s team has <a href="https://firefox-source-docs.mozilla.org/tools/profiler/markers-guide.html" target="_blank">landed some great documentation</a> for the new profiler marker API
+                                </li>
+                                <li>florian has some new visualization variations up for the BHR dashboard
+                                    <ul>
+                                        <li>
+                                            <a href="http://queze.net/bhr/test/#showFrames=1" target="_blank">showFrames</a>
+                                        </li>
+                                        <li>
+                                            <a href="http://queze.net/bhr/test/#showFrames=1&amp;onlyXulLeaf=1" target="_blank">onlyXulLeaf</a>
+                                        </li>
+                                        <li>
+                                            <a href="http://queze.net/bhr/test/#showFrames=1&amp;onlyXulLeaf=1&amp;skipKnownBugs=1" target="_blank">skipKnownBugs</a>
+                                        </li>
+                                        <li>This BHR dashboard helped identify a hang caused by the password manager code, <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1677710" target="_blank">which has been recently fixed</a>! Thanks, dimi!
+                                        </li>
+                                    </ul>
+                                </li>
+                                <li>Gijs <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1667237" target="_blank">made the Bookmarks Toolbar initialization occur later in the startup window</a>
+                                </li>
+                                <li>Gijs <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1681169" target="_blank">fixed some flicker</a> that occurred when launching the browser with the Bookmarks Toolbar enabled
+                                </li>
+                                <li>mconley fixed <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1673716" target="_blank">an AsyncShutdown hang caused by the about:home startup cache</a>
+                                </li>
+                                <li>mconley <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1651311" target="_blank">re-enabled TART</a>
+                                </li>
+                                <li>dthayer has <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1672789" target="_blank">some fixes</a> and <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1678488" target="_blank">polish</a> for <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1680258" target="_blank">the pre-XUL skeleton UI</a>
+                                </li>
+                            </ul>
+                            <h4>
+                                Picture-in-Picture
+                            </h4>
+                            <ul>
+                                <li>We’ve got the green light for another round of MSU students hacking on Picture-in-Picture next semester! mhowell and mtigley will be mentoring them.
+                                </li>
+                                <li>In progress:
+                                    <ul>
+                                        <li>
+                                            <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1677080" target="_blank">Bug 1677080 – Fullscreen PiP window is affected by switching video source</a>
+                                        </li>
+                                        <li>
+                                            <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1677107" target="_blank">Bug 1677107 – Add Telemetry for tracking multiple PiP support usage</a>
+                                        </li>
+                                        <li>
+                                            <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1681796" target="_blank">Bug 1681796 – Prevent superfluous PictureInPictureParent actors from being associated with each tab</a>
+                                        </li>
+                                        <li>
+                                            <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1680796" target="_blank">Bug 1680796 – Ensure that the tab’s Toolkit:PictureInPicture actor is destroyed before moving to next test</a>
+                                        </li>
+                                        <li>
+                                            <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1678390" target="_blank">Bug 1678390 – Prevent Picture-in-Picture windows from opening on top of one another</a>
+                                        </li>
+                                    </ul>
+                                </li>
+                            </ul>
+                            <h4>
+                                Search and Navigation
+                            </h4>
+                            <ul>
+                                <li>Fixed regressions related to Input Method Editor, in particular loss of the last token (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1673669" target="_blank">Bug 1673669</a>) and race conditions causing the wrong search engine to be used or Search Mode to be lost (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1679697" target="_blank">Bug 1679697</a>, <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1678647" target="_blank">Bug 1678647</a>)
+                                </li>
+                                <li>Introduced a new advanced preference to keep the Address Bar results panel open during IME composition. This provides a better experience for keyboard layouts that don’t open a picker panel. In the future we hope to be able to auto-detect that situation, but in the meanwhile, you can flip <i>browser.urlbar.imeCompositionClosesPanel</i> to false and test the alternative behavior (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1673971" target="_blank">Bug 1673971</a>)
+                                </li>
+                                <li>URL canonization (<a href="about:blank" target="_blank">www.*.com</a>) now uses https by default, the protocol can be customized through the <i>browser.fixup.alternate.protocol</i> advanced pref (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1638215" target="_blank">Bug 1638215</a>)
+                                </li>
+                                <li>Work continued on the weather QuickSuggest experiment, but its release has been moved to January.
+                                </li>
+                                <li>Region.jsm now can use a Geolocation monitor to update without hitting the network (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1663501" target="_blank">Bug 1663501</a>)
+                                </li>
+                                <li>Fixed a bug where search engines were being re-added on startup after their removal, when using a language pack (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1675624" target="_blank">Bug 1675624</a>)
+                                </li>
+                            </ul>
+                        </div>
+                        
+                    </article>
+                </div></article>

article_scraper/resources/tests/readability/folha/expected.html

@@ -0,0 +1,24 @@
+<article><div data-share-text="" data-news-content-text="" data-disable-copy="" data-continue-reading="" data-continue-reading-hide-others=".js-continue-reading-hidden" itemprop="articleBody" id="readability-page-1">
+                                                <p>
+                                                    Após rechaçar <a href="https://www1.folha.uol.com.br/esporte/2018/12/tite-se-recusa-a-encontrar-bolsonaro-antes-da-disputa-da-copa-america.shtml" target="_blank">um encontro da seleção brasileira com o presidente eleito Jair</a> <a href="https://www1.folha.uol.com.br/esporte/2018/12/tite-se-recusa-a-encontrar-bolsonaro-antes-da-disputa-da-copa-america.shtml" target="_blank">Bolsonaro</a>, o técnico Tite declarou que errou ao levar a taça da Copa Libertadores de 2012, conquistada pelo Corinthians, ao ex-presidente Luiz Inácio Lula da Silva.
+                                                </p>
+                                                <p>
+                                                    Ao lado de representantes do clube paulista, o atual comandante do Brasil ainda entregou uma réplica do troféu a Lula.
+                                                </p>
+                                                <p>
+                                                    "Em 2012 eu errei. Ele não era presidente, mas fui ao Instituto e mandei felicitações por um aniversário. Não me posicionei politicamente. Não tenho partido político, tenho sim a torcida para que o Brasil seja melhor em igualdade social. E que nossas prioridades sejam educação e punição. Que seja dada a possibilidade de estudo ao garoto de São Braz, que não tem chão batido para ir à escola, ou da periferia de Caixas ou do morro do Rio de Janeiro. Seja dada a ele a prioridade de estudo e não a outras situações", falou Tite ao programa "Grande Círculo", que ainda irá ao ar no SporTV.
+                                                </p>
+                                                <p>
+                                                    Na ocasião, Tite e outros representantes do Corinthians <a href="https://www1.folha.uol.com.br/poder/1124743-corinthians-leva-a-taca-da-libertadores-para-lula.shtml" target="_blank">foram ao Instituto Lula para mostrar a taça</a> original da Libertadores ao ex-presidente.
+                                                </p>
+                                                
+                                                <p>
+                                                    O assunto foi levantado porque recentemente Tite foi questionado se aceitaria um encontro da seleção brasileira com Bolsonaro em uma conquista de título ou <a href="https://www1.folha.uol.com.br/esporte/2018/12/selecao-brasileira-jogara-duas-vezes-em-sao-paulo-na-copa-america.shtml" target="_blank">antes da Copa América de 2019</a>, por exemplo. O treinador deixou claro que preferiria evitar esse tipo de formalidade.
+                                                </p>
+                                                <p>
+                                                    Apesar disso, Tite não questionou a ação de Palmeiras e CBF, que <a href="https://www1.folha.uol.com.br/esporte/2018/12/cbf-usa-festa-do-palmeiras-para-se-aproximar-de-governo-bolsonaro.shtml" target="_blank">convidaram Bolsonaro para a festa do título do Campeonato Brasileiro</a>. O presidente eleito até levantou a taça conquistada pelo clube alviverde.
+                                                </p>
+                                                <p>
+                                                    "Em 2012 eu fiz e errei. O protocolo e a situação gerada no jogo do Palmeiras são fatos de opinião pessoal. CBF e Palmeiras, enquanto instituições têm a opinião. Errei lá atrás, não faria com o presidente antes da Copa e nem agora porque entendo que misturar esporte e política não é legal. Fiz errado lá atrás? Sim. Faria de novo? Não", acrescentou o comandante.
+                                                </p>
+                                            </div></article>

article_scraper/resources/tests/readability/gmw/expected.html

@@ -0,0 +1,57 @@
+<article><div id="readability-page-1">
+                
+                
+                
+                <p>　　翱翔于距地球数千公里的太空中，进入广袤漆黑的未知领域，是一项艰苦卓绝的工作。这让人感到巨大压力和极度恐慌。那么，为什么不能让宇航员来一杯“地球末日”鸡尾酒来放松一下？</p>
+                <p>　　不幸的是，对于希望能喝上一杯的太空探险者，那些将他们送上太空的政府机构普遍禁止他们染指包括酒在内的含酒精饮料。</p>
+                <p>　　但是，很快普通人都会有机会向人类“最终的边疆”出发——以平民化旅行的形式，去探索和殖民火星。确实，火星之旅将是一次令人感到痛苦的旅行，可能一去不复返并要几年时间才能完成，但是否应该允许参与者在旅程中痛饮一番？或至少携带能在火星上发酵自制酒精饮料的设备？</p>
+                <p><img id="45395168" alt="(Credit: Nasa)" src="http://imgtech.gmw.cn/attachement/jpg/site2/20170310/448a5bc1e2861a2c4e5929.jpg" title="&#x5B87;&#x822A;&#x5458;&#x5728;&#x592A;&#x7A7A;&#x4E2D;&#x559D;&#x9152;&#x4F1A;&#x600E;&#x4E48;&#x6837;&#xFF1F;&#x540E;&#x679C;&#x5F88;&#x4E25;&#x91CD;"/></p>
+                <p>
+                    <span face="&#x6977;&#x4F53;">　　图注：巴兹?奥尔德林(Buzz Aldrin)可能是第二个在月球上行走的人，但他是第一个在月球上喝酒的人</span>
+                </p>
+                <p>　　事实是，历史上酒与太空探险有一种复杂的关系。让我们来看看喝了酒的航天员究竟会发生什么—— 如果我们开始给予进入太空的人类更大的自由度，又可能会发生什么。</p>
+                <p>　　人们普遍认为，当一个人所处的海拔越高，喝醉后会越容易感到头昏。因此，人们自然地想到，当人身处地球轨道上时，饮酒会对人体有更强烈的致眩作用。但这种说法可能不是正确的。</p>
+                <p>　　事实上，有证据表明，早在上世纪八十年代就澄清了这一传言。1985年，美国联邦航空管理局（UFAA）开展了一项研究，以验证人在不同的海拔高度饮酒，是否会影响执行复杂任务时的表现和酒精测定仪的读数。</p>
+                <p>　　在这项研究中，17名男子被要求在地面和一间模拟海拔3.7公里的房间内喝下一些伏特加。然后，他们被要求完成各种任务，包括心算口算问题、用操纵杆在示波器上跟踪灯光以及各种其它测试。研究人员发现，“酒精和海拔高度对酒精测定仪读数或完成任务的表现情况没有交互作用”。</p>
+                <p>　　所以，人乘坐飞机时醉得更快是个传说？纽约州立大学（State University of New York，SUNY）社会学荣誉教授戴夫·汉森（Dave Hanson）研究酒精和饮酒超过40年，他认为确实如此。他说：“我不认为它（在太空中饮酒）会有任何不同。”</p>
+                <p>　　他认为高原反应可能类似于宿醉，但它也可能类似于中毒。他说：“如果人们没有感受到充分的大气压力，他们也会觉得喝醉了一样。”</p>
+                <p>　　相反，那些声称在飞机上比在地面上醉得更快的人，可能只是经历了“自认喝醉（think-drink）”效应，这种效应多年来已被广泛研究。它表明，如果人们认为自己喝醉了，那他们的一举一动会真的像喝醉了一样—— 而不是实际上他们真的醉了。</p>
+                <p>　　汉森指出：“如果人们脑子里一直认为在飞机上酒精会对他们产生与平常不同的作用，那么他们乘坐飞机时真的会觉得酒精对他们产生了不同的作用。”</p>
+                <p>　　所以，如果酒精对人体的物理效应与海拔高度无关，那么在国际空间站上睡前小饮一杯不应该是一个大问题，对吧？错了。</p>
+                <p>　　美国宇航局约翰逊航天中心发言人丹尼尔·霍特(Daniel Huot)表示：“国际空间站上的宇航员不允许喝酒。在国际空间站上，酒精和其它挥发性化合物的使用受到控制，因为它们的挥发物可能对该站的水回收系统产生影响。”</p>
+                <p>　　为此，国际空间站上的宇航员甚至没有被提供含有酒精的产品，例如漱口水、香水或须后水。如果在国际空间站上饮酒狂欢，溢出的啤酒也可能存在损坏设备的风险。</p>
+                <p><img id="45395150" alt="(Credit: iStock)" src="http://imgtech.gmw.cn/attachement/jpg/site2/20170310/448a5bc1e2861a2c4e592a.jpg" title="&#x5B87;&#x822A;&#x5458;&#x5728;&#x592A;&#x7A7A;&#x4E2D;&#x559D;&#x9152;&#x4F1A;&#x600E;&#x4E48;&#x6837;&#xFF1F;&#x540E;&#x679C;&#x5F88;&#x4E25;&#x91CD;"/></p>
+                <p>
+                    <span face="&#x6977;&#x4F53;">　　图注：测试表明，有关人在高空中喝酒更容易醉的传言是不正确的</span>
+                </p>
+                <p>　　然后是责任的问题。我们不允许汽车司机或飞机飞行员喝醉后驾驶，所以并不奇怪同样的规则适用于国际空间站上的宇航员。毕竟国际空间站的造价高达1500亿美元，而且在接近真空的太空中其运行速度达到了每小时27680公里。</p>
+                <p>　　然而，2007年，美国宇航局(NASA)成立了一个负责调查宇航员健康状况的独立小组，称历史上该机构至少有两名宇航员在即将飞行前喝了大量的酒，但仍然被允许飞行。Nasa安全负责人随后的审查发现并没有证据支持这一指控。宇航员在飞行前12小时是严禁饮酒的，因为他们需要充分的思维能力和清醒的意识。</p>
+                <p>　　出台这一规则的原因很清楚。在1985年UFAA开展的关于酒精在不同海拔高度影响的研究中，研究人员得出结论，酒精的影响与海拔高度无关。无论参与测试的人员在什么海拔高度喝酒，其酒精测量仪的读数都是一样的。他们的行为表现受到的影响也相同，但如果提供给测试人员的是安慰剂，则身处高空比身处海平面的行为表现要更差一些。这表明，无论是否摄入酒精，海拔高度可能对心理表现有轻微的影响。</p>
+                <p>　　国际空间站禁止享用啤酒等有大量泡沫的饮料，可能有另一个原因：没有重力的帮助，液体和气体会在宇航员的胃里不停地翻滚，导致他们不断地打嗝。</p>
+                <p>　　然而，尽管有严格的规则，这并不意味着太空中的人类不会接触发酵液体。在国际空间站上进行了大量有关酒精的实验—— 但没有发生让众人去饮酒的情况，所以没有人真正了解太空中人体对酒精具体有怎样的反应。</p>
+                <p>　　NASA发言人斯蒂芬妮?席尔霍尔茨(Stephanie Schierhol）表示：“我们研究了太空中宇航员身体的各种变化，包括微生物层面的。我们有一个营养计划，以确保他们的身体获得保持健康所需要的营养。显然，在实施‘天空实验室(skylab)’项目时，他们曾将雪利酒与宇航员一起送到太空中，但宇航员在零重力飞行时使用雪利酒的测试结果不太好。”天空实验室是美国第一座空间站。</p>
+                <p>　　席尔霍尔茨补充说，在测试中使用雪利酒“引发呕吐反射，公众也反对”。</p>
+                <p>　　也许最令人惊讶的是，人类在月球表面上喝的第一种液体是葡萄酒。前NASA宇航员巴兹·奥尔德林(Buzz Aldrin)在采访和他撰写的书中表示，1969年，在和尼尔·阿姆斯特朗(Neil Armstrong)走出登月舱之前的圣餐仪式上，他喝了少量葡萄酒。举行这一仪式时与地面的通信出现了暂停，因此这一过程从来没有播出。</p>
+                <p>　　虽然Nasa对太空中酒精的使用有严格的规定，但在这方面俄罗斯过去似乎更为宽松。在其“和平号”空间站上，宇航员允许喝点干邑和伏特加。当他们发现国际空间站将严格禁止饮酒时，显然有不少怨言。</p>
+                <p>　　然而，奇怪的是，酒仍然能通过各种方式出现在国际空间站上。2015年，日本酿酒商三得利(Suntory)的全球创新中心将该公司一些获奖的威士忌运送到国际空间站，参与一项旨在验证“能否通过利用微重力环境增强酒精饮料醇厚性”的实验。换句话说，在微重力下酒的陈酿过程可能不同，导致它的陈酿进程更快、味道更好。对此，地球上的每家酿酒商都想进一步地了解。</p>
+                <p>　　几年前，即2011年9月至2014年9月，Nasa赞助了一个试验，研究微重力环境对威士忌中未发酵麦芽与烧焦橡木颗粒的影响，这两种物质能对威士忌的陈酿起帮助作用。在太空中逗留将近1000天后，用于测试的威士忌的单宁成分保持不变——但是太空中橡木颗粒产生了更高浓度的木质素分解产物，这种物质能赋予威士忌特别的风味。</p>
+                <p>　　Nasa表示：“这种试验不仅对麦芽威士忌行业有影响，而且对整个食品和饮料行业也有影响。送上太空的威士忌与对照样品之间的风味差异是如此显著，需要进一步分析以破解不同口味产生的原因。”</p>
+                <p>　　因此，即使宇航员自己被禁止在地球轨道上饮酒，但他们正在做的工作可以提高在地上消费的酒的质量。</p>
+                <p>　　相比之下，执行登陆火星任务的人将远离家乡几年，而不是几个月，因此可能会有人提出有关禁止饮酒的规定可以放松一些。</p>
+                <p>　　然而，像戴夫?汉森这样的专家认为，继续禁止饮酒并没有什么害处。除了实际的安全问题，饮酒还可能有其它挑战。汉森认为，地球人存在许多社会文化方面的差异，而且人连续几年时间呆在一个狭小的空间里，很容易突然发怒，这些因素都使饮酒问题变得很棘手。</p>
+                <p><img id="45395153" alt="(Credit: David Frohman/Peachstate Historical Consulting Inc)" src="http://imgtech.gmw.cn/attachement/jpg/site2/20170310/448a5bc1e2861a2c4e592d.jpg" title="&#x5B87;&#x822A;&#x5458;&#x5728;&#x592A;&#x7A7A;&#x4E2D;&#x559D;&#x9152;&#x4F1A;&#x600E;&#x4E48;&#x6837;&#xFF1F;&#x540E;&#x679C;&#x5F88;&#x4E25;&#x91CD;"/> </p>
+                <p>
+                    <span face="&#x6977;&#x4F53;">　　图注：奥尔德林的圣餐杯回到了地球上</span>
+                </p>
+                <p>　　他说：“这是一个政治问题，也是一个文化方面的问题，但不是一个科学上的问题。这将是未来一个可能产生冲突领域，因为人们具有不同的文化背景，他们对饮酒的态度不同。”他进一步指出，如果你与穆斯林、摩门教徒或禁酒主义者分配在同一间宿舍怎么办？面对未来人们可能在一个没有期限的时间内呆在一个有限的空间里，需要“尽早解决”如何协调不同文化观点的问题。</p>
+                <p>　　所以，当宇航员在地球轨道上时，将还不得不满足于通过欣赏外面的景色来振作精神，而不要指望沉溺于烈酒中。我们留在地球上的人，则可以准备好适量的香槟酒，以迎接他们的归来。</p>
+                <p>　　原标题：他晚于阿姆斯特朗登月 却是首个敢在月球喝酒的人</p>
+                <p><strong>　　出品︱网易科学人栏目组 胖胖</strong></p>
+                <p><strong>　　作者︱春春</strong>
+                    <a href="http://www.gmw.cn/" target="_blank"><img src="https://img.gmw.cn/pic/content_logo.png" title="&#x8FD4;&#x56DE;&#x5149;&#x660E;&#x7F51;&#x9996;&#x9875;"/></a>
+                </p>
+                
+                <p>[责任编辑:肖春芳]</p>
+                
+                
+            </div></article>

article_scraper/resources/tests/readability/google-sre-book-1/expected.html

@@ -0,0 +1,458 @@
+<article><section data-type="chapter" id="readability-page-1" role="main">
+                        <h2>
+                            Monitoring Distributed Systems
+                        </h2>
+                        
+                        <p>
+                            Google’s SRE teams have some basic principles and best practices for building successful monitoring and alerting systems. This chapter offers guidelines for what issues should interrupt a human via a page, and how to deal with issues that aren’t serious enough to trigger a page.
+                        </p>
+                        <section data-type="sect1" id="definitions-2ksZhN">
+                            <h2>
+                                Definitions
+                            </h2>
+                            <p>
+                                There’s no uniformly shared vocabulary for discussing all topics related to monitoring. Even within Google, usage of the following terms varies, but the most common interpretations are listed here.
+                            </p>
+                            <dl>
+                                
+                                <dd>
+                                    <p>
+                                        Collecting, processing, aggregating, and displaying real-time quantitative data about a system, such as query counts and types, error counts and types, processing times, and server lifetimes.
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        Monitoring based on metrics exposed by the internals of the system, including logs, interfaces like the Java Virtual Machine Profiling Interface, or an HTTP handler that emits internal statistics.
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        Testing externally visible behavior as a user would see it.
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        An application (usually web-based) that provides a summary view of a service’s core metrics. A dashboard may have filters, selectors, and so on, but is prebuilt to expose the metrics most important to its users. The dashboard might also display team information such as ticket queue length, a list of high-priority bugs, the current on-call engineer for a given area of responsibility, or recent pushes.
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        A notification intended to be read by a human and that is pushed to a system such as a bug or ticket queue, an email alias, or a pager. Respectively, these alerts are classified as <em>tickets</em>, <em>email alerts</em>,<sup><a data-type="noteref" id="id-LvQuvtYS7UvI8h4-marker" href="#id-LvQuvtYS7UvI8h4">22</a></sup> and <em>pages</em>.
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        A defect in a software or human system that, if repaired, instills confidence that this event won’t happen again in the same way. A given incident might have multiple root causes: for example, perhaps it was caused by a combination of insufficient process automation, software that crashed on bogus input, <em>and</em> insufficient testing of the script used to generate the configuration. Each of these factors might stand alone as a root cause, and each should be repaired.
+                                    </p>
+                                </dd>
+                                <dt id="node-and-machine">
+                                    Node and machine
+                                </dt>
+                                <dd>
+                                    <p>
+                                        Used interchangeably to indicate a single instance of a running kernel in either a physical server, virtual machine, or container. There might be multiple <em>services</em> worth monitoring on a single machine. The services may either be:
+                                    </p>
+                                    <ul>
+                                        <li>Related to each other: for example, a caching server and a web server
+                                        </li>
+                                        <li>Unrelated services sharing hardware: for example, a code repository and a master for a configuration system like <a href="https://puppetlabs.com/puppet/puppet-open-source" target="_blank">Puppet</a> or <a href="https://www.chef.io/chef/" target="_blank">Chef</a>
+                                        </li>
+                                    </ul>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        Any change to a service’s running software or its configuration.
+                                    </p>
+                                </dd>
+                            </dl>
+                        </section>
+                        <section data-type="sect1" id="why-monitor-pWsBTZ">
+                            <h2>
+                                Why Monitor?
+                            </h2>
+                            <p>
+                                There are many reasons to monitor a system, including:
+                            </p>
+                            <dl>
+                                
+                                <dd>
+                                    <p>
+                                        How big is my database and how fast is it growing? How quickly is my daily-active user count growing?
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        Are queries faster with Acme Bucket of Bytes 2.72 versus Ajax DB 3.14? How much better is my memcache hit rate with an extra node? Is my site slower than it was last week?
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        Something is broken, and somebody needs to fix it right now! Or, something might break soon, so somebody should look soon.
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        Dashboards should answer basic questions about your service, and normally include some form of the four golden signals (discussed in <a data-type="xref" href="#xref_monitoring_golden-signals">The Four Golden Signals</a>).
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        Our latency just shot up; what else happened around the same time?
+                                    </p>
+                                </dd>
+                            </dl>
+                            <p>
+                                System monitoring is also helpful in supplying raw input into business analytics and in facilitating analysis of security breaches. Because this book focuses on the engineering domains in which SRE has particular expertise, we won’t discuss these applications of monitoring here.
+                            </p>
+                            <p>
+                                Monitoring and alerting enables a system to tell us when it’s broken, or perhaps to tell us what’s about to break. When the system isn’t able to automatically fix itself, we want a human to investigate the alert, determine if there’s a real problem at hand, mitigate the problem, and determine the root cause of the problem. Unless you’re performing security auditing on very narrowly scoped components of a system, you should never trigger an alert simply because "something seems a bit weird."
+                            </p>
+                            <p>
+                                Paging a human is a quite expensive use of an employee’s time. If an employee is at work, a page interrupts their workflow. If the employee is at home, a page interrupts their personal time, and perhaps even their sleep. When pages occur too frequently, employees second-guess, skim, or even ignore incoming alerts, sometimes even ignoring a "real" page that’s masked by the noise. Outages can be prolonged because other noise interferes with a rapid diagnosis and fix. Effective alerting systems have good signal and very low noise.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="setting-reasonable-expectations-for-monitoring-o8svcM">
+                            <h2>
+                                Setting Reasonable Expectations for Monitoring
+                            </h2>
+                            <p>
+                                Monitoring a complex application is a significant engineering endeavor in and of itself. Even with substantial existing infrastructure for instrumentation, collection, display, and alerting in place, a Google SRE team with 10–12 members typically has one or sometimes two members whose primary assignment is to build and maintain monitoring systems for their service. This number has decreased over time as we generalize and centralize common monitoring infrastructure, but every SRE team typically has at least one “monitoring person.” (That being said, while it can be fun to have access to traffic graph dashboards and the like, SRE teams carefully avoid any situation that requires someone to “stare at a screen to watch for problems.”)
+                            </p>
+                            <p>
+                                In general, Google has trended toward simpler and faster monitoring systems, with better tools for <em>post hoc</em> analysis. We avoid "magic" systems that try to learn thresholds or automatically detect causality. Rules that detect unexpected changes in end-user request rates are one counterexample; while these rules are still kept as simple as possible, they give a very quick detection of a very simple, specific, severe anomaly. Other uses of monitoring data such as capacity planning and traffic prediction can tolerate more fragility, and thus, more complexity. Observational experiments conducted over a very long time horizon (months or years) with a low sampling rate (hours or days) can also often tolerate more fragility because occasional missed samples won’t hide a long-running trend.
+                            </p>
+                            <p>
+                                Google SRE has experienced only limited success with complex dependency hierarchies. We seldom use rules such as, "If I know the database is slow, alert for a slow database; otherwise, alert for the website being generally slow." Dependency-reliant rules usually pertain to very stable parts of our system, such as our system for draining user traffic away from a datacenter. For example, "If a datacenter is drained, then don’t alert me on its latency" is one common datacenter alerting rule. Few teams at Google maintain complex dependency hierarchies because our infrastructure has a steady rate of continuous refactoring.
+                            </p>
+                            <p>
+                                Some of the ideas described in this chapter are still aspirational: there is always room to move more rapidly from symptom to root cause(s), especially in ever-changing systems. So while this chapter sets out some goals for monitoring systems, and some ways to achieve these goals, it’s important that monitoring systems—especially the critical path from the onset of a production problem, through a page to a human, through basic triage and deep debugging—be kept simple and comprehensible by everyone on the team.
+                            </p>
+                            <p>
+                                Similarly, to keep noise low and signal high, the elements of your monitoring system that direct to a pager need to be very simple and robust. Rules that generate alerts for humans should be simple to understand and represent a clear failure.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="symptoms-versus-causes-g0sEi4">
+                            <h2>
+                                Symptoms Versus Causes
+                            </h2>
+                            <p>
+                                Your monitoring system should address two questions: what’s broken, and why?
+                            </p>
+                            <p>
+                                The "what’s broken" indicates the symptom; the "why" indicates a (possibly intermediate) cause. <a data-type="xref" href="#table_monitoring_symptoms">Table 6-1</a> lists some hypothetical symptoms and corresponding causes.
+                            </p>
+                            <table id="table_monitoring_symptoms">
+                                <caption>
+                                    <span>Table 6-1.</span> Example symptoms and causes
+                                </caption>
+                                <thead>
+                                    <tr>
+                                        <th>
+                                            <strong>Symptom</strong>
+                                        </th>
+                                        <th>
+                                            <strong>Cause</strong>
+                                        </th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    <tr>
+                                        <td>
+                                            <p>
+                                                <strong>I’m serving HTTP 500s or 404s</strong>
+                                            </p>
+                                        </td>
+                                        <td>
+                                            <p>
+                                                Database servers are refusing connections
+                                            </p>
+                                        </td>
+                                    </tr>
+                                    <tr>
+                                        <td>
+                                            <p>
+                                                <strong>My responses are slow</strong>
+                                            </p>
+                                        </td>
+                                        <td>
+                                            <p>
+                                                CPUs are overloaded by a bogosort, or an Ethernet cable is crimped under a rack, visible as partial packet loss
+                                            </p>
+                                        </td>
+                                    </tr>
+                                    <tr>
+                                        <td>
+                                            <p>
+                                                <strong>Users in Antarctica aren’t receiving animated cat GIFs</strong>
+                                            </p>
+                                        </td>
+                                        <td>
+                                            <p>
+                                                Your Content Distribution Network hates scientists and felines, and thus blacklisted some client IPs
+                                            </p>
+                                        </td>
+                                    </tr>
+                                    <tr>
+                                        <td>
+                                            <p>
+                                                <strong>Private content is world-readable</strong>
+                                            </p>
+                                        </td>
+                                        <td>
+                                            <p>
+                                                A new software push caused ACLs to be forgotten and allowed all requests
+                                            </p>
+                                        </td>
+                                    </tr>
+                                </tbody>
+                            </table>
+                            <p>
+                                "What" versus "why" is one of the most important distinctions in writing good monitoring with maximum signal and minimum noise.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="black-box-versus-white-box-q8sJuw">
+                            <h2>
+                                Black-Box Versus White-Box
+                            </h2>
+                            <p>
+                                We combine heavy use of white-box monitoring with modest but critical uses of black-box monitoring. The simplest way to think about black-box monitoring versus white-box monitoring is that black-box monitoring is symptom-oriented and represents active—not predicted—problems: "The system isn’t working correctly, right now." White-box monitoring depends on the ability to inspect the innards of the system, such as logs or HTTP endpoints, with instrumentation. White-box monitoring therefore allows detection of imminent problems, failures masked by retries, and so forth.
+                            </p>
+                            <p>
+                                Note that in a multilayered system, one person’s symptom is another person’s cause. For example, suppose that a database’s performance is slow. Slow database reads are a symptom for the database SRE who detects them. However, for the frontend SRE observing a slow website, the same slow database reads are a cause. Therefore, white-box monitoring is sometimes symptom-oriented, and sometimes cause-oriented, depending on just how informative your white-box is.
+                            </p>
+                            <p>
+                                When collecting telemetry for debugging, white-box monitoring is essential. If web servers seem slow on database-heavy requests, you need to know both how fast the web server perceives the database to be, and how fast the database believes itself to be. Otherwise, you can’t distinguish an actually slow database server from a network problem between your web server and your database.
+                            </p>
+                            <p>
+                                For paging, black-box monitoring has the key benefit of forcing discipline to only nag a human when a problem is both already ongoing and contributing to real symptoms. On the other hand, for not-yet-occurring but imminent problems, black-box monitoring is fairly useless.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="xref_monitoring_golden-signals">
+                            <h2>
+                                The Four Golden Signals
+                            </h2>
+                            <p>
+                                The four golden signals of monitoring are latency, traffic, errors, and saturation. If you can only measure four metrics of your user-facing system, focus on these four.
+                            </p>
+                            <dl>
+                                
+                                <dd>
+                                    <p>
+                                        The time it takes to service a request. It’s important to distinguish between the latency of successful requests and the latency of failed requests. For example, an HTTP 500 error triggered due to loss of connection to a database or other critical backend might be served very quickly; however, as an HTTP 500 error indicates a failed request, factoring 500s into your overall latency might result in misleading calculations. On the other hand, a slow error is even worse than a fast error! Therefore, it’s important to track error latency, as opposed to just filtering out errors.
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        A measure of how much demand is being placed on your system, measured in a high-level system-specific metric. For a web service, this measurement is usually HTTP requests per second, perhaps broken out by the nature of the requests (e.g., static versus dynamic content). For an audio streaming system, this measurement might focus on network I/O rate or concurrent sessions. For a key-value storage system, this measurement might be transactions and retrievals per <span>second</span>.
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        The rate of requests that fail, either explicitly (e.g., HTTP 500s), implicitly (for example, an HTTP 200 success response, but coupled with the wrong content), or by policy (for example, "If you committed to one-second response times, any request over one second is an error"). Where protocol response codes are insufficient to express all failure conditions, secondary (internal) protocols may be necessary to track partial failure modes. Monitoring these cases can be drastically different: catching HTTP 500s at your load balancer can do a decent job of catching all completely failed requests, while only end-to-end system tests can detect that you’re serving the wrong content.
+                                    </p>
+                                </dd>
+                                
+                                <dd>
+                                    <p>
+                                        How "full" your service is. A measure of your system fraction, emphasizing the resources that are most constrained (e.g., in a memory-constrained system, show memory; in an I/O-constrained system, show I/O). Note that many systems degrade in performance before they achieve 100% utilization, so having a utilization target is essential.
+                                    </p>
+                                    <p>
+                                        In complex systems, saturation can be supplemented with higher-level load measurement: can your service properly handle double the traffic, handle only 10% more traffic, or handle even less traffic than it currently receives? For very simple services that have no parameters that alter the complexity of the request (e.g., "Give me a nonce" or "I need a globally unique monotonic integer") that rarely change configuration, a static value from a load test might be adequate. As discussed in the previous paragraph, however, most services need to use indirect signals like CPU utilization or network bandwidth that have a known upper bound. Latency increases are often a leading indicator of saturation. Measuring your 99th percentile response time over some small window (e.g., one minute) can give a very early signal of saturation.
+                                    </p>
+                                    <p>
+                                        Finally, saturation is also concerned with predictions of impending saturation, such as "It looks like your database will fill its hard drive in 4 hours."
+                                    </p>
+                                </dd>
+                            </dl>
+                            <p>
+                                If you measure all four golden signals and page a human when one signal is problematic (or, in the case of saturation, nearly problematic), your service will be at least decently covered by monitoring.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="worrying-about-your-tail-or-instrumentation-and-performance-Yms9Ck">
+                            <h2>
+                                Worrying About Your Tail (or, Instrumentation and Performance)
+                            </h2>
+                            <p>
+                                When building a monitoring system from scratch, it’s tempting to design a system based upon the mean of some quantity: the mean latency, the mean CPU usage of your nodes, or the mean fullness of your databases. The danger presented by the latter two cases is obvious: CPUs and databases can easily be utilized in a very imbalanced way. The same holds for latency. If you run a web service with an average latency of 100 ms at 1,000 requests per second, 1% of requests might easily take 5 seconds.<sup><a data-type="noteref" id="id-QQLuAIXFxCz-marker" href="#id-QQLuAIXFxCz">23</a></sup> If your users depend on several such web services to render their page, the 99th percentile of one backend can easily become the median response of your <span>frontend</span>.
+                            </p>
+                            <p>
+                                The simplest way to differentiate between a slow average and a very slow "tail" of requests is to collect request counts bucketed by latencies (suitable for rendering a histogram), rather than actual latencies: how many requests did I serve that took between 0 ms and 10 ms, between 10 ms and 30 ms, between 30 ms and 100 ms, between 100 ms and 300 ms, and so on? Distributing the histogram boundaries approximately exponentially (in this case by factors of roughly 3) is often an easy way to visualize the distribution of your requests.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="choosing-an-appropriate-resolution-for-measurements-vJsBsE">
+                            <h2>
+                                Choosing an Appropriate Resolution for Measurements
+                            </h2>
+                            <p>
+                                Different aspects of a system should be measured with different levels of granularity. For example:
+                            </p>
+                            <ul>
+                                <li>Observing CPU load over the time span of a minute won’t reveal even quite long-lived spikes that drive high tail latencies.
+                                </li>
+                                <li>On the other hand, for a web service targeting no more than 9 hours aggregate downtime per year (99.9% annual uptime), probing for a 200 (success) status more than once or twice a minute is probably unnecessarily frequent.
+                                </li>
+                                <li>Similarly, checking hard drive fullness for a service targeting 99.9% availability more than once every 1–2 minutes is probably unnecessary.
+                                </li>
+                            </ul>
+                            <p>
+                                Take care in how you structure the granularity of your measurements. Collecting per-second measurements of CPU load might yield interesting data, but such frequent measurements may be very expensive to collect, store, and analyze. If your monitoring goal calls for high resolution but doesn’t require extremely low latency, you can reduce these costs by performing internal sampling on the server, then configuring an external system to collect and aggregate that distribution over time or across servers. You might:
+                            </p>
+                            <ol>
+                                <li>Record the current CPU utilization each second.
+                                </li>
+                                <li>Using buckets of 5% granularity, increment the appropriate CPU utilization bucket each second.
+                                </li>
+                                <li>Aggregate those values every minute.
+                                </li>
+                            </ol>
+                            <p>
+                                This strategy allows you to observe brief CPU hotspots without incurring very high cost due to collection and retention.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="as-simple-as-possible-no-simpler-lqskHx">
+                            <h2>
+                                As Simple as Possible, No Simpler
+                            </h2>
+                            <p>
+                                Piling all these requirements on top of each other can add up to a very complex monitoring system—your system might end up with the following levels of complexity:
+                            </p>
+                            <ul>
+                                <li>Alerts on different latency thresholds, at different percentiles, on all kinds of different metrics
+                                </li>
+                                <li>Extra code to detect and expose possible causes
+                                </li>
+                                <li>Associated dashboards for each of these possible causes
+                                </li>
+                            </ul>
+                            <p>
+                                The sources of potential complexity are never-ending. Like all software systems, monitoring can become so complex that it’s fragile, complicated to change, and a maintenance burden.
+                            </p>
+                            <p>
+                                Therefore, design your monitoring system with an eye toward simplicity. In choosing what to monitor, keep the following guidelines in mind:
+                            </p>
+                            <ul>
+                                <li>The rules that catch real incidents most often should be as simple, predictable, and reliable as possible.
+                                </li>
+                                <li>Data collection, aggregation, and alerting configuration that is rarely exercised (e.g., less than once a quarter for some SRE teams) should be up for removal.
+                                </li>
+                                <li>Signals that are collected, but not exposed in any prebaked dashboard nor used by any alert, are candidates for removal.
+                                </li>
+                            </ul>
+                            <p>
+                                In Google’s experience, basic collection and aggregation of metrics, paired with alerting and dashboards, has worked well as a relatively standalone system. (In fact Google’s monitoring system is broken up into several binaries, but typically people learn about all aspects of these binaries.) It can be tempting to combine monitoring with other aspects of inspecting complex systems, such as detailed system profiling, single-process debugging, tracking details about exceptions or crashes, load testing, log collection and analysis, or traffic inspection. While most of these subjects share commonalities with basic monitoring, blending together too many results in overly complex and fragile systems. As in many other aspects of software engineering, maintaining distinct systems with clear, simple, loosely coupled points of integration is a better strategy (for example, using web APIs for pulling summary data in a format that can remain constant over an extended period of time).
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="tying-these-principles-together-nqsJfw">
+                            <h2>
+                                Tying These Principles Together
+                            </h2>
+                            <p>
+                                The principles discussed in this chapter can be tied together into a philosophy on monitoring and alerting that’s widely endorsed and followed within Google SRE teams. While this monitoring philosophy is a bit aspirational, it’s a good starting point for writing or reviewing a new alert, and it can help your organization ask the right questions, regardless of the size of your organization or the complexity of your service or system.
+                            </p>
+                            <p>
+                                When creating rules for monitoring and alerting, asking the following questions can help you avoid false positives and pager burnout:<sup><a data-type="noteref" id="id-a82udF8IBfx-marker" href="#id-a82udF8IBfx">24</a></sup>
+                            </p>
+                            <ul>
+                                <li>Does this rule detect <em>an otherwise undetected condition</em> that is urgent, actionable, and actively or imminently user-visible?<sup><a data-type="noteref" id="id-0vYuEFpSjSMtLfG-marker" href="#id-0vYuEFpSjSMtLfG">25</a></sup>
+                                </li>
+                                <li>Will I ever be able to ignore this alert, knowing it’s benign? When and why will I be able to ignore this alert, and how can I avoid this scenario?
+                                </li>
+                                <li>Does this alert definitely indicate that users are being negatively affected? Are there detectable cases in which users aren’t being negatively impacted, such as drained traffic or test deployments, that should be filtered out?
+                                </li>
+                                <li>Can I take action in response to this alert? Is that action urgent, or could it wait until morning? Could the action be safely automated? Will that action be a long-term fix, or just a short-term workaround?
+                                </li>
+                                <li>Are other people getting paged for this issue, therefore rendering at least one of the pages unnecessary?
+                                </li>
+                            </ul>
+                            <p>
+                                These questions reflect a fundamental philosophy on pages and pagers:
+                            </p>
+                            <ul>
+                                <li>Every time the pager goes off, I should be able to react with a sense of urgency. I can only react with a sense of urgency a few times a day before I become fatigued.
+                                </li>
+                                <li>Every page should be actionable.
+                                </li>
+                                <li>Every page response should require intelligence. If a page merely merits a robotic response, it shouldn’t be a page.
+                                </li>
+                                <li>Pages should be about a novel problem or an event that hasn’t been seen before.
+                                </li>
+                            </ul>
+                            <p>
+                                Such a perspective dissipates certain distinctions: if a page satisfies the preceding four bullets, it’s irrelevant whether the page is triggered by white-box or black-box monitoring. This perspective also amplifies certain distinctions: it’s better to spend much more effort on catching symptoms than causes; when it comes to causes, only worry about very definite, very imminent causes.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="monitoring-for-the-long-term-NbsNS8">
+                            <h2>
+                                Monitoring for the Long Term
+                            </h2>
+                            <p>
+                                In modern production systems, monitoring systems track an ever-evolving system with changing software architecture, load characteristics, and performance targets. An alert that’s currently exceptionally rare and hard to automate might become frequent, perhaps even meriting a hacked-together script to resolve it. At this point, someone should find and eliminate the root causes of the problem; if such resolution isn’t possible, the alert response deserves to be fully automated.
+                            </p>
+                            <p>
+                                It’s important that decisions about monitoring be made with long-term goals in mind. Every page that happens today distracts a human from improving the system for tomorrow, so there is often a case for taking a short-term hit to availability or performance in order to improve the long-term outlook for the system. Let’s take a look at two case studies that illustrate this trade-off.
+                            </p>
+                            <section data-type="sect2" id="bigtable-sre-a-tale-of-over-alerting-dbsXtjSM">
+                                
+                                <p>
+                                    Google’s internal infrastructure is typically offered and measured against a service level objective (SLO; see <a data-type="xref" href="http://fakehost/sre/sre-book/chapters/service-level-objectives" target="_blank">Service Level Objectives</a>). Many years ago, the Bigtable service’s SLO was based on a synthetic well-behaved client’s mean performance. Because of problems in Bigtable and lower layers of the storage stack, the mean performance was driven by a "large" tail: the worst 5% of requests were often significantly slower than the rest.
+                                </p>
+                                <p>
+                                    Email alerts were triggered as the SLO approached, and paging alerts were triggered when the SLO was exceeded. Both types of alerts were firing voluminously, consuming unacceptable amounts of engineering time: the team spent significant amounts of time triaging the alerts to find the few that were really actionable, and we often missed the problems that actually affected users, because so few of them did. Many of the pages were non-urgent, due to well-understood problems in the infrastructure, and had either rote responses or received no response.
+                                </p>
+                                <p>
+                                    To remedy the situation, the team used a three-pronged approach: while making great efforts to improve the performance of Bigtable, we also temporarily dialed back our SLO target, using the 75th percentile request latency. We also disabled email alerts, as there were so many that spending time diagnosing them was infeasible.
+                                </p>
+                                <p>
+                                    This strategy gave us enough breathing room to actually fix the longer-term problems in Bigtable and the lower layers of the storage stack, rather than constantly fixing tactical problems. On-call engineers could actually accomplish work when they weren’t being kept up by pages at all hours. Ultimately, temporarily backing off on our alerts allowed us to make faster progress toward a better service.
+                                </p>
+                            </section>
+                            <section data-type="sect2" id="gmail-predictable-scriptable-responses-from-humans-BVs1h4SD">
+                                
+                                <p>
+                                    In the very early days of Gmail, the service was built on a retrofitted distributed process management system called Workqueue, which was originally created for batch processing of pieces of the search index. Workqueue was "adapted" to long-lived processes and subsequently applied to Gmail, but certain bugs in the relatively opaque codebase in the scheduler proved hard to beat.
+                                </p>
+                                <p>
+                                    At that time, the Gmail monitoring was structured such that alerts fired when individual tasks were “de-scheduled” by Workqueue. This setup was less than ideal because even at that time, Gmail had many, many thousands of tasks, each task representing a fraction of a percent of our users. We cared deeply about providing a good user experience for Gmail users, but such an alerting setup was unmaintainable.
+                                </p>
+                                <p>
+                                    To address this problem, Gmail SRE built a tool that helped “poke” the scheduler in just the right way to minimize impact to users. The team had several discussions about whether or not we should simply automate the entire loop from detecting the problem to nudging the rescheduler, until a better long-term solution was achieved, but some worried this kind of workaround would delay a real fix.
+                                </p>
+                                <p>
+                                    This kind of tension is common within a team, and often reflects an underlying mistrust of the team’s self-discipline: while some team members want to implement a “hack” to allow time for a proper fix, others worry that a hack will be forgotten or that the proper fix will be deprioritized indefinitely. This concern is credible, as it’s easy to build layers of unmaintainable technical debt by patching over problems instead of making real fixes. Managers and technical leaders play a key role in implementing true, long-term fixes by supporting and prioritizing potentially time-consuming long-term fixes even when the initial “pain” of paging subsides.
+                                </p>
+                                <p>
+                                    Pages with rote, algorithmic responses should be a red flag. Unwillingness on the part of your team to automate such pages implies that the team lacks confidence that they can clean up their technical debt. This is a major problem worth escalating.
+                                </p>
+                            </section>
+                            <section data-type="sect2" id="the-long-run-MQsWTMS7">
+                                
+                                <p>
+                                    A common theme connects the previous examples of Bigtable and Gmail: a tension between short-term and long-term availability. Often, sheer force of effort can help a rickety system achieve high availability, but this path is usually short-lived and fraught with burnout and dependence on a small number of heroic team members. Taking a controlled, short-term decrease in availability is often a painful, but strategic trade for the long-run stability of the system. It’s important not to think of every page as an event in isolation, but to consider whether the overall <em>level</em> of paging leads toward a healthy, appropriately available system with a healthy, viable team and long-term outlook. We review statistics about page frequency (usually expressed as incidents per shift, where an incident might be composed of a few related pages) in quarterly reports with management, ensuring that decision makers are kept up to date on the pager load and overall health of their teams.
+                                </p>
+                            </section>
+                        </section>
+                        <section data-type="sect1" id="conclusion-8ksvFj">
+                            <h2>
+                                Conclusion
+                            </h2>
+                            <p>
+                                A healthy monitoring and alerting pipeline is simple and easy to reason about. It focuses primarily on symptoms for paging, reserving cause-oriented heuristics to serve as aids to debugging problems. Monitoring symptoms is easier the further "up" your stack you monitor, though monitoring saturation and performance of subsystems such as databases often must be performed directly on the subsystem itself. Email alerts are of very limited value and tend to easily become overrun with noise; instead, you should favor a dashboard that monitors all ongoing subcritical problems for the sort of information that typically ends up in email alerts. A dashboard might also be paired with a log, in order to analyze historical correlations.
+                            </p>
+                            <p>
+                                Over the long haul, achieving a successful on-call rotation and product includes choosing to alert on symptoms or imminent real problems, adapting your targets to goals that are actually achievable, and making sure that your monitoring supports rapid diagnosis.
+                            </p>
+                        </section>
+                        
+                    </section></article>

article_scraper/resources/tests/readability/google-sre-book-1/source.html

@@ -0,0 +1,742 @@
+<!DOCTYPE html>
+<html lang="en" xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+    <head>
+        <meta charset="utf-8" />
+        <meta content="initial-scale=1, minimum-scale=1, width=device-width" name="viewport" />
+        <title>
+            Google - Site Reliability Engineering
+        </title>
+        <meta name="referrer" content="no-referrer" />
+        <link rel="apple-touch-icon-precomposed" sizes="180x180" href="https://lh3.googleusercontent.com/Yf2DCX8RKda6r4Jml9DLMByS2zQCBFs3kQpvBfN8UgIh4YVWIYSYIQOoTxJriyuM26cT5PDjyEb5aynDQ0Xyz46yHKnfg8JlUbDW" />
+        <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Google+Sans:400|Roboto:400,400italic,500,500italic,700,700italic|Roboto+Mono:400,500,700|Material+Icons" />
+        <link rel="icon" type="image/png" sizes="32x32" href="https://lh3.googleusercontent.com/Yf2DCX8RKda6r4Jml9DLMByS2zQCBFs3kQpvBfN8UgIh4YVWIYSYIQOoTxJriyuM26cT5PDjyEb5aynDQ0Xyz46yHKnfg8JlUbDW" />
+        <link rel="icon" type="image/png" sizes="16x16" href="https://lh3.googleusercontent.com/Yf2DCX8RKda6r4Jml9DLMByS2zQCBFs3kQpvBfN8UgIh4YVWIYSYIQOoTxJriyuM26cT5PDjyEb5aynDQ0Xyz46yHKnfg8JlUbDW" />
+        <link rel="shortcut icon" href="https://lh3.googleusercontent.com/Yf2DCX8RKda6r4Jml9DLMByS2zQCBFs3kQpvBfN8UgIh4YVWIYSYIQOoTxJriyuM26cT5PDjyEb5aynDQ0Xyz46yHKnfg8JlUbDW" />
+        <link href="/sre/sre-book/static/css/index.min.css?cache=0ffc48d" rel="stylesheet" />
+        <script>
+        <![CDATA[
+        (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+        (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+        m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+        })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
+
+        ga('create', 'UA-75468017-1', 'auto');
+        ga('send', 'pageview');
+        ]]>
+        </script>
+        <script src="/sre/sre-book/static/js/detect.min.js?cache=4cb778b"></script>
+    </head>
+    <body>
+        <main>
+            <div ng-controller="HeaderCtrl as headerCtrl">
+                <div id="curtain" class="menu-closed"></div>
+                <div class="header clearfix">
+                    <a id="burger-menu" class="expand"></a>
+                    <h2 class="chapter-title">
+                        Chapter 6 - Monitoring Distributed Systems
+                    </h2>
+                </div>
+                <div id="overlay-element" class="expands">
+                    <div class="logo">
+                        <a href="https://www.google.com"><img src="https://lh3.googleusercontent.com/YoVRtLOHMSRYQZ3OhFL8RIamcjFYbmQXX4oAQx02MRqqY9zlKNvsuZpS73khXiOqTH3qrFW27VrERJJIHTjPk-tAh46q8-Fd4w6qlw" alt="Google" /></a>
+                    </div>
+                    <ol id="drop-down" class="dropdown-content hide">
+                        <li>
+                            <a class="menu-buttons" href="/sre/sre-book/toc/">Table of Contents</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/foreword" class="menu-buttons">Foreword</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/preface" class="menu-buttons">Preface</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/part1" class="menu-buttons">Part I - Introduction</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/introduction" class="menu-buttons">1. Introduction</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/production-environment" class="menu-buttons">2. The Production Environment at Google, from the Viewpoint of an SRE</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/part2" class="menu-buttons">Part II - Principles</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/embracing-risk" class="menu-buttons">3. Embracing Risk</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/service-level-objectives" class="menu-buttons">4. Service Level Objectives</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/eliminating-toil" class="menu-buttons">5. Eliminating Toil</a>
+                        </li>
+                        <li class="active">
+                            <a href="/sre/sre-book/chapters/monitoring-distributed-systems" class="menu-buttons">6. Monitoring Distributed Systems</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/automation-at-google" class="menu-buttons">7. The Evolution of Automation at Google</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/release-engineering" class="menu-buttons">8. Release Engineering</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/simplicity" class="menu-buttons">9. Simplicity</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/part3" class="menu-buttons">Part III - Practices</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/practical-alerting" class="menu-buttons">10. Practical Alerting</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/being-on-call" class="menu-buttons">11. Being On-Call</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/effective-troubleshooting" class="menu-buttons">12. Effective Troubleshooting</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/emergency-response" class="menu-buttons">13. Emergency Response</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/managing-incidents" class="menu-buttons">14. Managing Incidents</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/postmortem-culture" class="menu-buttons">15. Postmortem Culture: Learning from Failure</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/tracking-outages" class="menu-buttons">16. Tracking Outages</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/testing-reliability" class="menu-buttons">17. Testing for Reliability</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/software-engineering-in-sre" class="menu-buttons">18. Software Engineering in SRE</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/load-balancing-frontend" class="menu-buttons">19. Load Balancing at the Frontend</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/load-balancing-datacenter" class="menu-buttons">20. Load Balancing in the Datacenter</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/handling-overload" class="menu-buttons">21. Handling Overload</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/addressing-cascading-failures" class="menu-buttons">22. Addressing Cascading Failures</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/managing-critical-state" class="menu-buttons">23. Managing Critical State: Distributed Consensus for Reliability</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/distributed-periodic-scheduling" class="menu-buttons">24. Distributed Periodic Scheduling with Cron</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/data-processing-pipelines" class="menu-buttons">25. Data Processing Pipelines</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/data-integrity" class="menu-buttons">26. Data Integrity: What You Read Is What You Wrote</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/reliable-product-launches" class="menu-buttons">27. Reliable Product Launches at Scale</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/part4" class="menu-buttons">Part IV - Management</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/accelerating-sre-on-call" class="menu-buttons">28. Accelerating SREs to On-Call and Beyond</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/dealing-with-interrupts" class="menu-buttons">29. Dealing with Interrupts</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/operational-overload" class="menu-buttons">30. Embedding an SRE to Recover from Operational Overload</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/communication-and-collaboration" class="menu-buttons">31. Communication and Collaboration in SRE</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/evolving-sre-engagement-model" class="menu-buttons">32. The Evolving SRE Engagement Model</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/part5" class="menu-buttons">Part V - Conclusions</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/lessons-learned" class="menu-buttons">33. Lessons Learned from Other Industries</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/conclusion" class="menu-buttons">34. Conclusion</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/availability-table" class="menu-buttons">Appendix A. Availability Table</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/service-best-practices" class="menu-buttons">Appendix B. A Collection of Best Practices for Production Services</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/incident-document" class="menu-buttons">Appendix C. Example Incident State Document</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/postmortem" class="menu-buttons">Appendix D. Example Postmortem</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/launch-checklist" class="menu-buttons">Appendix E. Launch Coordination Checklist</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/production-meeting" class="menu-buttons">Appendix F. Example Production Meeting Minutes</a>
+                        </li>
+                        <li>
+                            <a href="/sre/sre-book/chapters/bibliography" class="menu-buttons">Bibliography</a>
+                        </li>
+                    </ol>
+                </div>
+            </div>
+            <div id="maia-main" role="main">
+                <div class="maia-teleport" id="content"></div>
+                <div class="content">
+                    <section data-type="chapter" id="chapter_monitoring">
+                        <h1 class="heading jumptargets">
+                            Monitoring Distributed Systems
+                        </h1>
+                        <p class="byline author">
+                            Written by Rob Ewaschuk<br />
+                            Edited by Betsy Beyer
+                        </p>
+                        <p>
+                            Google’s SRE teams have some basic principles and best practices for building successful monitoring and alerting systems. This chapter offers guidelines for what issues should interrupt a human via a page, and how to deal with issues that aren’t serious enough to trigger a page.
+                        </p>
+                        <section data-type="sect1" id="definitions-2ksZhN">
+                            <h1 class="heading jumptargets">
+                                Definitions
+                            </h1>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="terminology" id="id-DnC1SWFMhD"></a>There’s no uniformly shared vocabulary for discussing all topics related to monitoring. Even within Google, usage of the following terms varies, but the most common interpretations are listed here.
+                            </p>
+                            <dl>
+                                <dt class="subheaders jumptargets" id="monitoring">
+                                    Monitoring
+                                </dt>
+                                <dd>
+                                    <p>
+                                        Collecting, processing, aggregating, and displaying real-time quantitative data about a system, such as query counts and types, error counts and types, processing times, and server lifetimes.
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="white-box-monitoring">
+                                    White-box monitoring
+                                </dt>
+                                <dd>
+                                    <p>
+                                        <a data-type="indexterm" data-primary="white-box monitoring" id="id-9nCjSDS4tZILhX"></a>Monitoring based on metrics exposed by the internals of the system, including logs, interfaces like the Java Virtual Machine Profiling Interface, or an HTTP handler that emits internal statistics.
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="black-box-monitoring">
+                                    Black-box monitoring
+                                </dt>
+                                <dd>
+                                    <p>
+                                        <a data-type="indexterm" data-primary="black-box monitoring" id="id-zdCxSrSgTWIdhb"></a>Testing externally visible behavior as a user would see it.
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="dashboard">
+                                    Dashboard
+                                </dt>
+                                <dd>
+                                    <p>
+                                        <a data-type="indexterm" data-primary="dashboards" data-secondary="defined" id="id-VMCPS2SribIkh4"></a>An application (usually web-based) that provides a summary view of a service’s core metrics. A dashboard may have filters, selectors, and so on, but is prebuilt to expose the metrics most important to its users. The dashboard might also display team information such as ticket queue length, a list of high-priority bugs, the current on-call engineer for a given area of responsibility, or recent pushes.
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="alert">
+                                    Alert
+                                </dt>
+                                <dd>
+                                    <p>
+                                        <a data-type="indexterm" data-primary="alerts" data-secondary="defined" id="id-wqC7SvSPUAIVhQ"></a>A notification intended to be read by a human and that is pushed to a system such as a bug or ticket queue, an email alias, or a pager. Respectively, these alerts are classified as <em>tickets</em>, <em>email alerts</em>,<sup><a class="jumptarget" data-type="noteref" id="id-LvQuvtYS7UvI8h4-marker" href="#id-LvQuvtYS7UvI8h4">22</a></sup> and <em>pages</em>.
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="root-cause">
+                                    Root cause
+                                </dt>
+                                <dd>
+                                    <p>
+                                        <a data-type="indexterm" data-primary="root cause" data-secondary="defined" id="id-PnCpSaSKsgIjho"></a>A defect in a software or human system that, if repaired, instills confidence that this event won’t happen again in the same way. A given incident might have multiple root causes: for example, perhaps it was caused by a combination of insufficient process automation, software that crashed on bogus input, <em>and</em> insufficient testing of the script used to generate the configuration. Each of these factors might stand alone as a root cause, and each should be repaired.
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="node-and-machine">
+                                    Node and machine
+                                </dt>
+                                <dd>
+                                    <p>
+                                        <a data-type="indexterm" data-primary="machines" data-secondary="defined" id="id-XmC9SkSlfnI1hK"></a>Used interchangeably to indicate a single instance of a running kernel in either a physical server, virtual machine, or container. There might be multiple <em>services</em> worth monitoring on a single machine. The services may either be:
+                                    </p>
+                                    <ul>
+                                        <li>Related to each other: for example, a caching server and a web server
+                                        </li>
+                                        <li>Unrelated services sharing hardware: for example, a code repository and a master for a configuration system like <a href="https://puppetlabs.com/puppet/puppet-open-source" target="_blank">Puppet</a> or <a href="https://www.chef.io/chef/" target="_blank">Chef</a>
+                                        </li>
+                                    </ul>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="push">
+                                    Push
+                                </dt>
+                                <dd>
+                                    <p>
+                                        Any change to a service’s running software or its configuration.
+                                    </p>
+                                </dd>
+                            </dl>
+                        </section>
+                        <section data-type="sect1" id="why-monitor-pWsBTZ">
+                            <h1 class="heading jumptargets">
+                                Why Monitor?
+                            </h1>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="benefits of monitoring" id="id-kVCkSpFnTl"></a>There are many reasons to monitor a system, including:
+                            </p>
+                            <dl>
+                                <dt class="subheaders jumptargets" id="analyzing-long-term-trends">
+                                    Analyzing long-term trends
+                                </dt>
+                                <dd>
+                                    <p>
+                                        How big is my database and how fast is it growing? How quickly is my daily-active user count growing?
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="comparing-over-time-or-experiment-groups">
+                                    Comparing over time or experiment groups
+                                </dt>
+                                <dd>
+                                    <p>
+                                        Are queries faster with Acme Bucket of Bytes 2.72 versus Ajax DB 3.14? How much better is my memcache hit rate with an extra node? Is my site slower than it was last week?
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="alerting">
+                                    Alerting
+                                </dt>
+                                <dd>
+                                    <p>
+                                        Something is broken, and somebody needs to fix it right now! Or, something might break soon, so somebody should look soon.
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="building-dashboards">
+                                    Building dashboards
+                                </dt>
+                                <dd>
+                                    <p>
+                                        <a data-type="indexterm" data-primary="dashboards" data-secondary="benefits of" id="id-rjCXSOS0iDIGT8"></a>Dashboards should answer basic questions about your service, and normally include some form of the four golden signals (discussed in <a data-type="xref" href="#xref_monitoring_golden-signals">The Four Golden Signals</a>).
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="conducting-ad-hoc-retrospective-analysis-ie-debugging">
+                                    Conducting <i class="italic">ad hoc</i> retrospective analysis (i.e., debugging)
+                                </dt>
+                                <dd>
+                                    <p>
+                                        Our latency just shot up; what else happened around the same time?
+                                    </p>
+                                </dd>
+                            </dl>
+                            <p>
+                                System monitoring is also helpful in supplying raw input into business analytics and in facilitating analysis of security breaches. Because this book focuses on the engineering domains in which SRE has particular expertise, we won’t discuss these applications of monitoring here.
+                            </p>
+                            <p>
+                                Monitoring and alerting enables a system to tell us when it’s broken, or perhaps to tell us what’s about to break. When the system isn’t able to automatically fix itself, we want a human to investigate the alert, determine if there’s a real problem at hand, mitigate the problem, and determine the root cause of the problem. Unless you’re performing security auditing on very narrowly scoped components of a system, you should never trigger an alert simply because "something seems a bit weird."
+                            </p>
+                            <p>
+                                Paging a human is a quite expensive use of an employee’s time. If an employee is at work, a page interrupts their workflow. If the employee is at home, a page interrupts their personal time, and perhaps even their sleep. When pages occur too frequently, employees second-guess, skim, or even ignore incoming alerts, sometimes even ignoring a "real" page that’s masked by the noise. Outages can be prolonged because other noise interferes with a rapid diagnosis and fix. Effective alerting systems have good signal and very low noise.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="setting-reasonable-expectations-for-monitoring-o8svcM">
+                            <h1 class="heading jumptargets">
+                                Setting Reasonable Expectations for Monitoring
+                            </h1>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="setting expectations for" id="id-4nCqSYFQcE"></a>Monitoring a complex application is a significant engineering endeavor in and of itself. Even with substantial existing infrastructure for instrumentation, collection, display, and alerting in place, a Google SRE team with 10–12 members typically has one or sometimes two members whose primary assignment is to build and maintain monitoring systems for their service. This number has decreased over time as we generalize and centralize common monitoring infrastructure, but every SRE team typically has at least one “monitoring person.” (That being said, while it can be fun to have access to traffic graph dashboards and the like, SRE teams carefully avoid any situation that requires someone to “stare at a screen to watch for problems.”)
+                            </p>
+                            <p>
+                                <a data-type="indexterm" data-primary="post hoc analysis" id="id-JnCDSjIVcG"></a>In general, Google has trended toward simpler and faster monitoring systems, with better tools for <em>post hoc</em> analysis. We avoid "magic" systems that try to learn thresholds or automatically detect causality. Rules that detect unexpected changes in end-user request rates are one counterexample; while these rules are still kept as simple as possible, they give a very quick detection of a very simple, specific, severe anomaly. Other uses of monitoring data such as capacity planning and traffic prediction can tolerate more fragility, and thus, more complexity. Observational experiments conducted over a very long time horizon (months or years) with a low sampling rate (hours or days) can also often tolerate more fragility because occasional missed samples won’t hide a long-running trend.
+                            </p>
+                            <p>
+                                <a data-type="indexterm" data-primary="dependency hierarchies" id="id-9nCjSOtmcj"></a>Google SRE has experienced only limited success with complex dependency hierarchies. We seldom use rules such as, "If I know the database is slow, alert for a slow database; otherwise, alert for the website being generally slow." Dependency-reliant rules usually pertain to very stable parts of our system, such as our system for draining user traffic away from a datacenter. For example, "If a datacenter is drained, then don’t alert me on its latency" is one common datacenter alerting rule. Few teams at Google maintain complex dependency hierarchies because our infrastructure has a steady rate of continuous refactoring.
+                            </p>
+                            <p>
+                                Some of the ideas described in this chapter are still aspirational: there is always room to move more rapidly from symptom to root cause(s), especially in ever-changing systems. So while this chapter sets out some goals for monitoring systems, and some ways to achieve these goals, it’s important that monitoring systems—especially the critical path from the onset of a production problem, through a page to a human, through basic triage and deep debugging—be kept simple and comprehensible by everyone on the team.
+                            </p>
+                            <p>
+                                Similarly, to keep noise low and signal high, the elements of your monitoring system that direct to a pager need to be very simple and robust. Rules that generate alerts for humans should be simple to understand and represent a clear failure.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="symptoms-versus-causes-g0sEi4">
+                            <h1 class="heading jumptargets">
+                                Symptoms Versus Causes
+                            </h1>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="symptoms vs. causes" id="id-JnCDSlFmiG"></a>Your monitoring system should address two questions: what’s broken, and why?
+                            </p>
+                            <p>
+                                The "what’s broken" indicates the symptom; the "why" indicates a (possibly intermediate) cause. <a data-type="xref" href="#table_monitoring_symptoms">Table 6-1</a> lists some hypothetical symptoms and corresponding causes.
+                            </p>
+                            <table id="table_monitoring_symptoms" class="pagebreak-before">
+                                <caption class="jumptarget">
+                                    <span class="label">Table 6-1.</span> Example symptoms and causes
+                                </caption>
+                                <thead>
+                                    <tr>
+                                        <th>
+                                            <strong>Symptom</strong>
+                                        </th>
+                                        <th>
+                                            <strong>Cause</strong>
+                                        </th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    <tr>
+                                        <td>
+                                            <p>
+                                                <strong>I’m serving HTTP 500s or 404s</strong>
+                                            </p>
+                                        </td>
+                                        <td>
+                                            <p>
+                                                Database servers are refusing connections
+                                            </p>
+                                        </td>
+                                    </tr>
+                                    <tr>
+                                        <td>
+                                            <p>
+                                                <strong>My responses are slow</strong>
+                                            </p>
+                                        </td>
+                                        <td>
+                                            <p>
+                                                CPUs are overloaded by a bogosort, or an Ethernet cable is crimped under a rack, visible as partial packet loss
+                                            </p>
+                                        </td>
+                                    </tr>
+                                    <tr>
+                                        <td>
+                                            <p>
+                                                <strong>Users in Antarctica aren’t receiving animated cat GIFs</strong>
+                                            </p>
+                                        </td>
+                                        <td>
+                                            <p>
+                                                Your Content Distribution Network hates scientists and felines, and thus blacklisted some client IPs
+                                            </p>
+                                        </td>
+                                    </tr>
+                                    <tr>
+                                        <td>
+                                            <p>
+                                                <strong>Private content is world-readable</strong>
+                                            </p>
+                                        </td>
+                                        <td>
+                                            <p>
+                                                A new software push caused ACLs to be forgotten and allowed all requests
+                                            </p>
+                                        </td>
+                                    </tr>
+                                </tbody>
+                            </table>
+                            <p>
+                                "What" versus "why" is one of the most important distinctions in writing good monitoring with maximum signal and minimum noise.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="black-box-versus-white-box-q8sJuw">
+                            <h1 class="heading jumptargets">
+                                Black-Box Versus White-Box
+                            </h1>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="blackbox vs. whitebox" id="id-9nCjSvFVuj"></a><a data-type="indexterm" data-primary="white-box monitoring" id="id-ZbC1FMFEu7"></a><a data-type="indexterm" data-primary="black-box monitoring" id="id-zdCXIGFvuy"></a>We combine heavy use of white-box monitoring with modest but critical uses of black-box monitoring. The simplest way to think about black-box monitoring versus white-box monitoring is that black-box monitoring is symptom-oriented and represents active—not predicted—problems: "The system isn’t working correctly, right now." White-box monitoring depends on the ability to inspect the innards of the system, such as logs or HTTP endpoints, with instrumentation. White-box monitoring therefore allows detection of imminent problems, failures masked by retries, and so forth.
+                            </p>
+                            <p>
+                                Note that in a multilayered system, one person’s symptom is another person’s cause. For example, suppose that a database’s performance is slow. Slow database reads are a symptom for the database SRE who detects them. However, for the frontend SRE observing a slow website, the same slow database reads are a cause. Therefore, white-box monitoring is sometimes symptom-oriented, and sometimes cause-oriented, depending on just how informative your white-box is.
+                            </p>
+                            <p>
+                                When collecting telemetry for debugging, white-box monitoring is essential. If web servers seem slow on database-heavy requests, you need to know both how fast the web server perceives the database to be, and how fast the database believes itself to be. Otherwise, you can’t distinguish an actually slow database server from a network problem between your web server and your database.
+                            </p>
+                            <p>
+                                For paging, black-box monitoring has the key benefit of forcing discipline to only nag a human when a problem is both already ongoing and contributing to real symptoms. On the other hand, for not-yet-occurring but imminent problems, black-box monitoring is fairly useless.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="xref_monitoring_golden-signals">
+                            <h1 class="heading jumptargets">
+                                The Four Golden Signals
+                            </h1>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="four golden signals of" id="id-ZbCxSMFjU7"></a>The four golden signals of monitoring are latency, traffic, errors, and saturation. If you can only measure four metrics of your user-facing system, focus on these four.
+                            </p>
+                            <dl>
+                                <dt class="subheaders jumptargets" id="latency">
+                                    Latency
+                                </dt>
+                                <dd>
+                                    <p>
+                                        <a data-type="indexterm" data-primary="service latency" data-secondary="monitoring for" id="id-yYCASJS9FKIWUb"></a><a data-type="indexterm" data-primary="latency" data-secondary="monitoring for" id="id-VMCpF2SXFbIwU4"></a><a data-type="indexterm" data-primary="request latency" id="id-rjCeIOSKFDIaU8"></a><a data-type="indexterm" data-primary="user requests" data-secondary="request latency monitoring" id="id-wqCDtvSGFAIMUQ"></a>The time it takes to service a request. It’s important to distinguish between the latency of successful requests and the latency of failed requests. For example, an HTTP 500 error triggered due to loss of connection to a database or other critical backend might be served very quickly; however, as an HTTP 500 error indicates a failed request, factoring 500s into your overall latency might result in misleading calculations. On the other hand, a slow error is even worse than a fast error! Therefore, it’s important to track error latency, as opposed to just filtering out errors.
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="traffic">
+                                    Traffic
+                                </dt>
+                                <dd>
+                                    <p>
+                                        <a data-type="indexterm" data-primary="user requests" data-secondary="traffic analysis" id="id-rjCXSOSxtDIaU8"></a><a data-type="indexterm" data-primary="traffic analysis" id="id-wqC4FvSBtAIMUQ"></a>A measure of how much demand is being placed on your system, measured in a high-level system-specific metric. For a web service, this measurement is usually HTTP requests per second, perhaps broken out by the nature of the requests (e.g., static versus dynamic content). For an audio streaming system, this measurement might focus on network I/O rate or concurrent sessions. For a key-value storage system, this measurement might be transactions and retrievals per <span class="keep-together">second</span>.
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="errors">
+                                    Errors
+                                </dt>
+                                <dd>
+                                    <p>
+                                        <a data-type="indexterm" data-primary="error rates" id="id-x1C4SjSlTLIMUJ"></a><a data-type="indexterm" data-primary="user requests" data-secondary="monitoring failures" id="id-PnCxFaS0TgIVUo"></a>The rate of requests that fail, either explicitly (e.g., HTTP 500s), implicitly (for example, an HTTP 200 success response, but coupled with the wrong content), or by policy (for example, "If you committed to one-second response times, any request over one second is an error"). Where protocol response codes are insufficient to express all failure conditions, secondary (internal) protocols may be necessary to track partial failure modes. Monitoring these cases can be drastically different: catching HTTP 500s at your load balancer can do a decent job of catching all completely failed requests, while only end-to-end system tests can detect that you’re serving the wrong content.
+                                    </p>
+                                </dd>
+                                <dt class="subheaders jumptargets" id="saturation">
+                                    Saturation
+                                </dt>
+                                <dd>
+                                    <p>
+                                        <a data-type="indexterm" data-primary="saturation" id="id-OnCNS2S4iDIYU8"></a>How "full" your service is. A measure of your system fraction, emphasizing the resources that are most constrained (e.g., in a memory-constrained system, show memory; in an I/O-constrained system, show I/O). Note that many systems degrade in performance before they achieve 100% utilization, so having a utilization target is essential.
+                                    </p>
+                                    <p>
+                                        In complex systems, saturation can be supplemented with higher-level load measurement: can your service properly handle double the traffic, handle only 10% more traffic, or handle even less traffic than it currently receives? For very simple services that have no parameters that alter the complexity of the request (e.g., "Give me a nonce" or "I need a globally unique monotonic integer") that rarely change configuration, a static value from a load test might be adequate. As discussed in the previous paragraph, however, most services need to use indirect signals like CPU utilization or network bandwidth that have a known upper bound. Latency increases are often a leading indicator of saturation. Measuring your 99th percentile response time over some small window (e.g., one minute) can give a very early signal of saturation.
+                                    </p>
+                                    <p>
+                                        Finally, saturation is also concerned with predictions of impending saturation, such as "It looks like your database will fill its hard drive in 4 hours."
+                                    </p>
+                                </dd>
+                            </dl>
+                            <p>
+                                If you measure all four golden signals and page a human when one signal is problematic (or, in the case of saturation, nearly problematic), your service will be at least decently covered by monitoring.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="worrying-about-your-tail-or-instrumentation-and-performance-Yms9Ck">
+                            <h1 class="heading jumptargets">
+                                Worrying About Your Tail (or, Instrumentation and Performance)
+                            </h1>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="instrumentation and performance" id="id-zdCxSGFQCy"></a><a data-type="indexterm" data-primary="performance" data-secondary="monitoring" id="id-yYCyFpFdCr"></a>When building a monitoring system from scratch, it’s tempting to design a system based upon the mean of some quantity: the mean latency, the mean CPU usage of your nodes, or the mean fullness of your databases. The danger presented by the latter two cases is obvious: CPUs and databases can easily be utilized in a very imbalanced way. The same holds for latency. If you run a web service with an average latency of 100 ms at 1,000 requests per second, 1% of requests might easily take 5 seconds.<sup><a class="jumptarget" data-type="noteref" id="id-QQLuAIXFxCz-marker" href="#id-QQLuAIXFxCz">23</a></sup> If your users depend on several such web services to render their page, the 99th percentile of one backend can easily become the median response of your <span class="keep-together">frontend</span>.
+                            </p>
+                            <p>
+                                The simplest way to differentiate between a slow average and a very slow "tail" of requests is to collect request counts bucketed by latencies (suitable for rendering a histogram), rather than actual latencies: how many requests did I serve that took between 0 ms and 10 ms, between 10 ms and 30 ms, between 30 ms and 100 ms, between 100 ms and 300 ms, and so on? Distributing the histogram boundaries approximately exponentially (in this case by factors of roughly 3) is often an easy way to visualize the distribution of your requests.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="choosing-an-appropriate-resolution-for-measurements-vJsBsE">
+                            <h1 class="heading jumptargets">
+                                Choosing an Appropriate Resolution for Measurements
+                            </h1>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="resolution" id="id-yYCASpFxsr"></a>Different aspects of a system should be measured with different levels of granularity. For example:
+                            </p>
+                            <ul>
+                                <li>Observing CPU load over the time span of a minute won’t reveal even quite long-lived spikes that drive high tail latencies.
+                                </li>
+                                <li>On the other hand, for a web service targeting no more than 9 hours aggregate downtime per year (99.9% annual uptime), probing for a 200 (success) status more than once or twice a minute is probably unnecessarily frequent.
+                                </li>
+                                <li>Similarly, checking hard drive fullness for a service targeting 99.9% availability more than once every 1–2 minutes is probably unnecessary.
+                                </li>
+                            </ul>
+                            <p>
+                                Take care in how you structure the granularity of your measurements. Collecting per-second measurements of CPU load might yield interesting data, but such frequent measurements may be very expensive to collect, store, and analyze. If your monitoring goal calls for high resolution but doesn’t require extremely low latency, you can reduce these costs by performing internal sampling on the server, then configuring an external system to collect and aggregate that distribution over time or across servers. You might:
+                            </p>
+                            <ol>
+                                <li>Record the current CPU utilization each second.
+                                </li>
+                                <li>Using buckets of 5% granularity, increment the appropriate CPU utilization bucket each second.
+                                </li>
+                                <li>Aggregate those values every minute.
+                                </li>
+                            </ol>
+                            <p>
+                                This strategy allows you to observe brief CPU hotspots without incurring very high cost due to collection and retention.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="as-simple-as-possible-no-simpler-lqskHx">
+                            <h1 class="heading jumptargets">
+                                As Simple as Possible, No Simpler
+                            </h1>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="avoiding complexity in" id="id-VMCPSrFpHm"></a>Piling all these requirements on top of each other can add up to a very complex monitoring system—your system might end up with the following levels of complexity:
+                            </p>
+                            <ul>
+                                <li>Alerts on different latency thresholds, at different percentiles, on all kinds of different metrics
+                                </li>
+                                <li>Extra code to detect and expose possible causes
+                                </li>
+                                <li>Associated dashboards for each of these possible causes
+                                </li>
+                            </ul>
+                            <p>
+                                The sources of potential complexity are never-ending. Like all software systems, monitoring can become so complex that it’s fragile, complicated to change, and a maintenance burden.
+                            </p>
+                            <p>
+                                Therefore, design your monitoring system with an eye toward simplicity. In choosing what to monitor, keep the following guidelines in mind:
+                            </p>
+                            <ul>
+                                <li>The rules that catch real incidents most often should be as simple, predictable, and reliable as possible.
+                                </li>
+                                <li>Data collection, aggregation, and alerting configuration that is rarely exercised (e.g., less than once a quarter for some SRE teams) should be up for removal.
+                                </li>
+                                <li>Signals that are collected, but not exposed in any prebaked dashboard nor used by any alert, are candidates for removal.
+                                </li>
+                            </ul>
+                            <p>
+                                In Google’s experience, basic collection and aggregation of metrics, paired with alerting and dashboards, has worked well as a relatively standalone system. (In fact Google’s monitoring system is broken up into several binaries, but typically people learn about all aspects of these binaries.) It can be tempting to combine monitoring with other aspects of inspecting complex systems, such as detailed system profiling, single-process debugging, tracking details about exceptions or crashes, load testing, log collection and analysis, or traffic inspection. While most of these subjects share commonalities with basic monitoring, blending together too many results in overly complex and fragile systems. As in many other aspects of software engineering, maintaining distinct systems with clear, simple, loosely coupled points of integration is a better strategy (for example, using web APIs for pulling summary data in a format that can remain constant over an extended period of time).
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="tying-these-principles-together-nqsJfw">
+                            <h1 class="heading jumptargets">
+                                Tying These Principles Together
+                            </h1>
+                            <p>
+                                The principles discussed in this chapter can be tied together into a philosophy on monitoring and alerting that’s widely endorsed and followed within Google SRE teams. While this monitoring philosophy is a bit aspirational, it’s a good starting point for writing or reviewing a new alert, and it can help your organization ask the right questions, regardless of the size of your organization or the complexity of your service or system.
+                            </p>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="creating rules for" id="id-wqC7SDIvfj"></a>When creating rules for monitoring and alerting, asking the following questions can help you avoid false positives and pager burnout:<sup><a class="jumptarget" data-type="noteref" id="id-a82udF8IBfx-marker" href="#id-a82udF8IBfx">24</a></sup>
+                            </p>
+                            <ul>
+                                <li>Does this rule detect <em>an otherwise undetected condition</em> that is urgent, actionable, and actively or imminently user-visible?<sup><a class="jumptarget" data-type="noteref" id="id-0vYuEFpSjSMtLfG-marker" href="#id-0vYuEFpSjSMtLfG">25</a></sup>
+                                </li>
+                                <li>Will I ever be able to ignore this alert, knowing it’s benign? When and why will I be able to ignore this alert, and how can I avoid this scenario?
+                                </li>
+                                <li>Does this alert definitely indicate that users are being negatively affected? Are there detectable cases in which users aren’t being negatively impacted, such as drained traffic or test deployments, that should be filtered out?
+                                </li>
+                                <li>Can I take action in response to this alert? Is that action urgent, or could it wait until morning? Could the action be safely automated? Will that action be a long-term fix, or just a short-term workaround?
+                                </li>
+                                <li>Are other people getting paged for this issue, therefore rendering at least one of the pages unnecessary?
+                                </li>
+                            </ul>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="monitoring philosophy" id="id-PnCpSwhJfa"></a>These questions reflect a fundamental philosophy on pages and pagers:
+                            </p>
+                            <ul>
+                                <li>Every time the pager goes off, I should be able to react with a sense of urgency. I can only react with a sense of urgency a few times a day before I become fatigued.
+                                </li>
+                                <li>Every page should be actionable.
+                                </li>
+                                <li>Every page response should require intelligence. If a page merely merits a robotic response, it shouldn’t be a page.
+                                </li>
+                                <li>Pages should be about a novel problem or an event that hasn’t been seen before.
+                                </li>
+                            </ul>
+                            <p>
+                                Such a perspective dissipates certain distinctions: if a page satisfies the preceding four bullets, it’s irrelevant whether the page is triggered by white-box or black-box monitoring. This perspective also amplifies certain distinctions: it’s better to spend much more effort on catching symptoms than causes; when it comes to causes, only worry about very definite, very imminent causes.
+                            </p>
+                        </section>
+                        <section data-type="sect1" id="monitoring-for-the-long-term-NbsNS8">
+                            <h1 class="heading jumptargets">
+                                Monitoring for the Long Term
+                            </h1>
+                            <p>
+                                <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="challenges of" id="id-wqC7SPFMSj"></a>In modern production systems, monitoring systems track an ever-evolving system with changing software architecture, load characteristics, and performance targets. An alert that’s currently exceptionally rare and hard to automate might become frequent, perhaps even meriting a hacked-together script to resolve it. At this point, someone should find and eliminate the root causes of the problem; if such resolution isn’t possible, the alert response deserves to be fully automated.
+                            </p>
+                            <p>
+                                It’s important that decisions about monitoring be made with long-term goals in mind. Every page that happens today distracts a human from improving the system for tomorrow, so there is often a case for taking a short-term hit to availability or performance in order to improve the long-term outlook for the system. Let’s take a look at two case studies that illustrate this trade-off.
+                            </p>
+                            <section data-type="sect2" id="bigtable-sre-a-tale-of-over-alerting-dbsXtjSM">
+                                <h2 class="subheaders jumptargets">
+                                    Bigtable SRE: A Tale of Over-Alerting
+                                </h2>
+                                <p>
+                                    <a data-type="indexterm" id="MDSbig6" data-primary="monitoring distributed systems" data-secondary="case studies"></a><a data-type="indexterm" data-primary="Bigtable" id="id-XmCpFOFytySv"></a>Google’s internal infrastructure is typically offered and measured against a service level objective (SLO; see <a data-type="xref" href="/sre/sre-book/chapters/service-level-objectives">Service Level Objectives</a>). Many years ago, the Bigtable service’s SLO was based on a synthetic well-behaved client’s mean performance. Because of problems in Bigtable and lower layers of the storage stack, the mean performance was driven by a "large" tail: the worst 5% of requests were often significantly slower than the rest.
+                                </p>
+                                <p>
+                                    Email alerts were triggered as the SLO approached, and paging alerts were triggered when the SLO was exceeded. Both types of alerts were firing voluminously, consuming unacceptable amounts of engineering time: the team spent significant amounts of time triaging the alerts to find the few that were really actionable, and we often missed the problems that actually affected users, because so few of them did. Many of the pages were non-urgent, due to well-understood problems in the infrastructure, and had either rote responses or received no response.
+                                </p>
+                                <p>
+                                    To remedy the situation, the team used a three-pronged approach: while making great efforts to improve the performance of Bigtable, we also temporarily dialed back our SLO target, using the 75th percentile request latency. We also disabled email alerts, as there were so many that spending time diagnosing them was infeasible.
+                                </p>
+                                <p>
+                                    This strategy gave us enough breathing room to actually fix the longer-term problems in Bigtable and the lower layers of the storage stack, rather than constantly fixing tactical problems. On-call engineers could actually accomplish work when they weren’t being kept up by pages at all hours. Ultimately, temporarily backing off on our alerts allowed us to make faster progress toward a better service.
+                                </p>
+                            </section>
+                            <section data-type="sect2" id="gmail-predictable-scriptable-responses-from-humans-BVs1h4SD">
+                                <h2 class="subheaders jumptargets">
+                                    Gmail: Predictable, Scriptable Responses from Humans
+                                </h2>
+                                <p>
+                                    <a data-type="indexterm" data-primary="Gmail" id="id-XmC9SOFZhySv"></a>In the very early days of Gmail, the service was built on a retrofitted distributed process management system called Workqueue, which was originally created for batch processing of pieces of the search index. Workqueue was "adapted" to long-lived processes and subsequently applied to Gmail, but certain bugs in the relatively opaque codebase in the scheduler proved hard to beat.
+                                </p>
+                                <p>
+                                    At that time, the Gmail monitoring was structured such that alerts fired when individual tasks were “de-scheduled” by Workqueue. This setup was less than ideal because even at that time, Gmail had many, many thousands of tasks, each task representing a fraction of a percent of our users. We cared deeply about providing a good user experience for Gmail users, but such an alerting setup was unmaintainable.
+                                </p>
+                                <p>
+                                    To address this problem, Gmail SRE built a tool that helped “poke” the scheduler in just the right way to minimize impact to users. The team had several discussions about whether or not we should simply automate the entire loop from detecting the problem to nudging the rescheduler, until a better long-term solution was achieved, but some worried this kind of workaround would delay a real fix.
+                                </p>
+                                <p>
+                                    This kind of tension is common within a team, and often reflects an underlying mistrust of the team’s self-discipline: while some team members want to implement a “hack” to allow time for a proper fix, others worry that a hack will be forgotten or that the proper fix will be deprioritized indefinitely. This concern is credible, as it’s easy to build layers of unmaintainable technical debt by patching over problems instead of making real fixes. Managers and technical leaders play a key role in implementing true, long-term fixes by supporting and prioritizing potentially time-consuming long-term fixes even when the initial “pain” of paging subsides.
+                                </p>
+                                <p>
+                                    Pages with rote, algorithmic responses should be a red flag. Unwillingness on the part of your team to automate such pages implies that the team lacks confidence that they can clean up their technical debt. This is a major problem worth escalating.<a data-type="indexterm" data-primary="" data-startref="MDSbig6" id="id-oPCASqT2hLSk"></a>
+                                </p>
+                            </section>
+                            <section data-type="sect2" id="the-long-run-MQsWTMS7">
+                                <h2 class="subheaders jumptargets">
+                                    The Long Run
+                                </h2>
+                                <p>
+                                    <a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="short- vs. long-term availability" id="id-jyCxSoFETNSd"></a>A common theme connects the previous examples of Bigtable and Gmail: a tension between short-term and long-term availability. Often, sheer force of effort can help a rickety system achieve high availability, but this path is usually short-lived and fraught with burnout and dependence on a small number of heroic team members. Taking a controlled, short-term decrease in availability is often a painful, but strategic trade for the long-run stability of the system. It’s important not to think of every page as an event in isolation, but to consider whether the overall <em>level</em> of paging leads toward a healthy, appropriately available system with a healthy, viable team and long-term outlook. We review statistics about page frequency (usually expressed as incidents per shift, where an incident might be composed of a few related pages) in quarterly reports with management, ensuring that decision makers are kept up to date on the pager load and overall health of their teams.
+                                </p>
+                            </section>
+                        </section>
+                        <section data-type="sect1" id="conclusion-8ksvFj">
+                            <h1 class="heading jumptargets">
+                                Conclusion
+                            </h1>
+                            <p>
+                                A healthy monitoring and alerting pipeline is simple and easy to reason about. It focuses primarily on symptoms for paging, reserving cause-oriented heuristics to serve as aids to debugging problems. Monitoring symptoms is easier the further "up" your stack you monitor, though monitoring saturation and performance of subsystems such as databases often must be performed directly on the subsystem itself. Email alerts are of very limited value and tend to easily become overrun with noise; instead, you should favor a dashboard that monitors all ongoing subcritical problems for the sort of information that typically ends up in email alerts. A dashboard might also be paired with a log, in order to analyze historical correlations.
+                            </p>
+                            <p>
+                                Over the long haul, achieving a successful on-call rotation and product includes choosing to alert on symptoms or imminent real problems, adapting your targets to goals that are actually achievable, and making sure that your monitoring supports rapid diagnosis.
+                            </p>
+                        </section>
+                        <div class="footnotes" data-type="footnotes">
+                            <p data-type="footnote" id="id-LvQuvtYS7UvI8h4">
+                                <sup><a class="jumptargets" href="#id-LvQuvtYS7UvI8h4-marker">22</a></sup>Sometimes known as "alert spam," as they are rarely read or acted on.
+                            </p>
+                            <p data-type="footnote" id="id-QQLuAIXFxCz">
+                                <sup><a class="jumptargets" href="#id-QQLuAIXFxCz-marker">23</a></sup>If 1% of your requests are 50x the average, it means that the rest of your requests are about twice as fast as the average. But if you’re not measuring your distribution, the idea that most of your requests are near the mean is just hopeful thinking.
+                            </p>
+                            <p data-type="footnote" id="id-a82udF8IBfx">
+                                <sup><a class="jumptargets" href="#id-a82udF8IBfx-marker">24</a></sup>See <em>Applying Cardiac Alarm Management Techniques to Your On-Call</em> <a data-type="xref" href="/sre/sre-book/chapters/bibliography#Hol14" target="_blank">[Hol14]</a> for an example of alert fatigue in another context.
+                            </p>
+                            <p data-type="footnote" id="id-0vYuEFpSjSMtLfG">
+                                <sup><a class="jumptargets" href="#id-0vYuEFpSjSMtLfG-marker">25</a></sup>Zero-redundancy (<em>N</em> + 0) situations count as imminent, as do "nearly full" parts of your service! For more details about the concept of redundancy, see <a href="https://en.wikipedia.org/wiki/N%2B1_redundancy" target="_blank"><em class="hyperlink">https://en.wikipedia.org/wiki/N%2B1_redundancy</em></a>.
+                            </p>
+                        </div>
+                    </section>
+                </div>
+            </div>
+            <div class="footer">
+                <div class="maia-aux">
+                    <div class="previous">
+                        <a href="/sre/sre-book/chapters/eliminating-toil">
+                        <p class="footer-caption">
+                            Previous
+                        </p>
+                        <p class="chapter-link">
+                            Chapter 5 - Eliminating Toil
+                        </p></a>
+                    </div>
+                    <div class="next">
+                        <a href="/sre/sre-book/chapters/automation-at-google">
+                        <p class="footer-caption">
+                            Next
+                        </p>
+                        <p class="chapter-link">
+                            Chapter 7 - The Evolution of Automation at Google
+                        </p></a>
+                    </div>
+                    <p class="footer-link">
+                        Copyright © 2017 Google, Inc. Published by O'Reilly Media, Inc. Licensed under <a href="https://creativecommons.org/licenses/by-nc-nd/4.0/" target="_blank">CC BY-NC-ND 4.0</a>
+                    </p>
+                </div>
+            </div>
+        </main>
+        <script src="//ajax.googleapis.com/ajax/libs/angularjs/1.6.6/angular.min.js"></script> 
+        <script src="//ajax.googleapis.com/ajax/libs/angularjs/1.6.6/angular-animate.min.js"></script> 
+        <script src="//ajax.googleapis.com/ajax/libs/angularjs/1.6.6/angular-touch.min.js"></script> 
+        <script src="/sre/sre-book/static/js/index.min.js?cache=5b7f90b"></script>
+    </body>
+</html>

article_scraper/resources/tests/readability/guardian-1/expected.html

@@ -0,0 +1,284 @@
+<article><div itemprop="articleBody" data-test-id="article-review-body" id="readability-page-1">
+                            <p>
+                                <span><span>W</span></span>hale whisperer Hori Parata was just seven years old when he attended his first mass stranding, a beaching of porpoises in New Zealand’s Northland, their cries screeching through the air on the deserted stretch of sand.
+                            </p>
+                            <p>
+                                Seven decades later, Parata, 75, has now overseen more than 500 strandings and is renowned in <a href="https://www.theguardian.com/world/newzealand" data-link-name="auto-linked-tag" data-component="auto-linked-tag" target="_blank">New Zealand</a> as the leading Māori whale expert, called on by tribes around the country for cultural guidance as marine strandings become increasingly complex and fatal.
+                            </p>
+                            <p>
+                                “Man’s greed in the ocean is hurting the whales,” says Parata, a fierce and uncompromising elder of the Ngātiwai tribe of eastern Northland.
+                            </p>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="05cb692c634cd90e5411aab92ca3e649474ff786" id="img-2">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f5b3d20f0dc5c22a83f96fb709ecd204"/>
+                                <meta itemprop="width" content="4800"/>
+                                <meta itemprop="height" content="3200"/><a href="#img-2" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="880px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=adbbf5d870d9cc7f0b9a24eb5472ebf3 1760w"/> <source media="(min-width: 1300px)" sizes="880px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f0a6d8fa60b5571e9e0e9a5673e407b7 880w"/> <source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="800px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=d39897e20bb677fda3feb14113aad381 1600w"/> <source media="(min-width: 1140px)" sizes="800px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=dbf240ee147c5a0a43321f1634ee41eb 800w"/> <source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="640px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f7926915234cc22c9fe718771f5837cc 1280w"/> <source media="(min-width: 980px)" sizes="640px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=fa3edf920739ca39d41e3fce38ab277b 640w"/> <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f9fd7969943957bd4893fe29d248626c 1240w"/> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=61ab70443d54f672febc609b4bfbc5c0 620w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=a416bff5ba9e0d62f1634aee83308528 1210w"/> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=5ae3b22ecb3c7bde8b49a212d52b707c 605w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=c43d04fbea54b99991b541ce674da43d 890w"/> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a0c813b07b8d5b99a33a7133ea7185db 445w"/> 
+                                    <img itemprop="contentUrl" alt="Hori Parata at his P&#x101;taua farm, the place where he was born and grew up." src="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=575838a657b26493e956c7f84b058080"/></picture>
+                                </div></a>
+                            </figure>
+                            <ul>
+                                <li>
+                                    <p>
+                                        Hori Parata at his Pātaua farm, the place where he was born and grew up
+                                    </p>
+                                </li>
+                            </ul>
+                            <p>
+                                “We’re having to put up with a lot of stuff today. The public want to hug the whales, they want to touch them, they want to feel good – that’s not the thing. We feel that is ridiculous.”
+                            </p>
+                            <p>
+                                Whale experts regard New Zealand – or Aotearoa as it is called by Māori – as the whale stranding capital of the world, with more than 5,000 incidents recorded since 1840, and an average of 300 individual animals beaching themselves each year.
+                            </p>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="98c683a7df9c83b2c13de2d93ca1825199ed5150" id="img-3">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d2432083289db6bd42f6ad7adc64a78d"/>
+                                <meta itemprop="width" content="4800"/>
+                                <meta itemprop="height" content="3166"/><a href="#img-3" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=7234718ccc92b4251bdfe4c505f064f8 1240w"/> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b80f7e8fcfbbe8e12fa6954ddeb3bd1e 620w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=78ebc960d0c15f627bfbe60cd770a139 1210w"/> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1319684ba57c18162a45e56384d418b0 605w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=3170b3a80d76d6c6422b65045444829e 890w"/> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e835570164cc241d4009af47fc1051f7 445w"/> 
+                                    <img itemprop="contentUrl" alt="Kauri (Tekaurinui Robert) Parata, watched by his father Hori Parata, carves a traditional Ma&#x14D;ri design at their home in Whang&#x101;rei. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata" src="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2f198e1958f140f3ac664a3fdd87177c"/></picture>
+                                </div></a>
+                            </figure>
+                            <ul>
+                                <li>
+                                    <p>
+                                        Kauri (Te Kaurinui Robert) Parata, watched by his father, Hori Parata, carves a traditional Māori design at their home in Whangārei. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father
+                                    </p>
+                                </li>
+                            </ul>
+                            <p>
+                                Concrete information on why whales strand remains elusive, but “sickness, navigational error, geographical features, a rapidly falling tide, being chased by a predator, or extreme weather” are all thought to contribute, according to the New Zealand Department of Conservation.
+                            </p>
+                            <p>
+                                Climate change is to blame too, <a href="https://www.radionz.co.nz/news/national/377272/new-zealand-beached-whales-why-are-so-many-getting-stranded" data-link-name="in body link" target="_blank">scientists think</a>, with warming ocean temperatures moving whales’ prey closer to the shore and forcing them to pursue their food into shallow waters.
+                            </p>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="0447972cf47ca67882fcfc648edf7e574b0853bc" id="img-4">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0e7302276c6cc7cb65be9bcdacd3081f"/>
+                                <meta itemprop="width" content="4800"/>
+                                <meta itemprop="height" content="3200"/><a href="#img-4" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=933c5693fbf9195f6e83a8928283f85e 1240w"/> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e07b988894308f3bfc052d1ff9dfc1e2 620w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=113cf92fe80bc5996634c34b3c7a0c09 1210w"/> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=5a7358223e80941bd1b0e0f427beefde 605w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=292e941797de1671c185c1b074c688ad 890w"/> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d1bc1c4d9341f9b6d63b64861a3de711 445w"/> 
+                                    <img itemprop="contentUrl" alt="A bin of small whale bones." src="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=718132fd888108a18383c24a8425523b"/></picture>
+                                </div></a>
+                            </figure>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="416800b8d06039780c3e6de28564e6f277b4e7b7" id="img-5">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=249ddc3119aa637c3a1ae4998509f604"/>
+                                <meta itemprop="width" content="4800"/>
+                                <meta itemprop="height" content="3200"/><a href="#img-5" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=4197b7a2bc3c7e32d3eaee927b9d08e6 1240w"/> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=82d6908c6ffe622566e1cc6513d60ddd 620w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=222e300fb2a60da1a841fbbc2bc8f752 1210w"/> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=63ba1cfe4b5e7c7d741d311d58997fcf 605w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=231139825b311ede8d01e6c702d6d12c 890w"/> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1065dd4439e14c72fe520c19566172e2 445w"/> 
+                                    <img itemprop="contentUrl" alt="The baleen recovered from a stranded Pygmy Right Whale." src="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c5a53178ebbe54490c97fad6b5e032c4"/></picture>
+                                </div></a>
+                            </figure>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="8c207197c0a9e6f407dcddfded5f868a142c9cab" id="img-6">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=bcfefc5593dcc2bdb0dd3b0543cfa4eb"/>
+                                <meta itemprop="width" content="4800"/>
+                                <meta itemprop="height" content="3200"/><a href="#img-6" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=d272c4dc57408daa0dcdf5461a230e9e 1240w"/> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=890f0e0cee54663a90292471a16f95d2 620w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=59a57c5bded93fe0efe0046226fe7c69 1210w"/> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f1b2a4c79e965aa76a322ad25072a052 605w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=ffbdfd671eb45e3a87e3dc9137e8b006 890w"/> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c560f0b0ef9736c1a66215cc29c59d43 445w"/> 
+                                    <img itemprop="contentUrl" alt="Squid beaks, from the stomach of a Sperm Whale." src="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0dd659ae339b1aea9a99ed7f6f8eadeb"/></picture>
+                                </div></a>
+                            </figure>
+                            <ul>
+                                <li>
+                                    <p>
+                                        Clockwise from top: small whale bones; squid beaks, from the stomach of a sperm whale; the baleen filter-feeder system recovered from a stranded pygmy right whale.
+                                    </p>
+                                </li>
+                            </ul>
+                            <h2>
+                                ‘Unprecedented’ strandings
+                            </h2>
+                            <p>
+                                November marked the beginning of whale stranding season, and it started with a surge in incidents, according to whale rescue group Project Jonah, with <a href="https://www.theguardian.com/environment/video/2018/nov/26/whales-die-stranded-stewart-island-new-zealand-beach-video" data-link-name="in body link" target="_blank">140 pilot whales</a> beaching and dying on Stewart Island, 10 rare pygmy whales on Ninety Mile beach, 51 stranded and dead on the Chatham Islands and a spate of individual cases around the country.
+                            </p>
+                            <p>
+                                And as more whales beach and die – from exhaustion, heat stroke or seagulls feasting on their flesh – an acute sense of grief is growing among New Zealand’s indigenous people, who regard whales as their ancestors and <em>taonga</em> (treasures).
+                            </p>
+                            <p>
+                                “These days it is like a zoo. People just want to come and gawk at us, without even trying to understand what is happening with the animals and the environment,” says Parata, bristling with anger.
+                            </p>
+                            <figure data-interactive="https://interactive.guim.co.uk/embed/iframe-wrapper/0.1/boot.js" data-canonical-url="https://interactive.guim.co.uk/uploader/embed/2019/01/stranded_whales/giv-3902U99iNUM3iDSZ/" data-alt="whale strandings">
+                                <a href="https://interactive.guim.co.uk/uploader/embed/2019/01/stranded_whales/giv-3902U99iNUM3iDSZ/" data-link-name="in body link" target="_blank">whale strandings</a>
+                            </figure>
+                            <p>
+                                “When will we talk about what is hurting these animals out on the sea? They are drowning out there, they can’t breathe, they beach themselves to be with the Aunties.”
+                            </p>
+                            <p>
+                                Ngātiwai believe the whales beach when they are ready to die and want to return to their families, the Māori people. Then, their human families use the whales’ gift of their bodies for sacred carvings, for traditional medicines, and even for compost.
+                            </p>
+                            <p>
+                                There are marked tribal differences across New Zealand and while some tribes work to refloat stranded whales, others like Parata’s Ngātiwai stand back and allow the Department of Conservation and volunteer groups to take the lead in rescue efforts.
+                            </p>
+                            <p>
+                                Then the tribe moves in en masse and holds a <em>karakia</em> (prayer), names each animal and sets to work removing their bones, blubber, eyes and teeth for cultural purposes.
+                            </p>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="0d13adeb0790af5c5fa317ce477c323d0e1c773c" id="img-7">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d73916fda0f6c60b5eebcb71fea643e9"/>
+                                <meta itemprop="width" content="4800"/>
+                                <meta itemprop="height" content="2334"/><a href="#img-7" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1300&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=524b92a6590a1b5c139bfe50581476a1 2600w"/> <source media="(min-width: 1300px)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=837d7df48e565ffad49febab20fbd179 1300w"/> <source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1140&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=fe98f4515ae3490f53804235a4dcf84b 2280w"/> <source media="(min-width: 1140px)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1140&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a3ec73ab7f20798ec264f5ecb59f6bcc 1140w"/> <source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1125&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=3ac3fd319accb248cf8a13e8ef55ebcc 2250w"/> <source media="(min-width: 980px)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1125&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2561c80af0d0e8eed2fc7c535be48696 1125w"/> <source media="(min-width: 740px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 740px) and (min-resolution: 120dpi)" sizes="965px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=965&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=539617a30ea852b01218eec69d6066e6 1930w"/> <source media="(min-width: 740px)" sizes="965px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=965&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=624139eae65dd4c96efd1a90243b9286 965w"/> <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="725px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=725&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=633619afab6a0ee26a3839b69160e854 1450w"/> <source media="(min-width: 660px)" sizes="725px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=725&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=98b9351340e8ee30063a4b6eb6501d6f 725w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="645px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=645&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e11a31f8adaab06b457fd9c7dda6bf6d 1290w"/> <source media="(min-width: 480px)" sizes="645px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=645&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=912104c53b38d1cd424bfb82302480e1 645w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="465px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=465&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=209259fab8ec9e81386838f2d5cdecc3 930w"/> <source media="(min-width: 0px)" sizes="465px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=465&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=06848a63793084a21bc86aa613b126a7 465w"/> 
+                                    <img itemprop="contentUrl" alt="Buck Cullen with his daughter Kaiarahi (10 months) in his back yard where he is storing a pair of massive Sperm Whale jawbones. Buck is a integral member of the whale recovery team, alongside Hori Parata." src="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=41b88ee343b9be76688b88443c7a8958"/></picture>
+                                </div></a>
+                            </figure>
+                            <ul>
+                                <li>
+                                    <p>
+                                        Buck Cullen with his daughter Kaiarahi (10 months) in his backyard, where he is storing a pair of massive sperm whale jawbones. Cullen is an integral member of Hori Parata’s whale recovery team
+                                    </p>
+                                </li>
+                            </ul>
+                            <p>
+                                But indigenous elders say they aren’t being listened to when they tell the government their whale kin are sick, and trying to escape an increasingly polluted and unpredictable ocean.
+                            </p>
+                            <p>
+                                Earlier this year in South Taranaki, a mass stranding that was described as <a href="https://www.stuff.co.nz/national/104249829/unprecedented-whale-strandings-reaches-11-in-total-on-taranaki-beach" data-link-name="in body link" target="_blank">“unprecedented”</a> left the local Māori tribe scrambling. Security was brought in when thieves attacked a sperm whale with an axe, trying to remove valuable teeth from its jaw.
+                            </p>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="4973b41f53b8ade499f99a305b01157eca659ca5" id="img-8">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ee921715aaf10e0816dcf15dfc3a21f5"/>
+                                <meta itemprop="width" content="1200"/>
+                                <meta itemprop="height" content="900"/><a href="#img-8" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=8eb5af62092c8a0eb0d283cb0887f5c5 1240w"/> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=9ace0f595be568263c5991993632dae8 620w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=97219f131f1013d15fed7c63308f2b2f 1210w"/> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=17191691b6cd81c2a67730d5475db08b 605w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=30d51b75767551399ba174bae5c39e94 890w"/> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=936568b6e7da0a710abcd2c015fd0a8b 445w"/> 
+                                    <img itemprop="contentUrl" alt="12 Par&#x101;oa Whales (Sperm Whales) recently stranded on the South Taranaki coast of Kaupokonui, on a scale not seen on their coast in recent memory." src="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7c255bf6f8a27c56365a86813cdd1517"/></picture>
+                                </div></a>
+                            </figure>
+                            <ul>
+                                <li>
+                                    <p>
+                                        12 parāoa whales (sperm whales) recently stranded on the South Taranaki coast of Kaupokonui, on a scale not seen near this location in recent memory
+                                    </p>
+                                </li>
+                            </ul>
+                            <p>
+                                Parata and his 22-year-old son, Te Kaurinui Robert Parata, were called in to assist. Te Kaurinui was called after the first whale his father ever named, and left university this year to return to Whangārei and study whale <em>tikanga</em> (protocol) and carving.
+                            </p>
+                            <p>
+                                He says mass strandings are getting more local and international attention and money from donations, but traditional knowledge is being dismissed as overly spiritual.
+                            </p>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="d1941b6a6908314fab28f44da222a4c892213341" id="img-9">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=4e46b70079819caee86359a058c92be1"/>
+                                <meta itemprop="width" content="4800"/>
+                                <meta itemprop="height" content="3120"/><a href="#img-9" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=b775d649981f7bc36cf0753dd9f47862 1760w"/> <source media="(min-width: 1300px)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=3aa0de336dad5991ff92d488288fcc92 880w"/> <source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=9133cf8cd42d42bb1342dbefbfc67c22 1600w"/> <source media="(min-width: 1140px)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2916efc01252a457768bfd28b40591ee 800w"/> <source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=c7044b93d3aa73f7eac1b56ab9d77d86 1280w"/> <source media="(min-width: 980px)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2342e6519bfe1ce74a04ca96c06ed4cd 640w"/> <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=b6218129f26e09d8133bf226df3e9bed 1240w"/> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=9d20a1ca789f2efe7ba2dfdef9c28725 620w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=73fb4a4f8d59a92325fef9faa2ddf90d 1210w"/> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7894b6600fbb6af1ec6552007ad12da8 605w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=19022ce9fcafebe383880059f34add78 890w"/> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ded05f2c1859d70d3b8d3de3dcb75e0f 445w"/> 
+                                    <img itemprop="contentUrl" alt="Kauri (Tekaurinui Robert) Parata, of the New Zealand M&#x101;ori tribe Ng&#x101;ti Wai, in front of the carving shed at Hihiaua Cultural Centre in Whangarei" src="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=76a26a289728e3d625e57d32eced57d8"/></picture>
+                                </div></a>
+                            </figure>
+                            <ul>
+                                <li>
+                                    <p>
+                                        Clockwise from top: Te Kaurinui Parata, in front of the carving shed at Hihiaua Cultural Centre in Whangārei; Parata holds three whale teeth recovered from a beached whale – the middle one shows marks where a poacher had attempted to hack it out with an axe before the recovery group arrived; the Pou, a tribal identifier, in front of the carving shed.
+                                    </p>
+                                </li>
+                            </ul>
+                            <h2>
+                                ‘We need to listen’
+                            </h2>
+                            <p>
+                                Māori harvest rights over dead whales have only been officially recognised since 1998, and the practice still elicits horror from some New Zealanders and visitors.
+                            </p>
+                            <p>
+                                “Our own ancestors wouldn’t say to go down there and hug the whales. That’s a modern thing,” says Te Kaurinui.
+                            </p>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="14b462f5d9489def554e0f9f436f13aec332f7b8" id="img-10">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b8bf0c299908608310fb47e4163afb38"/>
+                                <meta itemprop="width" content="4278"/>
+                                <meta itemprop="height" content="4800"/><a href="#img-10" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=1a392a17d0c74d513ff14129ae5f7e6c 1240w"/> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d5b05458659a4a06ff6583046f712ad1 620w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=649c2127cfbab69a7d6e4085ea098d95 1210w"/> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d45dfb664b5479650ba067c6a5af16c3 605w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f94f4c652479979124e52c237804eb83 890w"/> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f6c8d3b38a8af6d197d0a60d22a326f3 445w"/> 
+                                    <img itemprop="contentUrl" alt="The Pou in front of the carving shed at Hihiaua Cultural centre" src="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a45ec7578a392eec201d2f6920b609a0"/></picture>
+                                </div></a>
+                            </figure>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="e2cf54c36f17c6894844ea0cdd4346288a002da9" id="img-11">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=3691946929b4105ba35ff9d306cb4fbc"/>
+                                <meta itemprop="width" content="3172"/>
+                                <meta itemprop="height" content="3189"/><a href="#img-11" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=be6cf9beea564e7be872193b01c329a3 1240w"/> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c74497ec55bb1b680169d62684aa803d 620w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=06b2c03638556f85da582e419cf1817a 1210w"/> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=8cc04403e3902dbe1e4a9b01b8d4e517 605w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=893cbf39d6db1d28ba8fc85419382f9d 890w"/> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=6df725aaf59e21b09434b967e05b3272 445w"/> 
+                                    <img itemprop="contentUrl" alt="Kauri (Tekaurinui Robert) Parata, holds three whale teeth recovered from a beached whale. The middle tooth shows the marks where a poacher had attempted to hack it out with an axe before the recovery group arrived. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata." src="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0bd8e9f51bdf79a6e0a15ed176cfb57d"/></picture>
+                                </div></a>
+                            </figure>
+                            <p>
+                                The Ngātiwai are investigating a possible link between the crisis of the dieback disease killing New Zealand’s native kauri trees – and <a href="https://www.theguardian.com/world/2018/jul/14/like-losing-family-time-may-be-running-out-for-new-zealands-most-sacred-tree" data-link-name="in body link" target="_blank">threatening the giant Tāne Mahuta, which may be 2,000 years old</a> – and the increase in whale strandings.
+                            </p>
+                            <p>
+                                Parata and his family believe whale oil and byproducts could be used to try to cure Kauri dieback, and want more government money and attention directed towards indigenous knowledge of the interconnectedness of the New Zealand environment, and possible indigenous solutions.
+                            </p>
+                            <p>
+                                “People dismiss us when we tell them our spiritual understanding of whales – why they are beaching, why they are hurting,” says Te Kaurinui.
+                            </p>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="b5f3736b2ba2ef4df364258b0efcaba26f571d6e" id="img-12">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=72146577e379bdf9a21bc47994de5fb6"/>
+                                <meta itemprop="width" content="4800"/>
+                                <meta itemprop="height" content="3073"/><a href="#img-12" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1300&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=4bd713ae6f60b56e36378bb2fe45e9eb 2600w"/> <source media="(min-width: 1300px)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b93c6d33ca7496220eb30d0c8bfaccf9 1300w"/> <source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1140&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f66f5a256170eced3c0a52aee235ae29 2280w"/> <source media="(min-width: 1140px)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1140&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1f60eb7cf5fc6f899c7c9af9c05df9a0 1140w"/> <source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1125&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=27526c9caf1a62110dfa16214ceccf14 2250w"/> <source media="(min-width: 980px)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1125&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7d47ba3d7898a398e1f9d8f2d19d391e 1125w"/> <source media="(min-width: 740px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 740px) and (min-resolution: 120dpi)" sizes="965px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=965&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=8974cfeb3e33bc51b55518f1bc81e68d 1930w"/> <source media="(min-width: 740px)" sizes="965px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=965&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=04f953b4ec9555178204a1621b8a1f59 965w"/> <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="725px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=725&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=7e2c1494173f83f555806120a1fe43cc 1450w"/> <source media="(min-width: 660px)" sizes="725px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=725&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=8f53cf25b514e283cce3d995220ca19b 725w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="645px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=645&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=2b770ca04273c921c8abe0a7de72030a 1290w"/> <source media="(min-width: 480px)" sizes="645px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=645&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0da775328d9ade5ca605079a9118a64a 645w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="465px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=465&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e2e51037c5a6b8bf22d87cb927e45888 930w"/> <source media="(min-width: 0px)" sizes="465px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=465&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7ecf73b514ceb1b5e0391f17c7a4d3b0 465w"/> 
+                                    <img itemprop="contentUrl" alt="Whang&#x101;rei Harbour from Tamaterau, looking south through Mangrove sprouts coming up through the harbourside silt." src="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7fcadc35b3a44ebafe3c469c6e89241d"/></picture>
+                                </div></a>
+                            </figure>
+                            <ul>
+                                <li>
+                                    <p>
+                                        Whangārei Harbour seen from Tamaterau, with mangrove sprouts coming up through the harbourside silt
+                                    </p>
+                                </li>
+                            </ul>
+                            <p>
+                                “We are not foreigners in this land. We did not take this land off anyone else. We were not lost waiting for some bullheads to tell us what was going on.”
+                            </p>
+                            <p>
+                                Kaitaia conservation department ranger Jamie Werner of Ngātiwai recently attended his first mass beaching on Ninety Mile Beach. It was the first recorded time pygmy whales had stranded on New Zealand shores.
+                            </p>
+                            <p>
+                                “I arrived at the beach and we leapfrogged between the animals. They were calling out to each other and reassuring each other,” says Werner. “It was a shock. We’re working to adapt but the ocean is changing so fast.”
+                            </p>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="d5aaf60e3a427f278747acf0c3e7ba39b39ef923" id="img-13">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1d00d3ea91d32110b40d85ff43227728"/>
+                                <meta itemprop="width" content="4800"/>
+                                <meta itemprop="height" content="3200"/><a href="#img-13" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=2a64f318f02a31bdf9f17ae01fca72a3 1760w"/> <source media="(min-width: 1300px)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=811b3a2ecee1a87b72813d328f59aa85 880w"/> <source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=47418e3f630c00af6124c471fdc54c20 1600w"/> <source media="(min-width: 1140px)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=93ec03888887d5eb6eabf07e40f2ed3b 800w"/> <source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=476d165704890f40cc5f2d74f4d09b95 1280w"/> <source media="(min-width: 980px)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=6179da6bca697f83ac063a56660aa31b 640w"/> <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=df0c5b9ccac02dcd010d7f3ed8ff8c85 1240w"/> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e9fd2eedc9bb80286a6cabd1c7c3b0e2 620w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e48f277772be7e455f9a4a4139b3bf4c 1210w"/> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2e0af911fcd7011f04ee91d445290e84 605w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=15161e07957f327b44aa124d94ae8291 890w"/> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ef6c6809f2adcff0f4ff32899095839b 445w"/> 
+                                    <img itemprop="contentUrl" alt="The skull of a Brydes whale, in the storage container at Hihiaua Cultural Centre, Whang&#x101;rei." src="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1b74b488aedb864287ff160f86d74c9d"/></picture>
+                                </div></a>
+                            </figure>
+                            <ul>
+                                <li>
+                                    <p>
+                                        Above, the skull of a bryde’s whale; right, a large-calibre bullet of the type that the New Zealand Department of Conservation uses for euthanasing stranded whales that are beyond rescue
+                                    </p>
+                                </li>
+                            </ul>
+                            <figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="3766106f73e858d5b140ae3cdd2eef84060180cd" id="img-14">
+                                <meta itemprop="url" content="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7dca3d798aef3526f42e281771d50f91"/>
+                                <meta itemprop="width" content="4800"/>
+                                <meta itemprop="height" content="3200"/><a href="#img-14" data-link-name="Launch Article Lightbox" data-is-ajax="">
+                                <div>
+                                    <picture>
+                                    <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=0daf794415132fc19259f8d2f654f57f 1240w"/> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=98ce62f9d983fe8059c936a6c6cdff33 620w"/> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=9ff4aac2a3b07867dfdfb8f470ea6977 1210w"/> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=10d1b71094ecf1722f593eb49bb2effe 605w"/> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=0705690fcb523f2781a5952f83528ff9 890w"/> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=4c748337df5f0348eb0e7d3e3ec46571 445w"/> 
+                                    <img itemprop="contentUrl" alt="A large calibre bullet of the type that the New Zealand Department of Conservation (DOC) uses for euthanasing stranded whales that are beyond rescue." src="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d2f5bb7c3c3642ac8733ca40509f6e20"/></picture>
+                                </div></a>
+                            </figure>
+                            <p>
+                                The recent spate of mass strandings has been described as “heartbreaking” by the conservation department.
+                            </p>
+                            <p>
+                                But for Parata and his family the slow, painful deaths of their ancestors are personal – and ultimately devastating – for the health of the tribe and the sea.
+                            </p>
+                            <p>
+                                “It’s very emotional. Our ancestors tell us the strandings are a sign from the sea. So what is the sea telling us? We need to listen.”
+                            </p>
+                        </div></article>

article_scraper/resources/tests/readability/heise/expected.html

@@ -0,0 +1,34 @@
+<article><div id="readability-page-1">
+            
+    
+    <figure>
+            <img src="http://3.f.ix.de/scale/geometry/600/q75/imgs/18/1/4/6/2/3/5/1/Barcode-Scanner-With-Border-fc08c913da5cea5d.jpeg"/>
+        
+            <figcaption>
+                
+                <p>1Password scannt auch QR-Codes.</p>
+                
+                
+                <p>(Bild: Hersteller)</p>
+                
+            </figcaption>
+        
+    </figure>
+    
+
+
+            <p><strong>Das in der iOS-Version bereits enthaltene TOTP-Feature ist nun auch für OS X 10.10 verfügbar. Zudem gibt es neue Zusatzfelder in der Datenbank und weitere Verbesserungen.</strong></p>
+            <p><a rel="external" target="_blank" href="https://itunes.apple.com/de/app/1password-password-manager/id443987910">AgileBits hat Version 5.3 seines bekannten Passwortmanagers 1Password für OS X freigegeben.</a> Mit dem Update wird eine praktische Funktion nachgereicht, die <a href="http://fakehost/mac-and-i/meldung/Passwortmanager-1Password-mit-groesseren-Updates-fuer-OS-X-und-iOS-2529204.html" target="_blank">die iOS-Version der Anwendung bereits seit längerem beherrscht</a>: Das direkte Erstellen von Einmal-Passwörtern. Unterstützt wird dabei der <a rel="external" target="_blank" href="https://blog.agilebits.com/2015/01/26/totp-for-1password-users/">TOTP-Standard</a> (Time-Based One-Time Passwords), den unter anderem Firmen wie Evernote, Dropbox oder Google einsetzen, um ihre Zugänge besser abzusichern. Neben Account und regulärem Passwort wird dabei dann ein Zusatzcode verlangt, der nur kurze Zeit gilt.</p>
+<p>Zur TOTP-Nutzung muss zunächst ein Startwert an 1Password übergeben werden. Das geht unter anderem per QR-Code, den die App über ein neues Scanfenster selbst einlesen kann – etwa aus dem Webbrowser. Eine Einführung in die Technik gibt <a rel="external" target="_blank" href="http://1pw.ca/TOTPvideoMac">ein kurzes Video</a>. Die TOTP-Unterstützung in 1Password erlaubt es, auf ein zusätzliches Gerät (z.B. ein iPhone) neben dem Mac zu verzichten, das den Code liefert – was allerdings auch die Sicherheit verringert, weil es keinen "echten" zweiten Faktor mehr gibt.</p>
+<p>Update 5.3 des Passwortmanagers liefert auch noch weitere Verbesserungen. So gibt es die Möglichkeit, FaceTime-Audio- oder Skype-Anrufe aus 1Password zu starten, die Zahl der Zusatzfelder in der Datenbank wurde erweitert und der Umgang mit unterschiedlichen Zeitzonen klappt besser. Die Engine zur Passworteingabe im Browser soll beschleunigt worden sein.</p>
+<p>1Password kostet aktuell knapp 50 Euro im Mac App Store und setzt in seiner aktuellen Version mindestens OS X 10.10 voraus.
+
+
+
+<span>(<a title="Ben Schwan" href="mailto:bsc@heise.de" target="_blank">bsc</a>)</span>
+<br/>
+
+
+</p>
+
+        </div></article>

article_scraper/resources/tests/readability/heise/source.html

@@ -0,0 +1,887 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" lang="de"><head>
+    <title>1Password für Mac generiert Einmal-Passwörter | Mac &amp; i</title>
+    
+
+    
+
+
+
+<meta charset="utf-8" />
+<meta content="Heise Medien" name="publisher" />
+<meta content="width=1175" name="viewport" />
+<link id="mobil_variante" href="//m.heise.de/meldung/1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html" media="only screen and (max-width: 640px)" rel="alternate" />
+
+    <link href="/mac-and-i/meldung/HBO-Zugriff-per-Apple-Geraet-verfuegbar-2596982.html" title="Vorige Meldung" rel="prev" />    <link href="/mac-and-i/meldung/GPS-Streckenerfasser-Speed-PRO-aktuell-kostenlos-2597093.html" title="Naechste Meldung" rel="next" /><link href="/mac-and-i/" title="Startseite" type="text/html" rel="home" />
+<link href="/mac-and-i/impressum.html" title="Copyright" rel="copyright" />    <meta content="2015-04-08T12:46:00" name="date" />
+    <meta content="1Password, Mac OS X, Passwort, Passwortmanager, Sicherheit, TOTP" name="keywords" />
+            <meta content="News" name="topic" />
+        <meta content="Das in der iOS-Version bereits enthaltene TOTP-Feature ist nun auch für OS X 10.10 verfügbar. Zudem gibt es neue Zusatzfelder in der Datenbank und weitere Verbesserungen." name="description" />
+        <meta content="1Password für Mac generiert Einmal-Passwörter" name="fulltitle" />
+        <meta content="1Password für Mac generiert Einmal-Passwörter" name="DC.title" />
+        <meta content="Das in der iOS-Version bereits enthaltene TOTP-Feature ist nun auch für OS X 10.10 verfügbar. Zudem gibt es neue Zusatzfelder in der Datenbank und weitere Verbesserungen." name="DC.description" />
+        <meta scheme="DCTERMS.URI" content="http://heise.de/-2596987" name="DC.identifier" />
+        <meta content="" name="kill_switch" />    <!--googleoff: all-->
+    <meta content="1Password für Mac generiert Einmal-Passwörter" property="og:title" />
+    <meta content="website" property="og:type" />
+    <meta content="de_DE" property="og:locale" />
+    <meta content="http://www.heise.de/mac-and-i/meldung/1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html" property="og:url" />
+    <meta content="Mac &amp; i" property="og:site_name" />
+            <meta content="http://www.heise.de/imgs/18/1/4/6/2/3/5/1/Barcode-Scanner-With-Border-f0c62350bd8d9d96.jpeg" property="og:image" />
+    <meta content="Das in der iOS-Version bereits enthaltene TOTP-Feature ist nun auch für OS X 10.10 verfügbar. Zudem gibt es neue Zusatzfelder in der Datenbank und weitere Verbesserungen." property="og:description" />
+
+    <!--googleon: all--><!--googleoff: all-->
+<meta content="InterRed V15.4.1, http://www.interred.de/, InterRed GmbH" name="generator" />
+<!--googleon: all-->
+
+    
+
+
+
+
+
+
+
+<script src="//cdn.optimizely.com/js/2310910166.js"></script>
+
+
+    
+<script src="/js/jquery/jquery-1.7.1.min.js" type="text/javascript"></script>
+    
+
+    
+<script src="/js/plugins/jquery-ui-1.8.18_cycle.custom.min.js" type="text/javascript"></script>
+    
+
+
+
+<script src="/js/plugins/jquery.equalheights.min.js" type="text/javascript"></script>
+
+
+<script src="/support/lib/teaser_linking.js" type="text/javascript"></script>
+
+<script src="/js/ho/link_inline_images.min.js" type="text/javascript"></script>
+
+
+
+<script src="//script.ioam.de/iam.js" type="text/javascript"></script>
+
+
+<script src="//ad.yieldlab.net/yp/66424,66430,66442,66444,66446?ts=20150409195044" type="text/javascript"></script>
+
+
+
+
+
+
+    
+<link type="text/css" rel="stylesheet" href="/stil/standard2008.css?383ad47c92d5add95b5f" />
+<link media="print" type="text/css" rel="stylesheet" href="/stil/drucken.css?fead9096de1c0d0ca94e" />
+
+<link type="text/css" rel="stylesheet" href="/stil/heise.css?65bb0f586a7f0285f4fc" />
+
+
+    <!--googleoff: all-->
+    <link rel="shortcut icon" href="/favicon_mac-and-i.ico" />
+    <link href="mailto:redaktion%40mac%2Dand%2Di.de" title="Kontakt" rel="author" />
+    <link href="http://www.heise.de/mac-and-i/suche/" title="Suche" rel="search" />
+    <link type="application/atom+xml" href="http://www.heise.de/mac-and-i/news-atom.xml" title="Aktuelle News von Mac &amp; i" rel="alternate" />
+    <link type="application/rss+xml" href="http://www.heise.de/mac-and-i/news.rdf" title="Aktuelle News von Mac &amp; i (für ältere RSS-Reader)" rel="alternate" />
+    <link href="/mac-and-i/icons/apple-touch-icon-57x57-precomposed.png" rel="apple-touch-icon-precomposed" />
+    <link href="/mac-and-i/icons/apple-touch-icon-114x114-precomposed.png" sizes="114x114" rel="apple-touch-icon-precomposed" />
+    <link href="/mac-and-i/icons/apple-touch-icon-72x72-precomposed.png" sizes="72x72" rel="apple-touch-icon-precomposed" />
+    <meta content="Magazin rund um Apple" name="application-name" />
+    <meta content="Mac &amp; i" name="msapplication-tooltip" />
+    <meta content="/mac-and-i/" name="msapplication-starturl" />
+    <meta content="#666666" name="msapplication-TileColor" />
+    <meta content="/mac-and-i/icons/windows_tiny.png" name="msapplication-square70x70logo" />
+    <meta content="/mac-and-i/icons/windows_square.png" name="msapplication-square150x150logo" />
+    <meta content="/mac-and-i/icons/windows_wide.png" name="msapplication-wide310x150logo" />
+    <meta content="/mac-and-i/icons/windows_large.png" name="msapplication-square310x310logo" />
+    <meta content="frequency=30;polling-uri=http://notifications.buildmypinnedsite.com/?feed=http://www.heise.de/mac-and-i/news-atom.xml&amp;id=1;polling-uri2=http://notifications.buildmypinnedsite.com/?feed=http://www.heise.de/mac-and-i/news-atom.xml&amp;id=2;polling-uri3=http://notifications.buildmypinnedsite.com/?feed=http://www.heise.de/mac-and-i/news-atom.xml&amp;id=3;polling-uri4=http://notifications.buildmypinnedsite.com/?feed=http://www.heise.de/mac-and-i/news-atom.xml&amp;id=4;polling-uri5=http://notifications.buildmypinnedsite.com/?feed=http://www.heise.de/mac-and-i/news-atom.xml&amp;id=5; cycle=1" name="msapplication-notification" />
+    <meta content="Mac &amp; i" name="DC.creator" />
+    <link type="text/css" rel="stylesheet" href="/stil/mac-and-i/mac2012.css?29c941bf0f8196521a1c" />
+    <link rel="publisher" href="https://plus.google.com/117217265406849882611" />
+    <script src="/support/lib/jquery/jquery.clearfield.js?ce79f657201b214e3560" type="text/javascript"></script>
+    
+
+
+
+<style type="text/css"></style></head>
+
+<body class="apple">
+
+
+
+
+
+
+
+    
+        <div class="heisetopnavi heisetopnavi_relaunch">
+    <div class="heisetopnavi_header">
+        <header role="banner">
+            <div class="heisetopnavi_login">
+                <script src="/js/ho/login.min.js" type="text/javascript"></script>
+                <div id="navi_login">
+                    <p><span class="heise_foren">Heise-Foren:</span> <a href="https://www.heise.de/login?forward=http%3A%2F%2Fwww.heise.de%2Fmac-and-i%2Fmeldung%2F1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html">Einloggen</a> | <a href="https://www.heise.de/register?forward=http%3A%2F%2Fwww.heise.de%2Fmac-and-i%2Fmeldung%2F1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html">Registrieren</a></p>
+                </div>
+            </div>
+            <nav role="navigation">
+                <div class="heisetopnavi_hover_legacy">
+                    <a title="heise online" class="heisetopnavi_logo" href="/index.html/from/navi_oben_ho">
+                        <img width="200" height="44" id="heisetopnavi_ho_logo_top_img" alt="heise online" src="//www.heise.de/icons/ho/heise_online_logo_top.gif" />
+                    </a>
+                    <a aria-haspopup="true" class="heisetopnavi_button" href="#heisetopnavi_sub_container">Menü auf-/zuklappen</a>
+                    <div id="heisetopnavi_sub_container">
+                        <ul id="heisetopnavi_sub">
+                            <li>
+                                <ul>
+                                    <li class="heisetopnavi_ho"><a name="dachzeile.ho.ho" title="heise online – IT-News" href="/index.html/from/navi_oben_ho">News</a></li>
+                                    <li class="heisetopnavi_ct"><a name="dachzeile.ho.ct" title="c't – Magazin für Computertechnik" href="/ct/from/navi_oben_ct">c't</a></li>
+                                    <li class="heisetopnavi_ix"><a name="dachzeile.ho.ix" title="iX – Magazin für professionelle Informationstechnik" href="/ix/from/navi_oben_ix">iX</a></li>
+                                    <li class="heisetopnavi_tr"><a name="dachzeile.ho.tr" title="Technology Review – Das M.I.T.-Magazin für Innovation" href="/tr/from/navi_oben_tr">Technology Review</a></li>
+                                    <li class="heisetopnavi_mac"><a name="dachzeile.ho.mac-and-i" title="Mac &amp; i – Nachrichten, Tests, Tipps und Meinungen rund um Apple" href="/mac-and-i/from/navi_oben_mac">Mac &amp; i</a></li>
+                                    <li class="heisetopnavi_tp"><a name="dachzeile.ho.tp" title="Telepolis" href="/tp/from/navi_oben_tp">Telepolis</a></li>
+                                    <li class="heisetopnavi_hh"><a name="dachzeile.ho.make" title="Make – Kreativ mit Technik" href="/make/from/navi_oben_hh">Make</a></li>
+                                    <li class="heisetopnavi_df"><a name="dachzeile.ho.digitale-fotografie" title="Digitale Fotografie" href="/foto/special/from/navi_oben_df">Digitale Fotografie</a></li>
+                                </ul>
+                            </li>
+                            <li>
+                                <ul>
+                                    <li class="heisetopnavi_aut"><a name="dachzeile.ho.autos" title="heise Autos – News, Tests, Technik, Service rund ums Auto" href="/autos/from/navi_oben_aut">heise Autos</a></li>
+                                    <li class="heisetopnavi_dev"><a name="dachzeile.ho.developer" title="heise Developer – Informationen für Entwickler" href="/developer/from/navi_oben_dev">heise Developer</a></li>
+                                    <li class="heisetopnavi_foto"><a name="dachzeile.ho.foto" title="heise Foto – Das Online-Magazin rund ums digitale Bild" href="/foto/from/navi_oben_foto">heise Foto</a></li>
+                                    <li class="heisetopnavi_net"><a name="dachzeile.ho.netze" title="heise Netze – Alles über Netzwerk-Technik" href="/netze/from/navi_oben_net">heise Netze</a></li>
+                                    <li class="heisetopnavi_op"><a name="dachzeile.ho.open" title="heise open – Open Source im Unternehmen" href="/open/from/navi_oben_op">heise Open Source</a></li>
+                                    <li class="heisetopnavi_sec"><a name="dachzeile.ho.security" title="heise Security – News, Dienste und Foren zum Thema Computer-Sicherheit" href="/security/from/navi_oben_sec">heise Security</a></li>
+                                    <li class="heisetopnavi_video"><a name="dachzeile.ho.video" title="heise Video – Clips zu Nachrichten und Artikeln" href="/video/from/navi_oben_video">heise Video</a></li>
+                                    <li class="heisetopnavi_tech"><a name="dachzeile.ho.techstage" title="TechStage – News, Tests &amp; Praxis zu Smartphones und Tablets mit Android, iOS, Windows Phone &amp; Co." href="/redirect-to/techstage/from/navi_oben_techstage">TechStage</a></li>
+                                </ul>
+                            </li>
+                            <li class="heisetopnavi_third">
+                                <ul>
+                                    <li class="heisetopnavi_swv"><a name="dachzeile.ho.download" title="Software-Verzeichnis – Software zum Download fuer Windows, Mac, Linux, iPhone, Symbian, Android" href="/download/from/navi_oben_swv">Download</a></li>
+                                    <li class="heisetopnavi_pvg"><a name="dachzeile.ho.preisvergleich" title="Preisvergleich – Günstige Preise, Meinungen und Bewertungen zu vielen Produkten" href="/preisvergleich/from/navi_oben_pvg">Preisvergleich</a></li>
+                                    <li class="heisetopnavi_job"><a name="dachzeile.ho.jobs" title="heise jobs – Jobbörse für qualifizierte Fach- und Führungskräfte aus der IT-Branche" href="/jobs/from/navi_oben_job">Stellenmarkt</a></li>
+                                    <li class="heisetopnavi_eve"><a name="dachzeile.ho.events" title="heise Events – Konferenzen, Seminare, Workshops" href="/events/from/navi_oben_eve">Veranstaltungen</a></li>
+                                    <li class="heisetopnavi_itm"><a name="dachzeile.ho.itmarkt" title="IT-Markt – Branchenverzeichnis der IT-Fachhändler" href="/itmarkt/from/navi_oben_itm">IT-Markt</a></li>
+                                    <li class="heisetopnavi_whi"><a name="dachzeile.ho.whitepapers" title="heise Whitepapers – Kostenloser Download aktueller Praxisbeispiele, Firmeninfos, Case Studies und Webcasts zu neuen Produkten, Strategien und Lösungen namhafter IT-Hersteller" href="/whitepapers/from/navi_oben_whi">Whitepapers</a></li>
+                                    <li class="heisetopnavi_webcasts"><a name="dachzeile.ho.webcasts" title="heise Webcasts – Kompakte Firmeninformationen zu komplexen IT-Sachverhalten" href="/redirect-to/webcasts/from/navi_oben_webcasts">Webcasts</a></li>
+                                    <li class="heisetopnavi_tarifr"><a name="dachzeile.ho.tarifrechner" title="Tarifrechner – DSL, Mobiles Internet, Handy, Telefon, Strom, Gas" href="/redirect-to/tarifrechner/from/navi_oben_tarifr">Tarifrechner</a></li>
+                                </ul>
+                            </li>
+                            <li class="heisetopnavi_fourth">
+                                <ul>
+                                    <li class="heisetopnavi_shop"><a name="dachzeile.ho.shop" title="heise shop – IT Fachzeitschriften, Bücher, CD/DVD/Blu-ray" href="/redirect-to/shop/from/navi_oben_shop">heise shop</a></li>
+                                    <li class="heisetopnavi_kio"><a name="dachzeile.ho.artikel-archiv" title="Artikel-Archiv – c't, iX, Technology Review, Digitale Fotografie, Mac &amp; i, Sonderhefte" href="/artikel-archiv/from/navi_oben_kio">Artikel-Archiv</a></li>
+                                    <li class="heisetopnavi_abo"><a name="dachzeile.ho.abo" title="Abo – c't, iX, Technology Review, Digitale Fotografie, Mac &amp; i" href="/redirect-to/abo/from/navi_oben_abo">Zeitschriften-Abo</a></li>
+                                    <li class="heisetopnavi_hmg"><a name="dachzeile.ho.hmg" title="Arbeiten bei heise – Aktuelle Stellenangebote der Heise Gruppe" href="/redirect-to/hmg/from/navi_oben_hmg">Arbeiten bei heise</a></li>
+                                </ul>
+                            </li>
+                        </ul>
+                    </div>
+                </div>
+            </nav>
+
+            
+
+            
+                
+                    
+                    
+            
+
+            <form role="search" id="heisetopnavi_search" action="/mac-and-i/suche/" method="get" accept-charset="utf-8">
+                <fieldset>
+                    <input type="text" placeholder="in Mac &amp; i suchen" name="q" value="" class="search_text" /><input type="image" name="search_submit" class="search_submit" alt="Los" src="//www.heise.de/icons/ho/heise_online_lupe.gif" />
+                    <input type="hidden" name="rm" value="search" class="search_hidden" />
+                </fieldset>
+            </form>
+        </header>
+    </div>
+    <div class="clear"></div>
+</div>
+
+<script type="text/javascript">
+if (typeof jQuery !== "undefined") {
+    jQuery(document).ready(function($) {
+        // Support Touch-Devices
+        if ('ontouchstart' in document) {
+            $('.heisetopnavi_relaunch').removeClass('no-touch');
+        }
+        $('.heisetopnavi_button').bind('touchstart', function(e) {
+            e.preventDefault();
+            if ($(e.target).is('a.heisetopnavi_button')) {
+                $(this).toggleClass('active');
+                $('.heisetopnavi header &gt; nav').toggleClass('hover');
+            }
+        });
+
+        $('.heisetopnavi_relaunch .heisetopnavi_button').click(function(e) {
+            e.preventDefault();
+        });
+
+        // Support Keyboard
+        $('.heisetopnavi_relaunch .heisetopnavi_button').keyup(function(e) {
+            if (e.which == 13 &amp;&amp; $(e.target).is('a.heisetopnavi_button')) {
+                $('.heisetopnavi_relaunch .heisetopnavi_button').toggleClass('active');
+                $('.heisetopnavi_relaunch header &gt; nav').toggleClass('hover');
+            }
+        });
+
+        /*
+         * Open/Close Menu with JavaScript for more Usability
+         */
+
+        $('.heisetopnavi_relaunch').removeClass('no-touch');
+
+        // Logo + Icon
+        $('.heisetopnavi_relaunch .heisetopnavi_hover_legacy').mouseenter(function() {
+            var timeout_id = window.setTimeout(function() {
+                $('.heisetopnavi_relaunch .heisetopnavi_button').addClass('active');
+                $('.heisetopnavi_relaunch nav').addClass('hover');
+                $('body').addClass('refreshDOM');
+                $('body').removeClass('refreshDOM');
+            }, 300);
+            $(this).data('timeout_id', timeout_id);
+        });
+        $('.heisetopnavi_relaunch .heisetopnavi_hover_legacy').mouseleave(function() {
+            var timeout_id = $(this).data('timeout_id');
+            window.clearTimeout(timeout_id);
+            timeout_id = window.setTimeout(function() {
+                $('.heisetopnavi_relaunch .heisetopnavi_button').removeClass('active');
+                $('.heisetopnavi_relaunch nav').removeClass('hover');
+                $('body').addClass('refreshDOM');
+                $('body').removeClass('refreshDOM');
+            }, 0);
+            $(this).data('timeout_id', timeout_id);
+        });
+    });
+
+    if (typeof heiseLogin != "undefined") {
+        heiseLogin.login_fill_navigation({});
+    }
+}
+</script>
+
+    
+
+
+<div id="container">
+    <div id="container_content">
+        <div id="logo_bereich">
+            <a title="Mac &amp; i" href="/mac-and-i/"><img id="logo" alt="heise Mac &amp; i" src="//3.f.ix.de/mac-and-i/icons/heise_mac-and-i_logo.gif" /></a>
+        </div>
+        <ul class="navigation_news">
+            <li><a href="/mac-and-i/news/7_tage_news/">7-Tage-News</a></li>
+            <li><a href="/mac-and-i/news/archiv/">News-Archiv</a></li>
+            <li><a href="http://live.mac-and-i.de/#AllEvents">Liveticker</a></li>
+            <li><a href="/mac-and-i/news/foren/">News-Foren</a></li>
+            <li><a href="/mac-and-i/artikel/foren/">Artikel-Foren</a></li>
+            <li><a href="/newsletter/manage/mac-and-i">Newsletter</a></li>
+            <li><a href="/mac-and-i/kontakt/">Kontakt</a></li>
+            <li><a title="News RSS-Feed (Atom)" href="/mac-and-i/news-atom.xml">RSS</a></li>
+        </ul>
+        <ul id="navi_main">
+                    <li id="first" class="aktiv"><a href="/mac-and-i/news/">News</a></li>
+                    <li><a href="/mac-and-i/artikel/">Artikel</a></li>
+                    <li><a href="/mac-and-i/forum/">Forum</a></li>
+                    <li><a href="/mac-and-i/produkte/">Produkte</a></li>
+                    <li><a href="/mac-and-i/heft/">Heft</a></li>
+                    <li><a href="/mac-and-i/heftarchiv/">Archiv</a></li>
+                    <li id="last"><a href="http://shop.heise.de/mac-and-i-abo/">Abo</a></li>
+        </ul>
+        <ul class="meta_navi">
+            <li><a rel="external" title="Mac &amp; i Twitter" href="http://twitter.com/#%21/mac_and_i"><img alt="Mac &amp; i Twitter" src="//1.f.ix.de/icons/ho/navi_icon_twitter_big.png" /></a></li>
+            <li><a rel="external" title="Mac &amp; i Facebook" href="http://www.facebook.com/ct.Mac.and.i"><img alt="Mac &amp; i Facebook" src="//3.f.ix.de/icons/ho/navi_icon_facebook_big.png" /></a></li>
+            <li><a target="_blank" title="Mac &amp; i Google+" rel="publisher" href="https://plus.google.com/+Mac-and-iDe"><img alt="Mac &amp; i Google+" src="//3.f.ix.de/icons/ho/navi_icon_google_big.png" /></a></li>
+            <li><a target="_blank" title="Mac &amp; i YouTube" rel="publisher" href="https://www.youtube.com/macandiVideo"><img alt="Mac &amp; i YouTube" src="//3.f.ix.de/icons/ho/navi_icon_youtube_black_big.png" /></a></li>
+            <li>
+                <dl class="apps">
+                    <dt><img alt="Mac &amp; i Apps" src="//2.f.ix.de/mac-and-i/icons/navi_icon_apps.png" /></dt>
+                    <dd><a rel="external" title="Mac &amp; i für das iPhone" href="http://itunes.apple.com/de/app/mac-i/id424199222?mt=8"><img alt="Mac &amp; i für das iPhone" src="//2.f.ix.de/icons/ho/navi_icon_mobi_mac.png" /></a></dd>
+                    <dd><a rel="external" title="Mac &amp; i für das iPad" href="http://itunes.apple.com/de/app/mac-i/id424199222?mt=8"><img alt="Mac &amp; i für das iPad" src="//2.f.ix.de/icons/ho/navi_icon_ipad.png" /></a></dd>
+                </dl>
+            </li>
+        </ul>
+            
+	<div id="breadcrumb">
+	
+		<a title="Mac &amp; i" href="/mac-and-i/">Mac &amp; i</a>
+		<span class="bread_gt">&gt;</span>
+	
+		<a title="News" href="/mac-and-i/news/">News</a>
+		<span class="bread_gt">&gt;</span>
+	
+		<a title="2015" href="/mac-and-i/news/archiv/?jahr=2015">2015</a>
+		<span class="bread_gt">&gt;</span>
+	
+		<a title="KW 15" href="/mac-and-i/news/archiv/?jahr=2015;woche=15">KW 15</a>
+		<span class="bread_gt">&gt;</span>
+	
+		<span class="titel">1Password für Mac generiert Einmal-Passwörter</span>
+		
+	
+	</div>
+
+
+        <div id="mitte">
+                <div id="mitte_links">
+            
+
+<div id="mitte_news">
+        <p class="news_navi">                <a href="/mac-and-i/meldung/HBO-Zugriff-per-Apple-Geraet-verfuegbar-2596982.html"><span class="rsaquo">«</span> Vorige</a>            |                <a href="/mac-and-i/meldung/GPS-Streckenerfasser-Speed-PRO-aktuell-kostenlos-2597093.html">Nächste <span class="rsaquo">»</span></a>        </p>    <article>
+        <p class="news_datum">08.04.2015 12:46</p>
+        <!--googleon: all-->
+        <!-- RSPEAK_START -->
+        <h1> 1Password für Mac generiert Einmal-Passwörter</h1>
+        <div class="meldung_wrapper">
+            
+    <!-- RSPEAK_STOP -->
+    <figure class="aufmacherbild">
+            <img src="//3.f.ix.de/scale/geometry/600/q75/imgs/18/1/4/6/2/3/5/1/Barcode-Scanner-With-Border-fc08c913da5cea5d.jpeg" />
+        
+            <figcaption>
+                
+                <p class="caption">1Password scannt auch QR-Codes.</p>
+                
+                
+                <p class="source">(Bild: Hersteller)</p>
+                
+            </figcaption>
+        
+    </figure>
+    <!-- RSPEAK_START -->
+
+
+            <p class="meldung_anrisstext"><strong>Das in der iOS-Version bereits enthaltene TOTP-Feature ist nun auch für OS X 10.10 verfügbar. Zudem gibt es neue Zusatzfelder in der Datenbank und weitere Verbesserungen.</strong></p>
+            <p><a rel="external" target="_blank" href="https://itunes.apple.com/de/app/1password-password-manager/id443987910">AgileBits hat Version 5.3 seines bekannten Passwortmanagers 1Password für OS X freigegeben.</a> Mit dem Update wird eine praktische Funktion nachgereicht, die <a href="/mac-and-i/meldung/Passwortmanager-1Password-mit-groesseren-Updates-fuer-OS-X-und-iOS-2529204.html">die iOS-Version der Anwendung bereits seit längerem beherrscht</a>: Das direkte Erstellen von Einmal-Passwörtern. Unterstützt wird dabei der <a rel="external" target="_blank" href="https://blog.agilebits.com/2015/01/26/totp-for-1password-users/">TOTP-Standard</a> (Time-Based One-Time Passwords), den unter anderem Firmen wie Evernote, Dropbox oder Google einsetzen, um ihre Zugänge besser abzusichern. Neben Account und regulärem Passwort wird dabei dann ein Zusatzcode verlangt, der nur kurze Zeit gilt.</p>
+<p>Zur TOTP-Nutzung muss zunächst ein Startwert an 1Password übergeben werden. Das geht unter anderem per QR-Code, den die App über ein neues Scanfenster selbst einlesen kann – etwa aus dem Webbrowser. Eine Einführung in die Technik gibt <a rel="external" target="_blank" href="http://1pw.ca/TOTPvideoMac">ein kurzes Video</a>. Die TOTP-Unterstützung in 1Password erlaubt es, auf ein zusätzliches Gerät (z.B. ein iPhone) neben dem Mac zu verzichten, das den Code liefert – was allerdings auch die Sicherheit verringert, weil es keinen "echten" zweiten Faktor mehr gibt.</p>
+<p>Update 5.3 des Passwortmanagers liefert auch noch weitere Verbesserungen. So gibt es die Möglichkeit, FaceTime-Audio- oder Skype-Anrufe aus 1Password zu starten, die Zahl der Zusatzfelder in der Datenbank wurde erweitert und der Umgang mit unterschiedlichen Zeitzonen klappt besser. Die Engine zur Passworteingabe im Browser soll beschleunigt worden sein.</p>
+<p>1Password kostet aktuell knapp 50 Euro im Mac App Store und setzt in seiner aktuellen Version mindestens OS X 10.10 voraus.<!-- AUTHOR-DATA-MARKER-BEGIN -->
+
+
+<!-- RSPEAK_STOP -->
+<span class="ISI_IGNORE">(<a title="Ben Schwan" href="mailto:bsc@heise.de">bsc</a>)</span>
+<br class="clear" />
+<!-- RSPEAK_START -->
+<!-- AUTHOR-DATA-MARKER-END -->
+</p>
+
+        </div>
+
+    <!-- RSPEAK_STOP -->
+    <!--googleoff: all-->
+
+        <footer class="article-footer">            <p class="news_navi">                    <a href="/mac-and-i/meldung/HBO-Zugriff-per-Apple-Geraet-verfuegbar-2596982.html"><span class="rsaquo">«</span> Vorige</a>                |                    <a href="/mac-and-i/meldung/GPS-Streckenerfasser-Speed-PRO-aktuell-kostenlos-2597093.html">Nächste <span class="rsaquo">»</span></a>            </p>
+            <div class="link_forum_beitrag news">
+                
+                    
+                
+                <p>
+                    <a href="http://www.heise.de/forum/Mac-i/News-Kommentare/1Password-fuer-Mac-generiert-Einmal-Passwoerter/forum-65645/comment/">
+                        <b>
+                            
+                                Kommentare lesen
+                            
+                             
+                                (1 Beitrag)
+                            
+                        </b>
+                    </a>
+                </p>
+            </div>    
+        
+                    <p class="themen_foren">
+                
+                    Forum zum Thema: 
+                
+                <a title="Zum Themenforum Software für Mac OS X" href="http://www.heise.de/forum/Mac-i/Themen-Hilfe/Software-fuer-Mac-OS-X/forum-12/comment/">
+                    
+                        Software für Mac OS X
+                    
+                </a>
+            </p>
+        <div data-services="[&quot;facebook&quot;,&quot;twitter&quot;,&quot;googleplus&quot;,&quot;mail&quot;,&quot;info&quot;]" class="shariff clear" data-backend-url="/shariff-backend/"><ul class="theme-color orientation-horizontal"><li class="shariff-button facebook"><a href="https://www.facebook.com/sharer/sharer.php?u=http%3A%2F%2Fwww.heise.de%2Fmac-and-i%2Fmeldung%2F1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html" rel="popup" title="Bei Facebook teilen"><span class="fa fa-facebook"></span><span class="share_text">teilen</span><span class="share_count">17</span></a></li><li class="shariff-button twitter"><a href="https://twitter.com/intent/tweet?text=1Password%20f%C3%BCr%20Mac%20generiert%20Einmal-Passw%C3%B6rter%20-%20Mac%20%26%20i&amp;url=http%3A%2F%2Fwww.heise.de%2Fmac-and-i%2Fmeldung%2F1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html" rel="popup" title="Bei Twitter teilen"><span class="fa fa-twitter"></span><span class="share_text">tweet</span><span class="share_count">0</span></a></li><li class="shariff-button googleplus"><a href="https://plus.google.com/share?url=http%3A%2F%2Fwww.heise.de%2Fmac-and-i%2Fmeldung%2F1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html" rel="popup" title="Bei Google+ teilen"><span class="fa fa-google-plus"></span><span class="share_text">+1</span><span class="share_count">2</span></a></li><li class="shariff-button mail"><a href="http://www.heise.de/mac-and-i/meldung/1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html?view=mail" target="_blank" title="Per E-Mail versenden"><span class="fa fa-envelope"></span><span class="share_text">mail</span></a></li><li class="shariff-button info"><a href="http://ct.de/-2467514" target="_blank" title="weitere Informationen"><span class="fa fa-info"></span><span class="share_text">Info</span></a></li></ul></div>
+            <p class="permalink">Permalink: <a href="http://heise.de/-2596987">http://heise.de/-2596987</a></p>
+
+        <p class="printversion">
+            <a rel="nofollow" href="/mac-and-i/meldung/1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html?view=print">Version zum Drucken</a>
+        </p>
+
+        
+
+
+
+<div class="related_items">
+	<h4>Auch auf heise online:</h4>
+	<ul>
+	
+	    <li><a title="1Password verbessert iOS-8-Erweiterung" href="http://www.heise.de/mac-and-i/meldung/1Password-verbessert-iOS-8-Erweiterung-2587926.html/from/related">1Password verbessert iOS-8-Erweiterung</a></li>
+	
+	    <li><a title="Parallels-Mac-Bundle mit 1Password und Waltr" href="http://www.heise.de/mac-and-i/meldung/Parallels-Mac-Bundle-mit-1Password-und-Waltr-2572511.html/from/related">Parallels-Mac-Bundle mit 1Password und Waltr</a></li>
+	
+	    <li><a title="1Password 5 für Yosemite angepasst" href="http://www.heise.de/mac-and-i/meldung/1Password-5-fuer-Yosemite-angepasst-2428275.html/from/related">1Password 5 für Yosemite angepasst</a></li>
+	
+	    <li><a title="1Password für iOS bringt iTunes-Synchronisation zurück – teilweise" href="http://www.heise.de/mac-and-i/meldung/1Password-fuer-iOS-bringt-iTunes-Synchronisation-zurueck-teilweise-2213740.html/from/related">1Password für iOS bringt iTunes-Synchronisation zurück – teilweise</a></li>
+	
+	    <li><a title="1Password mit Heartbleed-Warnfunktion" href="http://www.heise.de/mac-and-i/meldung/1Password-mit-Heartbleed-Warnfunktion-2183120.html/from/related">1Password mit Heartbleed-Warnfunktion</a></li>
+	
+	    <li><a title="1Password mit Updates für OS X und iOS" href="http://www.heise.de/mac-and-i/meldung/1Password-mit-Updates-fuer-OS-X-und-iOS-2143961.html/from/related">1Password mit Updates für OS X und iOS</a></li>
+	
+	</ul>
+</div>
+
+
+        
+<p class="themenseiten">
+    <span class="themen_label">Mehr zum Thema</span>
+    
+        <a title="Themenseite Mac OS X" href="/thema/Mac-OS-X">Mac OS X</a>
+    
+        <a title="Themenseite Passwort" href="/thema/Passwort">Passwort</a>
+    
+</p>
+
+
+        
+
+
+        </footer>
+    </article>
+
+</div>
+
+            <div class="adbottom"><!--googleoff: index-->
+<!-- RSPEAK_STOP -->
+<a class="hinweis_anzeige" target="_blank" href="http://www.heise.de/mediadaten/online/">Anzeige</a><br /><script type="text/javascript">
+&lt;!--//--&gt;&lt;![CDATA[//&gt;&lt;!--
+var dfp_ord; if (!dfp_ord) { dfp_ord = Math.floor(Math.random()*1000000000)+1000000000; }
+var yp_res; if (typeof yl !== "undefined" &amp;&amp; yl.YpResult !== "undefined") { yp_res = yl.YpResult.get(''); }
+document.write('&lt;script src="http://ad-emea.doubleclick.net/N6514/adj/mac/mac-inhalt;sz=500x500;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=1;_YL_;ord=_ORD_?" type="text/javascript"&gt;&lt;\/script&gt;'.replace('_ORD_', dfp_ord).replace(';_YL_', typeof yp_res !== "undefined" &amp;&amp; yp_res.pricerange ? ';pricerange=' + yp_res.pricerange : ''));
+if (typeof jQuery != 'undefined') {
+    jQuery( function() {
+        jQuery('.adbottom div[id^=google_ads_div_]').parent().prepend('&lt;span style="font-family: Arial, Helvetica, sans-serif; color: Black; font-size:7pt; font-weight: normal; text-align: left;"&gt;Anzeige&lt;/span&gt;&lt;br/&gt;');
+    });
+}
+
+//--&gt;&lt;!]]&gt;
+</script><script type="text/javascript" src="http://ad-emea.doubleclick.net/N6514/adj/mac/mac-inhalt;sz=500x500;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=1;ord=1875073260?"></script>
+<noscript>&lt;div&gt;&lt;a href="http://ad-emea.doubleclick.net/N6514/jump/mac/mac-inhalt;sz=500x500;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=1;_YL_;ord=7840756744?" target="_blank"&gt;&lt;img alt="" src="http://ad-emea.doubleclick.net/N6514/ad/mac/mac-inhalt;sz=500x500;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=1;_YL_;ord=7840756744?" /&gt;&lt;/a&gt;&lt;/div&gt;</noscript>
+
+<!-- RSPEAK_START -->
+<!--googleon: index--><!--googleoff: all--></div>
+                </div>
+                <div id="mitte_rechts">
+                        <!--googleoff: index-->
+<!-- RSPEAK_STOP -->
+<div class="bcadv ISI_IGNORE bcadv_oben"><a class="hinweis_anzeige" target="_blank" href="http://www.heise.de/mediadaten/online/">Anzeige</a><br /><script type="text/javascript">
+&lt;!--//--&gt;&lt;![CDATA[//&gt;&lt;!--
+var dfp_ord; if (!dfp_ord) { dfp_ord = Math.floor(Math.random()*1000000000)+1000000000; }
+var yp_res; if (typeof yl !== "undefined" &amp;&amp; yl.YpResult !== "undefined") { yp_res = yl.YpResult.get('66442'); }
+document.write('&lt;script src="http://ad-emea.doubleclick.net/N6514/adj/mac/mac-inhalt;sz=300x250,336x280;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=2;_YL_;ord=_ORD_?" type="text/javascript"&gt;&lt;\/script&gt;'.replace('_ORD_', dfp_ord).replace(';_YL_', typeof yp_res !== "undefined" &amp;&amp; yp_res.pricerange ? ';pricerange=' + yp_res.pricerange : ''));
+//--&gt;&lt;!]]&gt;
+</script><script type="text/javascript" src="http://ad-emea.doubleclick.net/N6514/adj/mac/mac-inhalt;sz=300x250,336x280;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=2;ord=1875073260?"></script>
+<noscript>&lt;div&gt;&lt;a href="http://ad-emea.doubleclick.net/N6514/jump/mac/mac-inhalt;sz=300x250,336x280;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=2;_YL_;ord=7840756744?" target="_blank"&gt;&lt;img alt="" src="http://ad-emea.doubleclick.net/N6514/ad/mac/mac-inhalt;sz=300x250,336x280;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=2;_YL_;ord=7840756744?" /&gt;&lt;/a&gt;&lt;/div&gt;</noscript>
+</div>
+<!-- RSPEAK_START -->
+<!--googleon: index--><!-- RSPEAK_STOP -->
+                        <!--googleoff: all-->
+                    
+    
+    <div class="magazinteaser">
+        <a href="/mac-and-i/heft/">
+            <img width="127" height="180" alt="" src="//1.f.ix.de/mac-and-i/imgs/65/1/3/7/0/9/3/7/mac-2015-02-92ccee0374aa9def.jpeg" />
+            <h3>Heft 2/2015</h3>
+            <p>Billig kontra Apple • Zaubern mit Skripten • HomeKit • Datenrettung • iPad automatisieren • Standort freigeben • Verräterische Daten • Kameras fernsteuern •  MacBooks • Bluetooth-Kopfhörer • iPhone-6-Hüllen • Ableton-Live-Apps • Kommt das Apple-Auto?</p>
+        </a>
+        <ul>
+            <li class="first"><a href="/mac-and-i/heft/">Aktuelles Heft</a></li>
+            <li><a target="_blank" href="http://shop.heise.de/mac-and-i-abo/">Abonnieren</a></li>
+            <li><a href="/mac-and-i/heftarchiv/">Heftarchiv</a></li>
+        </ul>
+    </div>
+
+
+
+                    
+    
+    <form id="webcode" method="get" action="/bin/softlink">
+        <div class="ident">Webcode:</div> <div class="url">www.mac-and-i.de/</div>
+        <fieldset>
+           <input type="text" id="webcode_input" class="textfield" required="required" placeholder="mi1101003" size="9" name="ctid" />
+           <input type="submit" class="submit" value="enter" />
+           <input type="hidden" name="objekt" value="mi" />
+        </fieldset>
+    </form>
+
+
+
+                    
+
+<div class="teaser_frei">
+        <div class="anriss_mit_bild_links">            <h3><a href="http://shop.heise.de/ct-mac-special-2014?pid=5001002207">c't special Mac</a></h3>            <a href="http://shop.heise.de/ct-mac-special-2014?pid=5001002207"><img alt="" src="//1.f.ix.de/mac-and-i/imgs/65/1/3/8/3/2/3/7/Titel-97485b58f23ceb92.png" /></a>            <p><a href="http://shop.heise.de/ct-mac-special-2014?pid=5001002207"><a href="http://shop.heise.de/ct-mac-special-2014?pid=5001002207">
+                Mac: Der bessere PC • Kaufberatung: Alle Macs im Vergleich • Umsteigen von Windows • 150 Seiten Praxistipps zu OS X und Anwendungen • Mac schneller machen • Erste Hilfe, wenns klemmt • Terminal <a href="http://shop.heise.de/ct-mac-special-2014?pid=5001002207"><a href="http://shop.heise.de/ct-mac-special-2014?pid=5001002207"><span class="mehr_schnipsel">Mehr…</span></a></a>
+            </a></a></p>    </div>
+</div>
+
+
+
+                    
+    
+    <script type="text/javascript">
+        $(document).ready( function() {
+            $('div#meistgelesen_tabbox').tabs();
+        });
+    </script>
+    <div id="meistgelesen_tabbox" class="ui-tabs ui-widget ui-widget-content ui-corner-all">
+        <ul id="meistgelesen_tabs" class="ui-tabs-nav ui-helper-reset ui-helper-clearfix ui-widget-header ui-corner-all">
+            <li class="ui-state-default ui-corner-top ui-tabs-selected ui-state-active"><a href="#meistgelesen_tabs-1">Meistgelesen</a></li>
+            <li class="ui-state-default ui-corner-top"><a href="#meistgelesen_tabs-2">Meistkommentiert</a></li>
+        </ul>
+        <div id="meistgelesen_tabs-1" class="ui-tabs-panel ui-widget-content ui-corner-bottom">
+            <ul>
+                <li><a href="/mac-and-i/artikel/Pro-Contra-Hat-Apple-den-Bogen-ueberspannt-2580439.html">Pro &amp; Contra: Hat Apple den Bogen überspannt?</a></li>
+                <li><a href="/meldung/Fotos-und-mehr-Apple-stellt-OS-X-10-10-3-zum-Download-bereit-2597165.html">"Fotos" und mehr: Apple stellt OS X 10.10.3 zum Download bereit</a></li>
+                <li><a href="/meldung/iOS-8-3-verfuegbar-2597556.html">iOS 8.3 verfügbar</a></li>
+                <li><a href="/meldung/Apple-Watch-Erste-Testberichte-fallen-ordentlich-aus-2597244.html">Apple Watch: Erste Testberichte fallen ordentlich aus</a></li>
+                <li><a href="/meldung/LG-kuendigt-8K-iMac-an-und-wieder-ab-2596469.html">LG kündigt 8K-iMac an – und wieder ab</a></li>
+            </ul>
+        </div>
+        <div id="meistgelesen_tabs-2" class="ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide">
+            <ul>
+                
+  
+    <li>
+      <a href="/mac-and-i/meldung/Quadratischer-Monitor-fuer-das-Buero-Eizos-FlexScan-EV2730Q-2460773.html">
+	Quadratischer Monitor für das Büro: Eizos FlexScan EV2730Q
+      </a>
+    </li>
+  
+    <li>
+      <a href="/mac-and-i/meldung/Umfangreiches-Firmware-Update-fuer-Amazon-Fire-TV-und-Fire-TV-Stick-2584775.html">
+	Umfangreiches Firmware-Update für Amazon Fire TV und Fire TV Stick
+      </a>
+    </li>
+  
+    <li>
+      <a href="/mac-and-i/meldung/Jay-Z-verbuendet-sich-mit-anderen-Musikern-fuer-Streaming-Dienst-Tidal-2591043.html">
+	Jay Z verbündet sich mit anderen Musikern für Streaming-Dienst Tidal
+      </a>
+    </li>
+  
+    <li>
+      <a href="/mac-and-i/meldung/Apple-Browser-Update-fuer-drei-Safari-Versionen-behebt-Sicherheitsluecken-2578006.html">
+	Apple-Browser: Update für drei Safari-Versionen behebt Sicherheitslücken
+      </a>
+    </li>
+  
+    <li>
+      <a href="/mac-and-i/meldung/Safari-Alte-Sicherheitsluecke-speichert-URLs-auch-im-Private-Browsing-Modus-2575426.html">
+	Safari: Alte Sicherheitslücke speichert URLs auch im Private-Browsing-Modus
+      </a>
+    </li>
+  
+
+
+            </ul>
+        </div>
+    </div>
+
+
+                    
+        <p class="us_ad">
+            <span><a target="_blank" href="/mediadaten/heise-online/">Anzeige</a></span><br />
+            <a target="_blank" href="http://pubads.g.doubleclick.net/gampad/clk?id=31532750&amp;iu=/6514/www.heise.de/clicktracking/usAd"><img width="336" height="200" alt="" src="//2.f.ix.de/mac-and-i/imgs/65/1/4/5/5/6/0/2/alto_heise_hp_tour_mpu__20150324-e4f5620ffc9f3323.jpg" /></a>
+            <script type="text/javascript">
+            &lt;!--//--&gt;&lt;![CDATA[//&gt;&lt;!--
+                (function (){
+                    var AVW = '&lt;' + 'img id="avw_pixel_intern" src="/avw-bin/ivw/CP/barfoo/ho/2585134/0.gif?d=_ORD_" width="1" height="1" alt=""&gt;'.replace('_ORD_', Math.floor(Math.random()*1000000000)+1000000000);
+                    document.write(AVW);
+                })();
+            //--&gt;&lt;!]]&gt;
+            </script><img width="1" height="1" alt="" src="/avw-bin/ivw/CP/barfoo/ho/2585134/0.gif?d=1029284408" id="avw_pixel_intern" />
+            <noscript>&lt;img id="avw_pixel_intern" src="/avw-bin/ivw/CP/barfoo/ho/2585134/0.gif" width="1" height="1" alt=""&gt;</noscript>
+        </p>
+
+                    
+    <div class="teaser_adliste">
+        <p><a target="_blank" href="/mediadaten/heise-online/">Anzeige</a></p>
+        <ul class="microsites">
+            <li><a target="_blank" rel="nofollow" href="http://pubads.g.doubleclick.net/gampad/clk?id=30995390&amp;iu=/6514/www.heise.de/clicktracking/textlink">Datenberge in Informationen verwandeln</a></li>
+            <li><a target="_blank" rel="nofollow" href="http://pubads.g.doubleclick.net/gampad/clk?id=31000430&amp;iu=/6514/www.heise.de/clicktracking/textlink">IT Angriffe proaktiv erkennen und abwehren</a></li>
+            <li><a target="_blank" rel="nofollow" href="http://pubads.g.doubleclick.net/gampad/clk?id=30944270&amp;iu=/6514/www.heise.de/clicktracking/textlink">Webcast Next Gen Enduser Protection</a></li>
+            <li><a target="_blank" rel="nofollow" href="http://pubads.g.doubleclick.net/gampad/clk?id=29918270&amp;iu=/6514/www.heise.de/clicktracking/textlink">Nerd-Shirts für Informatiker</a></li>
+            <li><a target="_blank" rel="nofollow" href="http://pubads.g.doubleclick.net/gampad/clk?id=31718150&amp;iu=/6514/www.heise.de/clicktracking/textlink">Der 3. IT-Jobtag bei Heise am 16. April</a></li>
+            <li><a target="_blank" rel="nofollow" href="http://pubads.g.doubleclick.net/gampad/clk?id=31588190&amp;iu=/6514/www.heise.de/clicktracking/textlink">Risiko Logfiles: Protokollieren Sie rechtssicher!</a></li>
+            <li><a target="_blank" rel="nofollow" href="http://pubads.g.doubleclick.net/gampad/clk?id=31533230&amp;iu=/6514/www.heise.de/clicktracking/textlink">Skype for Business macht Unternehmen produktiver</a></li>
+            <li><a target="_blank" rel="nofollow" href="http://pubads.g.doubleclick.net/gampad/clk?id=31069190&amp;iu=/6514/www.heise.de/clicktracking/textlink">kostenfreies IT-Event im Fußballstadion</a></li>
+            <li><a target="_blank" rel="nofollow" href="http://pubads.g.doubleclick.net/gampad/clk?id=31048910&amp;iu=/6514/www.heise.de/clicktracking/textlink">Ratgeber: Desktop-Virtualisierung kurz und knapp</a></li>
+        </ul>
+    </div>
+
+                    <!--googleoff: all-->
+                    
+    
+    <div class="newsteaser">            <h4><a href="/mac-and-i/news/">News</a></h4>                            <div class="anriss_mit_bild_links">
+                    <h3><a title="Apple stopft Sicherheitslücken in iOS und mehreren OS-X-Versionen" href="/mac-and-i/meldung/Apple-stopft-Sicherheitsluecken-in-iOS-und-mehreren-OS-X-Versionen-2597723.html">Apple stopft Sicherheitslücken in iOS und mehreren OS-X-Versionen</a></h3>
+                    
+                        <a href="/mac-and-i/meldung/Apple-stopft-Sicherheitsluecken-in-iOS-und-mehreren-OS-X-Versionen-2597723.html"><img alt="OS X Yosemite" src="//1.f.ix.de/scale/geometry/160x90/q75/imgs/18/1/4/6/2/8/0/3/urn-newsml-dpa-com-20090101-141017-99-01179_large_4_3-c0577d3c5419b335.jpeg" /></a>
+                    
+                        <p><a href="/mac-and-i/meldung/Apple-stopft-Sicherheitsluecken-in-iOS-und-mehreren-OS-X-Versionen-2597723.html">Apple hat mit den Betriebssystem-Updates vom Mittwoch ein Bündel an Security-Fixes mitgeliefert. Sicherheitsaktualisierungen gibt es auch für OS X 10.8, 10.9 und Apple TV. <a href="/mac-and-i/meldung/Apple-stopft-Sicherheitsluecken-in-iOS-und-mehreren-OS-X-Versionen-2597723.html"><span class="mehr_schnipsel">Mehr…</span></a></a></p>                </div>            
+                                    <div class="anriss_mit_bild_links">
+                    <h3><a title="Ex-HP-Chefin kritisiert Tim Cooks Kritik an Gesetz zur Religionsfreiheit" href="/mac-and-i/meldung/Ex-HP-Chefin-kritisiert-Tim-Cooks-Kritik-an-Gesetz-zur-Religionsfreiheit-2597091.html">Ex-HP-Chefin kritisiert Tim Cooks Kritik an Gesetz zur Religionsfreiheit</a></h3>
+                    
+                        <a href="/mac-and-i/meldung/Ex-HP-Chefin-kritisiert-Tim-Cooks-Kritik-an-Gesetz-zur-Religionsfreiheit-2597091.html"><img alt="Ehemalige HP-Chefin kritisiert Tim Cooks Kritik an Gesetz zur Religionsfreiheit" src="//3.f.ix.de/scale/geometry/160x90/q75/imgs/18/1/4/6/2/4/2/4/1024px-Carly_Fiorina_by_Gage_Skidmore-e15691fffb93dd28.jpeg" /></a>
+                    
+                        <p><a href="/mac-and-i/meldung/Ex-HP-Chefin-kritisiert-Tim-Cooks-Kritik-an-Gesetz-zur-Religionsfreiheit-2597091.html">Der Apple-Chef hatte sich gegen die Gesetzgebung eines US-Bundesstaates ausgesprochen, der die Diskriminierung von Schwulen und Lesben unter bestimmten Umständen erlauben soll. Carly Fiorina hält dies für "heuchlerisch". <a href="/mac-and-i/meldung/Ex-HP-Chefin-kritisiert-Tim-Cooks-Kritik-an-Gesetz-zur-Religionsfreiheit-2597091.html"><span class="mehr_schnipsel">Mehr…</span></a></a></p>                </div>            
+                                    <div class="anriss_mit_bild_links">
+                    <h3><a title="Apple Watch: Bestellungen können offenbar dauern" href="/mac-and-i/meldung/Apple-Watch-Bestellungen-koennen-offenbar-dauern-2597741.html">Apple Watch: Bestellungen können offenbar dauern</a></h3>
+                    
+                        <a href="/mac-and-i/meldung/Apple-Watch-Bestellungen-koennen-offenbar-dauern-2597741.html"><img alt="Apple Watch: Bestellungen können offenbar dauern" src="//2.f.ix.de/scale/geometry/160x90/q75/imgs/18/1/4/6/2/8/1/7/apple_watch_liefer1-ecb875926df4a41b.jpeg" /></a>
+                    
+                        <p><a href="/mac-and-i/meldung/Apple-Watch-Bestellungen-koennen-offenbar-dauern-2597741.html">Im deutschen und im britischen Apple Online Store sind kurzzeitig Liefertermine für die Computeruhr aufgetaucht. Manche Modelle brauchen demnach bis zu sechs Wochen. <a href="/mac-and-i/meldung/Apple-Watch-Bestellungen-koennen-offenbar-dauern-2597741.html"><span class="mehr_schnipsel">Mehr…</span></a></a></p>                </div>            
+                                    <div class="anriss_mit_bild_links">
+                    <h3><a title="Apple aktualisiert Xcode, OS X Server und Apple Configurator" href="/mac-and-i/meldung/Apple-aktualisiert-Xcode-OS-X-Server-und-Apple-Configurator-2597739.html">Apple aktualisiert Xcode, OS X Server und Apple Configurator</a></h3>
+                    
+                        <a href="/mac-and-i/meldung/Apple-aktualisiert-Xcode-OS-X-Server-und-Apple-Configurator-2597739.html"><img alt="Apple aktualisiert Xcode, OS X Server und Apple Configurator" src="//3.f.ix.de/scale/geometry/160x90/q75/imgs/18/1/4/6/2/8/1/5/screen800x500-ed2bb768f9ab35c2.jpeg" /></a>
+                    
+                        <p><a href="/mac-and-i/meldung/Apple-aktualisiert-Xcode-OS-X-Server-und-Apple-Configurator-2597739.html">Neben iOS 8.3 und OS X 10.10.3 hat der Hersteller auch drei weitere hauseigene Anwendungen auf den neuesten Stand gebracht. Xcode 6.3 bringt unter anderem Support für die neue Force-Touch-Technik und ein neues Swift. <a href="/mac-and-i/meldung/Apple-aktualisiert-Xcode-OS-X-Server-und-Apple-Configurator-2597739.html"><span class="mehr_schnipsel">Mehr…</span></a></a></p>                </div>            
+        
+    </div>
+
+
+
+                    
+
+<div class="teaser_frei">
+        <div class="anriss_mit_bild_links">            <h3><a href="/meldung/Quiz-Wie-gut-kennen-Sie-Apple-2504501.html">Quiz: Wie gut kennen Sie Apple?</a></h3>            <a href="/meldung/Quiz-Wie-gut-kennen-Sie-Apple-2504501.html"><img alt="Quiz: Wie gut kennen Sie Apple?" src="//3.f.ix.de/mac-and-i/imgs/65/1/4/1/5/1/2/2/Quiz-Frage-5b1e2e989f5a3cc9.png" /></a>            <p><a href="/meldung/Quiz-Wie-gut-kennen-Sie-Apple-2504501.html"><a href="/meldung/Quiz-Wie-gut-kennen-Sie-Apple-2504501.html">
+                12 Fragen rund um das Unternehmen und seine Produkte – einige davon dürften nur Fortgeschrittene knacken können.  <a href="/meldung/Quiz-Wie-gut-kennen-Sie-Apple-2504501.html"><a href="/meldung/Quiz-Wie-gut-kennen-Sie-Apple-2504501.html"><span class="mehr_schnipsel">Mehr…</span></a></a>
+            </a></a></p>    </div>
+</div>
+
+
+
+                    
+    
+    <div class="artikelteaser">            <h4><a href="/mac-and-i/artikel/">Artikel </a></h4>                            <div class="anriss_mit_bild_links">
+                    <h3><a title="Pro &amp; Contra: Hat Apple den Bogen überspannt?" href="/mac-and-i/artikel/Pro-Contra-Hat-Apple-den-Bogen-ueberspannt-2580439.html">Pro &amp; Contra: Hat Apple den Bogen überspannt?</a></h3>
+                    
+                        <a href="/mac-and-i/artikel/Pro-Contra-Hat-Apple-den-Bogen-ueberspannt-2580439.html"><img alt="" src="//3.f.ix.de/scale/geometry/160x90/q75/mac-and-i/imgs/65/1/4/5/2/5/6/5/Screen_Shot-4a5e2819f374a051.jpeg" /></a>
+                                            <p><a href="/mac-and-i/artikel/Pro-Contra-Hat-Apple-den-Bogen-ueberspannt-2580439.html">Beim neuen MacBook 12" gibt es außer der Kopfhörerbuchse nur eine Schnittstelle: USB Typ C. Darüber scheiden sich wieder mal die Geister. <a href="/mac-and-i/artikel/Pro-Contra-Hat-Apple-den-Bogen-ueberspannt-2580439.html"><span class="mehr_schnipsel">Mehr…</span></a></a></p>                </div>                                        <div class="anriss_mit_bild_links">
+                    <h3><a title="Praxistipp: Größere SSD im MacBook Air 2011" href="/mac-and-i/artikel/Praxistipp-Groessere-SSD-im-MacBook-Air-2011-2560061.html">Praxistipp: Größere SSD im MacBook Air 2011</a></h3>
+                    
+                        <a href="/mac-and-i/artikel/Praxistipp-Groessere-SSD-im-MacBook-Air-2011-2560061.html"><img alt="" src="//1.f.ix.de/scale/geometry/160x90/q75/mac-and-i/imgs/65/1/4/4/0/1/2/3/MacBook-mit-SSD-breit-d974d1a97ebb442b.png" /></a>
+                                            <p><a href="/mac-and-i/artikel/Praxistipp-Groessere-SSD-im-MacBook-Air-2011-2560061.html">Wenn das MacBook an seine Grenzen stößt, brauchen Sie nicht unbedingt ein neues: Ersatz-SSDs gibt es ab 170 Euro, der Umbau ist auch für Laien zu schaffen und in wenigen Minuten erledigt. <a href="/mac-and-i/artikel/Praxistipp-Groessere-SSD-im-MacBook-Air-2011-2560061.html"><span class="mehr_schnipsel">Mehr…</span></a></a></p>                </div>                                        <div class="anriss_mit_bild_links">
+                    <h3><a title="Pro &amp; Contra: Ist Apple zu streng?" href="/mac-and-i/artikel/Pro-Contra-Ist-Apple-zu-streng-2530140.html">Pro &amp; Contra: Ist Apple zu streng?</a></h3>
+                    
+                        <a href="/mac-and-i/artikel/Pro-Contra-Ist-Apple-zu-streng-2530140.html"><img width="71" height="100" title="" alt="" src="//2.f.ix.de/mac-and-i/imgs/65/1/4/2/2/6/3/4/Voransicht-8552184f61c1fab3.jpeg" /></a>
+                                            <p><a href="/mac-and-i/artikel/Pro-Contra-Ist-Apple-zu-streng-2530140.html">Das Prüferteam im iOS-App-Store lehnt immer häufiger ganze Apps ab oder verlangt eine Beschneidung der Funktionen. Ist das richtig so? <a href="/mac-and-i/artikel/Pro-Contra-Ist-Apple-zu-streng-2530140.html"><span class="mehr_schnipsel">Mehr…</span></a></a></p>                </div>                                        <div class="anriss_mit_bild_links">
+                    <h3><a title="Die besseren Mac minis" href="/mac-and-i/artikel/Die-besseren-Mac-minis-2445333.html">Die besseren Mac minis</a></h3>
+                    
+                        <a href="/mac-and-i/artikel/Die-besseren-Mac-minis-2445333.html"><img alt="" src="//3.f.ix.de/scale/geometry/160x90/q75/mac-and-i/imgs/65/1/3/7/4/7/9/6/MacMiniDiagonal-ddc35b43efc0f9d5-f08e367febe26a99.jpeg" /></a>
+                                            <p><a href="/mac-and-i/artikel/Die-besseren-Mac-minis-2445333.html">Nachdem wir den günstigsten Mac mini mit 1,4 GHz bereits in Mac &amp; i Heft 6/2014 vorstellen konnten, reichen wir nun – wie versprochen – einen Test der beiden  besseren Konfigurationen nach. <a href="/mac-and-i/artikel/Die-besseren-Mac-minis-2445333.html"><span class="mehr_schnipsel">Mehr…</span></a></a></p>                </div>                </div>
+
+
+
+                    
+    
+
+
+
+                    
+
+<div class="teaser_frei">
+        <div class="anriss_mit_bild_links">            <h3><a href="/meldung/In-eigener-Sache-Mac-i-im-Digitalabo-2137456.html">In eigener Sache: Mac &amp; i im Digitalabo</a></h3>            <a href="/meldung/In-eigener-Sache-Mac-i-im-Digitalabo-2137456.html"><img alt="In eigener Sache: Mac &amp;amp; i im Digitalabo" src="//2.f.ix.de/mac-and-i/imgs/65/1/1/8/9/2/5/6/14_auf_iPad_Air-32d65b8788224ed5.png" /></a>            <p><a href="/meldung/In-eigener-Sache-Mac-i-im-Digitalabo-2137456.html"><a href="/meldung/In-eigener-Sache-Mac-i-im-Digitalabo-2137456.html">
+                Heise Medien bietet Lesern, die kein gedrucktes Heft mehr wollen, nun auch ein vergünstigtes Digitalabo von Mac &amp; i für das iPad an. <a href="/meldung/In-eigener-Sache-Mac-i-im-Digitalabo-2137456.html"><a href="/meldung/In-eigener-Sache-Mac-i-im-Digitalabo-2137456.html"><span class="mehr_schnipsel">Mehr…</span></a></a>
+            </a></a></p>    </div>
+</div>
+
+
+
+                    
+    
+
+
+
+                </div>
+        </div>
+    </div>
+    <div id="bannerzone">
+        <div class="leaderboard"><script type="text/javascript">
+&lt;!--//--&gt;&lt;![CDATA[//&gt;&lt;!--
+var dfp_ord; if (!dfp_ord) { dfp_ord = Math.floor(Math.random()*1000000000)+1000000000; }
+var yp_res; if (typeof yl !== "undefined" &amp;&amp; yl.YpResult !== "undefined") { yp_res = yl.YpResult.get('66444'); }
+var rb = false; document.write('&lt;script src="http://ad-emea.doubleclick.net/N6514/adj/mac/mac-inhalt;sz=728x90,468x60;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=3;_YL_;ord=_ORD_?" type="text/javascript"&gt;&lt;\/script&gt;'.replace('_ORD_', dfp_ord).replace(';_YL_', typeof yp_res !== "undefined" &amp;&amp; yp_res.pricerange ? ';pricerange=' + yp_res.pricerange : ''));
+//--&gt;&lt;!]]&gt;
+</script><script type="text/javascript" src="http://ad-emea.doubleclick.net/N6514/adj/mac/mac-inhalt;sz=728x90,468x60;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=3;ord=1875073260?"></script>
+<noscript>&lt;div&gt;&lt;a href="http://ad-emea.doubleclick.net/N6514/jump/mac/mac-inhalt;sz=728x90,468x60;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=3;_YL_;ord=7840756744?" target="_blank"&gt;&lt;img alt="" src="http://ad-emea.doubleclick.net/N6514/ad/mac/mac-inhalt;sz=728x90,468x60;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=3;_YL_;ord=7840756744?" /&gt;&lt;/a&gt;&lt;/div&gt;</noscript>
+</div>
+        <div class="heiseadvert"></div>
+        <div class="skyscraper"><script type="text/javascript">
+&lt;!--//--&gt;&lt;![CDATA[//&gt;&lt;!--
+var dfp_ord; if (!dfp_ord) { dfp_ord = Math.floor(Math.random()*1000000000)+1000000000; }
+var yp_res; if (typeof yl !== "undefined" &amp;&amp; yl.YpResult !== "undefined") { yp_res = yl.YpResult.get('66424'); }
+if (typeof rb != "undefined" &amp;&amp; rb != true) document.write('&lt;script src="http://ad-emea.doubleclick.net/N6514/adj/mac/mac-inhalt;sz=120x600,120x800,160x600,160x800;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=4;_YL_;ord=_ORD_?" type="text/javascript"&gt;&lt;\/script&gt;'.replace('_ORD_', dfp_ord).replace(';_YL_', typeof yp_res !== "undefined" &amp;&amp; yp_res.pricerange ? ';pricerange=' + yp_res.pricerange : ''));
+//--&gt;&lt;!]]&gt;
+</script><script type="text/javascript" src="http://ad-emea.doubleclick.net/N6514/adj/mac/mac-inhalt;sz=120x600,120x800,160x600,160x800;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=4;ord=1875073260?"></script>
+<noscript>&lt;div&gt;&lt;a href="http://ad-emea.doubleclick.net/N6514/jump/mac/mac-inhalt;sz=120x600,120x800,160x600,160x800;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=4;_YL_;ord=7840756744?" target="_blank"&gt;&lt;img alt="" src="http://ad-emea.doubleclick.net/N6514/ad/mac/mac-inhalt;sz=120x600,120x800,160x600,160x800;kw=1Password,Mac%20OS%20X,Passwort,Passwortmanager,Sicherheit,TOTP;tile=4;_YL_;ord=7840756744?" /&gt;&lt;/a&gt;&lt;/div&gt;</noscript>
+</div>
+    </div>
+</div>
+
+
+
+
+<div style="display: inline;"><img width="1" height="1" alt="" src="/ivw-bin/ivw/CP/mac-and-i/meldung/1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html?url=%2Fmac-and-i%2Fmeldung%2F1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html" id="ivw_pixel_intern" /></div>
+    
+    
+
+    
+    <script type="text/javascript">
+    var iam_data = {
+        "st":"heise",
+        "cp":"mac",
+        "sv":"in",
+        "co":"%2Fmac-and-i%2Fmeldung%2F1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html"
+    }
+    iom.c(iam_data);
+    </script><script src="http://de.ioam.de/tx.io?st=heise&amp;cp=mac&amp;sv=in&amp;co=%252Fmac-and-i%252Fmeldung%252F1Password-fuer-Mac-generiert-Einmal-Passwoerter-2596987.html&amp;pt=CP&amp;rf=&amp;r2=&amp;ur=www.heise.de&amp;xy=1440x900x24&amp;lo=GB%2FBirmingham&amp;cb=0002&amp;vr=307&amp;id=q4s141&amp;lt=1428601857876&amp;ev=&amp;cs=bt2w10&amp;mo=1"></script>
+
+
+
+
+<!-- Webtrekk 3.2.2, (c) www.webtrekk.com -->
+    <script src="/js/ho/webtrekk-v3-bundle-heise-2013-01-21.js" type="text/javascript"></script>
+    <script type="text/javascript">
+    &lt;!--
+    var pageConfig = {
+        linkTrack : "standard", // Activate Link Tracking    [link or standard]
+        heatmap : "0",          // Activate Heatmap Tracking [1 = on | 0 = off]
+        form : "0",             // Activate Form Tracking    [1 = on | 0 = off]
+        trackId : "288689636920174",
+        pixelSampling : "",
+        contentId: "www.heise.de.mac-and-i.meldung.1password-fuer-mac-generiert-einmal-passwoerter-2596987"
+    };
+    var wt = new webtrekkV3(pageConfig);
+    wt.contentGroup = {};
+    wt.contentGroup['1']  = "www.heise.de";
+    wt.contentGroup['2']  = "mac-and-i";
+    wt.contentGroup['3']  = "meldung";
+    wt.contentGroup['4']  = "1password-fuer-mac-generiert-einmal-passwoerter-2596987";
+    
+    
+    
+    
+    wt.contentGroup['9']  = "1password-fuer-mac-generiert-einmal-passwoerter-2596987";
+    wt.contentGroup['10'] = "meldung";
+
+    wt.customParameter = {};
+    
+    wt.customParameter['2']  = "1password;mac os x;passwort;passwortmanager;sicherheit;totp";
+    
+    
+    
+    wt.customParameter['6']  = "1password;mac os x;passwort;passwortmanager;sicherheit;totp";
+    
+    wt.customParameter['8']  = "2015-04-08T12:46:00";
+    wt.customParameter['9']  = "mac-and-i";
+    wt.customParameter['10'] = "mac-and-i";
+
+    
+    
+
+    
+    
+
+    wt.heatmapRefpoint = 'container_content';
+    wt.sendinfo();
+
+    //--&gt;
+    </script>
+    <noscript>&lt;div&gt;&lt;img src="//prophet.heise.de/288689636920174/wt.pl?p=322,www.heise.de.mac-and-i.meldung.1password-fuer-mac-generiert-einmal-passwoerter-2596987&amp;cg1=www.heise.de&amp;cg10=meldung&amp;cg2=mac-and-i&amp;cg3=meldung&amp;cg4=1password-fuer-mac-generiert-einmal-passwoerter-2596987&amp;cg9=1password-fuer-mac-generiert-einmal-passwoerter-2596987&amp;cp10=mac-and-i&amp;cp2=1password%3Bmac%20os%20x%3Bpasswort%3Bpasswortmanager%3Bsicherheit%3Btotp&amp;cp6=1password%3Bmac%20os%20x%3Bpasswort%3Bpasswortmanager%3Bsicherheit%3Btotp&amp;cp8=2015-04-08T12%3A46%3A00&amp;cp9=mac-and-i" height="1" width="1" alt="" /&gt;&lt;/div&gt;</noscript>
+
+
+<!-- /Webtrekk -->
+
+
+
+    
+    <script type="text/javascript">
+        &lt;!--//--&gt;&lt;![CDATA[//&gt;&lt;!--
+        // VGWORT-CROP-MARKER
+        var vgwort_token='c9016af5c9984399bf413fa66e5ba786';
+                document.write('&lt;div style="display: inline;"&gt;&lt;img src="http://heise.met.vgwort.de/na/' +  vgwort_token);
+        document.write('" width="1" height="1" alt="" /&gt;&lt;' + '/div&gt;');
+        //--&gt;&lt;!]]&gt;
+    </script><div style="display: inline;"><img width="1" height="1" alt="" src="http://heise.met.vgwort.de/na/c9016af5c9984399bf413fa66e5ba786" /></div>
+
+
+
+
+
+
+
+
+
+
+
+
+	<ul id="navi_bottom">
+
+	
+		<li class="left"><a href="/Privacy-Policy-der-Heise-Medien-GmbH-Co-KG-4860.html">Datenschutzhinweis</a> </li>
+	
+
+	
+		<li class="left"><a href="/mac-and-i/impressum.html">Impressum</a></li>
+	
+
+	
+		<li class="left"><a href="/mac-and-i/kontakt/">Kontakt</a> </li>
+	
+
+	
+		<li class="left"><a rel="external" target="_blank" href="/mediadaten/mac_and_i/">Mediadaten</a> </li>
+	
+
+	
+		<li class="left"><a href="http://m.heise.de/mac-and-i/">News mobil</a></li>
+	
+
+
+<li class="bid_anzeige">
+1462351
+</li>
+
+
+    
+        <li class="right"><a style="margin-right: 0;" rel="external" href="http://www.interred.de/">Content Management</a> by <b style="margin-right: 0.5em;">Inter<span class="tx_red">Red</span></b></li>
+    
+
+	
+		
+		<li class="right"><a href="http://www.heise-medien.de/">Copyright © 2015 Heise Medien</a></li>
+		
+	
+
+	
+		
+	
+</ul>
+
+<script src="/js/heise.min.js"></script><section class="heise-modal newsletter_2014_modal"><div class="wrapper-transparent"></div><div class="stage"><button data-role="close">×</button></div></section>
+
+
+
+
+
+
+
+
+</body></html>
+<!-- Created with InterRed V15.4-x.x.x.x.1, http://www.interred.de/, by InterRed GmbH -->
+<!-- BID: 1462351, iBID: 1462432, CID: 2596987, iCID: 2597101 -->
+<!-- Link: $(LB1462351:Linktext)$ $(LC2596987:Linktext)$ -->
+<!-- Generiert: 2015-04-08 12:48:46 -->

article_scraper/resources/tests/readability/herald-sun-1/expected.html

@@ -0,0 +1,21 @@
+<article><div id="readability-page-1">
+    <div>
+                                <p><img data-src="http://api.news.com.au/content/1.0/heraldsun/images/1227261885862?format=jpg&amp;group=iphone&amp;size=medium" alt="A new Bill would require telecommunications service providers to store so-called &#x2018;metadat"/>
+                                </p>
+                                    <p>
+                                        <span id="imgCaption">A new Bill would require telecommunications service providers to store so-called ‘metadata’ for two years.</span>
+                                        <span><em>Source:</em>
+                                        Supplied</span>
+                                    </p>
+                            </div>
+        <p><strong>
+            A HIGH-powered federal government team has been doing the rounds of media organisations in the past few days in an attempt to allay concerns about the impact of new surveillance legislation on press freedom. It failed.
+            </strong></p>
+    <p>The roadshow featured the Prime Minister’s national security adviser, Andrew Shearer, Justin Bassi, who advises Attorney-General George Brandis on crime and security matters, and Australian Federal Police Commissioner Andrew Colvin. Staffers from the office of Communications Minister Malcolm Turnbull also took part.</p><p>They held meetings with executives from News Corporation and Fairfax, representatives of the TV networks, the ABC top brass and a group from the media union and the Walkley journalism foundation. I was involved as a member of the Walkley board.</p><p>The initiative, from Tony Abbott’s office, is evidence that the Government has been alarmed by the strength of criticism from media of the Data Retention Bill it wants passed before Parliament rises in a fortnight. Bosses, journalists, even the Press Council, are up in arms, not only over this measure, but also over aspects of two earlier pieces of national security legislation that interfere with the ability of the media to hold government to account.</p>
+    
+            
+      
+<div id="read-more">
+    <p>The Bill would require telecommunications service providers to store so-called “metadata” — the who, where, when and how of a communication, but not its content — for two years so security and law enforcement agencies can access it without warrant. Few would argue against the use of such material to catch criminals or terrorists. But, as Parliament’s Joint Committee on Intelligence and Security has pointed out, it would also be used “for the purpose of determining the identity of a journalist’s sources”.</p><p>And that should ring warning bells for anyone genuinely concerned with the health of our democracy. Without the ability to protect the identity of sources, journalists would be greatly handicapped in exposing corruption, dishonesty, waste, incompetence and misbehaviour by public officials.</p><p>The Press Council is concerned the laws would crush investigative journalism.</p><p>“These legitimate concerns cannot be addressed effectively short of exempting journalists and media organisations,” says president David Weisbrot.</p><p>The media union is adamant journalists’ metadata must be exempted from the law. That’s what media bosses want, too, though they have a fallback position based on new safeguards being implemented in Britain.</p><p>That would prevent access to the metadata of journalists or media organisations without a judicial warrant. There would be a code including — according to the explanatory notes of the British Bill — “provision to protect the public interest in the confidentiality of journalistic sources”.</p><p>In their meetings this week, the government team boasted of concessions in the new Data Retention Bill. The number of agencies able to access metadata will be reduced by excluding such organisations as the RSPCA and local councils. And whenever an authorisation is issued for access to information about a journalist’s sources, the Ombudsman (or, where ASIO is involved, the Inspector-General of Intelligence and Security) will receive a copy.</p><p>That does nothing to solve the problem. The Government has effectively admitted as much by agreeing that the parliamentary committee should conduct a separate review of how to deal with the issue of journalists’ sources.</p><p>But another inquiry would be a waste of time — the committee has already received and considered dozens of submissions on the subject. The bottom line is that the Government does not deny that the legislation is flawed, but is demanding it be passed anyway with the possibility left open of a repair job down the track. That is a ridiculous approach.</p><p>Claims that immediate action is imperative do not stand up. These are measures that won’t come into full effect for two years. Anyway, amending the Bill to either exempt journalists or adopt the UK model could be done quickly, without any risk to national security.</p><p>AS Opposition Leader Bill Shorten said in a letter to Abbott last month: “Press freedom concerns about mandatory data retention would ideally be addressed in this Bill to avoid the need for future additional amendments or procedures to be put in place in the future.”</p><p>The Data Retention Bill will be debated in the House of Representatives this week. Then, on Friday, CEOs from leading media organisations will front the parliamentary committee to air their concerns before the legislation goes to the Senate.</p><p>Those CEOs should make it clear they are just as angry about this as they were about Stephen Conroy’s attempt to impinge on press freedom through media regulation under the previous Labor government.</p><p>Memories of the grief Conroy brought down on his head would undoubtedly make Abbott sit up and take notice.</p><p><b>LAURIE OAKES IS THE NINE NETWORK POLITICAL EDITOR </b></p>
+</div>
+</div></article>

article_scraper/resources/tests/readability/hidden-nodes/expected.html

@@ -0,0 +1,14 @@
+<article><DIV id="readability-page-1">
+        
+        <p>
+            Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+        </p>
+        <h2>
+            Secondary header
+        </h2>
+        
+        <h2>
+            Third header
+        </h2>
+        
+    </DIV></article>

article_scraper/resources/tests/readability/hidden-nodes/source.html

@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html lang="en" xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+    <head>
+        <meta charset="utf-8" />
+        <title>
+            Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
+        </title>
+    </head>
+    <body>
+        <h1>
+            Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
+        </h1>
+        <p>
+            Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+        </p>
+        <h2>
+            Secondary header
+        </h2>
+        <p style="display: none;">
+            Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+        </p>
+        <h2>
+            Third header
+        </h2>
+        <p hidden="hidden">
+            Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+        </p>
+    </body>
+</html>

article_scraper/resources/tests/readability/hukumusume/expected.html

@@ -0,0 +1,240 @@
+<article><DIV id="readability-page-1">
+                    <td>
+                        <table>
+                            <tbody>
+                                <tr>
+                                    <td>
+                                        <img src="http://fakehost/366/logo_bana/corner_1.gif" width="7" height="7"/>
+                                    </td>
+                                    <td><empty></empty></td>
+                                    <td>
+                                        <img src="http://fakehost/366/logo_bana/corner_2.gif" width="7" height="7"/>
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                        <table>
+                            <tbody>
+                                <tr>
+                                    <td>
+                                         
+                                        
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                         
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                    </td>
+                    <td>
+                        <p>
+                            <a href="http://fakehost/index.html" target="_blank">福娘童話集</a> &gt; <a href="http://fakehost/test/index.html" target="_blank">きょうのイソップ童話</a> &gt; <a href="http://fakehost/test/itiran/01gatu.htm" target="_blank">１月のイソップ童話</a> &gt; 欲張りなイヌ
+                        </p>
+                        <DIV>
+                            <p><span color="#FF0000" size="+2">元旦のイソップ童話</span></p><p>
+                            <img src="http://fakehost/gazou/pc_gazou/aesop/aesop052.jpg" alt="&#x3088;&#x304F;&#x3070;&#x308A;&#x306A;&#x30A4;&#x30CC;" width="480" height="360"/></p><p>
+                            欲張りなイヌ</p><p>
+                            <a href="http://hukumusume.com/douwa/English/aesop/01/01_j.html" target="_blank">ひらがな</a>　←→　<a href="http://hukumusume.com/douwa/English/aesop/01/01_j&amp;E.html" target="_blank">日本語・英語</a>　←→　<a href="http://hukumusume.com/douwa/English/aesop/01/01_E.html" target="_blank">English</a></p></DIV>
+                        <DIV>
+                                        <table>
+                                            <tbody>
+                                                <tr>
+                                                    <td>
+                                                        <img src="http://fakehost/366/logo_bana/corner_1.gif" width="7" height="7"/>
+                                                    </td>
+                                                    <td>
+                                                        <span color="#FF0000"><b>おりがみをつくろう</b></span>
+                                                    </td>
+                                                    <td>
+                                                        <span size="-1">( <a href="http://www.origami-club.com/index.html" target="_blank">おりがみくらぶ</a> より)</span>
+                                                    </td>
+                                                    <td>
+                                                        <img src="http://fakehost/366/logo_bana/corner_2.gif" width="7" height="7"/>
+                                                    </td>
+                                                </tr>
+                                                <tr>
+                                                    <td colspan="4">
+                                                        <table>
+                                                            <tbody>
+                                                                <tr>
+                                                                    <td>
+                                                                        <a href="http://www.origami-club.com/easy/dogfase/index.html" target="_blank"><span size="+2"><img src="http://fakehost/gazou/origami_gazou/kantan/dogface.gif" alt="&#x72AC;&#x306E;&#x9854;&#x306E;&#x6298;&#x308A;&#x7D19;" width="73" height="51"/>いぬのかお</span></a>　　　<a href="http://www.origami-club.com/easy/dog/index.html" target="_blank"><img src="http://fakehost/gazou/origami_gazou/kantan/dog.gif" alt="&#x72AC;&#x306E;&#x9854;&#x306E;&#x7D19;" width="62" height="43"/><span size="+2">いぬ</span></a>
+                                                                    </td>
+                                                                </tr>
+                                                            </tbody>
+                                                        </table>
+                                                    </td>
+                                                </tr>
+                                            </tbody>
+                                        </table>
+                                    </DIV>
+                        <table>
+                            <tbody>
+                                <tr>
+                                    <td>
+                                        ♪音声配信(html5)
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <audio src="http://ohanashi2.up.seesaa.net/mp3/ae_0101.mp3" controls=""><empty></empty></audio>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <a href="http://www.voiceblog.jp/onokuboaki/" target="_blank"><span size="-1">亜姫の朗読☆　イソップ童話より</span></a>
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                        <p>
+                            　肉をくわえたイヌが、橋を渡っていました。 　ふと下を見ると、川の中にも肉をくわえたイヌがいます。　イヌはそれを見て、思いました。（あいつの肉の方が、大きそうだ） 　イヌは、くやしくてたまりません。 （そうだ、あいつをおどかして、あの肉を取ってやろう） 　そこでイヌは、川の中のイヌに向かって思いっきり吠えました。 「ウゥー、ワン！！」 　そのとたん、くわえていた肉はポチャンと川の中に落ちてしまいました。 「ああー、ぁぁー」 　川の中には、がっかりしたイヌの顔がうつっています。 　さっきの川の中のイヌは、水にうつった自分の顔だったのです。 　同じ物を持っていても、人が持っている物の方が良く見え、また、欲張るとけっきょく損をするというお話しです。
+                        </p>
+                        <p>
+                            おしまい
+                        </p>
+                        <p>
+                            <span><img src="http://fakehost/gazou/pc_gazou/all/top_bana/back_logo_r.gif" alt="&#x524D;&#x306E;&#x30DA;&#x30FC;&#x30B8;&#x3078;&#x623B;&#x308B;" name="Image10" width="175" height="32" id="Image10"/></span></p>
+                    </td>
+                    <td>
+                        <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1"/>
+                    </td>
+                    <td>
+                        <table>
+                            <tbody>
+                                <tr>
+                                    <td>
+                                        <img src="http://fakehost/366/logo_bana/corner_1.gif" width="7" height="7"/>
+                                    </td>
+                                    <td><empty></empty></td>
+                                    <td>
+                                        <img src="http://fakehost/366/logo_bana/corner_2.gif" width="7" height="7"/>
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                        <table>
+                            <tbody>
+                                <tr>
+                                    <td>
+                                             <span size="-1"><b>1月 1日の豆知識</b></span><p>
+                                        <span size="-2"><u><p>
+                                        366日への旅</p></u></span></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97" width="1" height="1"/><b><span size="-1">きょうの記念日</span></b><p>
+                                        <a href="http://fakehost/366/kinenbi/pc/01gatu/1_01.htm" target="_blank"><span size="-1">元旦</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1"/><b><span size="-1">きょうの誕生花</span></b><p>
+                                        <a href="http://fakehost/366/hana/pc/01gatu/1_01.htm" target="_blank"><span size="-1">松(まつ)</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <b><span size="-1">きょうの誕生日・出来事</span></b><p>
+                                        <a href="http://fakehost/366/birthday/pc/01gatu/1_01.htm" target="_blank"><span size="-1">1949年　Mr.マリック(マジシャン)</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <b><span size="-1">恋の誕生日占い</span></b><p>
+                                        <a href="http://fakehost/sakura/uranai/birthday/01/01.html" target="_blank"><span size="-1">自分の考えをしっかりと持った女の子。</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <b><span size="-1">なぞなぞ小学校</span></b><p>
+                                        <a href="http://fakehost/nazonazo/new/2012/04/02.html" target="_blank"><span size="-1">○(丸)を取ったらお母さんになってしまう男の人は？</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <b><span size="-1">あこがれの職業紹介</span></b><p>
+                                        <a href="http://fakehost/sakura/navi/work/2017/041.html" target="_blank"><span size="-1">歌手</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <b><span size="-1">恋の魔法とおまじない</span></b> 001<p>
+                                        <a href="http://fakehost/omajinai/new/2012/00/re01.html" target="_blank"><span size="-1">両思いになれる おまじない</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <span size="-1">　　<b>1月 1日の童話・昔話</b><u><span size="-2"><p>
+                                        福娘童話集</p></span></u></span>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <b><span size="-1">きょうの日本昔話</span></b><p>
+                                        <a href="http://fakehost/douwa/pc/jap/01/01.htm" target="_blank"><span size="-1">ネコがネズミを追いかける訳</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <b><span size="-1">きょうの世界昔話<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1"/></span></b><p>
+                                        <a href="http://fakehost/douwa/pc/world/01/01a.htm" target="_blank"><span size="-1">モンゴルの十二支話</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1"/><b><span size="-1">きょうの日本民話</span></b><p>
+                                        <a href="http://fakehost/douwa/pc/minwa/01/01c.html" target="_blank"><span size="-1">仕事の取替えっこ</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <b><span size="-1">きょうのイソップ童話</span></b><p>
+                                        <a href="http://fakehost/douwa/pc/aesop/01/01.htm" target="_blank"><span size="-1">欲張りなイヌ</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <b><span size="-1">きょうの江戸小話</span></b><p>
+                                        <a href="http://fakehost/douwa/pc/kobanashi/01/01.htm" target="_blank"><span size="-1">ぞうきんとお年玉</span></a></p></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <b><span size="-1">きょうの百物語</span></b><p>
+                                        <a href="http://fakehost/douwa/pc/kaidan/01/01.htm" target="_blank"><span size="-1">百物語の幽霊</span></a></p></td>
+                                </tr>
+                            </tbody>
+                        </table>
+                        <table>
+                            <tbody>
+                                <tr>
+                                    <td>
+                                        <b><span size="-1">福娘のサイト</span></b>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <span size="-1"><b>366日への旅</b><p>
+                                        <a href="http://hukumusume.com/366/" target="_blank">毎日の記念日・誕生花 ・有名人の誕生日と性格判断</a></p></span>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <span size="-1"><b>福娘童話集</b><p>
+                                        <a href="http://hukumusume.com/douwa/" target="_blank">世界と日本の童話と昔話</a></p></span>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <span size="-1"><b>女の子応援サイト -さくら-</b><p>
+                                        <a href="http://hukumusume.com/sakura/index.html" target="_blank">誕生日占い、お仕事紹介、おまじない、など</a></p></span>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <span size="-1"><b>子どもの病気相談所</b><p>
+                                        <a href="http://hukumusume.com/my_baby/sick/" target="_blank">病気検索と対応方法、症状から検索するWEB問診</a></p></span>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <span size="-1"><b>世界60秒巡り</b><p>
+                                        <a href="http://hukumusume.com/366/world/" target="_blank">国旗国歌や世界遺産など、世界の国々の豆知識</a></p></span>
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                    </td>
+                </DIV></article>

article_scraper/resources/tests/readability/hukumusume/source.html

@@ -0,0 +1,356 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+    <head>
+        <title>
+            欲張りなイヌ　＜福娘童話集　きょうのイソップ童話＞
+        </title>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    </head>
+    <body bgcolor="#FFFFFF" text="#000000">
+        <table width="969" border="0" align="center" cellpadding="0" cellspacing="0">
+            <tbody>
+                <tr>
+                    <td colspan="5" height="12">
+                        <div align="center">
+                            <table width="100%" border="0">
+                                <tbody>
+                                    <tr>
+                                        <td width="27%" align="center">
+                                            <a href="../../../index.html">福娘童話集</a> &gt; <a href="../index.html">きょうのイソップ童話</a>
+                                        </td>
+                                        <td width="46%" align="center">
+                                            <a href="http://hukumusume.com/douwa/pc/aesop/index.html"><img src="../../../gazou/pc_gazou/all/aesop_logo_llll.gif" alt="福娘童話集　きょうのイソップ童話" width="320" height="100" border="0" /></a>
+                                        </td>
+                                        <td width="27%" align="center" valign="bottom">
+                                            <a href="http://hukumusume.com/douwa/index.html"><img src="../../../gazou/pc_gazou/all/douwa_logo_top_.gif" alt="童話・昔話・おとぎ話の福娘童話集" width="170" height="50" border="0" /></a>
+                                        </td>
+                                    </tr>
+                                </tbody>
+                            </table>
+                        </div>
+                    </td>
+                </tr>
+                <tr>
+                    <td width="166" height="830" valign="top">
+                        <table width="166" border="0" cellpadding="0" cellspacing="0" bgcolor="#C8FFC8">
+                            <tbody>
+                                <tr>
+                                    <td height="7" valign="top">
+                                        <img src="../../../../366/logo_bana/corner_1.gif" width="7" height="7" />
+                                    </td>
+                                    <td></td>
+                                    <td align="right" valign="top">
+                                        <img src="../../../../366/logo_bana/corner_2.gif" width="7" height="7" />
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                        <table width="166" border="0" cellpadding="0" cellspacing="0">
+                            <tbody>
+                                <tr>
+                                    <td width="310" height="610" align="center" bgcolor="#C8FFC8">
+                                        <script type="text/javascript">
+                                        //<![CDATA[
+                                        <!--
+                                        google_ad_client = "ca-pub-2746615155806331";
+                                        /* 1a月160x600 */
+                                        google_ad_slot = "0764542773";
+                                        google_ad_width = 160;
+                                        google_ad_height = 600;
+                                        //-->
+                                        //]]>
+                                        </script> 
+                                        <script type="text/javascript" src="http://pagead2.googlesyndication.com/pagead/show_ads.js"></script>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td align="center">
+                                        &#160;
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                    </td>
+                    <td width="619" valign="top">
+                        <p align="center">
+                            <a href="../../../index.html">福娘童話集</a> &gt; <a href="../index.html">きょうのイソップ童話</a> &gt; <a href="../itiran/01gatu.htm">１月のイソップ童話</a> &gt; 欲張りなイヌ
+                        </p>
+                        <p align="center">
+                            <font color="#FF0000" size="+2">元旦のイソップ童話</font><br />
+                            <br />
+                            <br />
+                            <br />
+                            <img src="../../../gazou/pc_gazou/aesop/aesop052.jpg" alt="よくばりなイヌ" width="480" height="360" border="1" /><br />
+                            <br />
+                            <br />
+                            <br />
+                            欲張りなイヌ<br />
+                            <br />
+                            <br />
+                            <br />
+                            <a href="http://hukumusume.com/douwa/English/aesop/01/01_j.html">ひらがな</a>　←→　<a href="http://hukumusume.com/douwa/English/aesop/01/01_j&amp;E.html">日本語・英語</a>　←→　<a href="http://hukumusume.com/douwa/English/aesop/01/01_E.html">English</a>
+                        </p>
+                        <table width="100%" border="0" cellspacing="0" cellpadding="0">
+                            <tbody>
+                                <tr>
+                                    <td height="90" align="center">
+                                        <table width="80%" border="0" cellpadding="0" cellspacing="0" bgcolor="#C8FFC8">
+                                            <tbody>
+                                                <tr>
+                                                    <td width="35%" height="25" valign="top">
+                                                        <img src="../../../../366/logo_bana/corner_1.gif" width="7" height="7" />
+                                                    </td>
+                                                    <td width="29%" align="center">
+                                                        <font color="#FF0000"><b>おりがみをつくろう</b></font>
+                                                    </td>
+                                                    <td width="35%" valign="bottom">
+                                                        <font size="-1">( <a href="http://www.origami-club.com/index.html">おりがみくらぶ</a> より)</font>
+                                                    </td>
+                                                    <td width="1%" align="right" valign="top">
+                                                        <img src="../../../../366/logo_bana/corner_2.gif" width="7" height="7" />
+                                                    </td>
+                                                </tr>
+                                                <tr>
+                                                    <td height="80" colspan="4" align="center" valign="top">
+                                                        <table width="98%" border="0" cellspacing="0" cellpadding="0">
+                                                            <tbody>
+                                                                <tr>
+                                                                    <td height="75" align="center" valign="middle" bgcolor="#ECFFEC">
+                                                                        <a href="http://www.origami-club.com/easy/dogfase/index.html"><font size="+2"><img src="../../../gazou/origami_gazou/kantan/dogface.gif" alt="犬の顔の折り紙" width="73" height="51" border="0" />いぬのかお</font></a>　　　<a href="http://www.origami-club.com/easy/dog/index.html"><img src="../../../gazou/origami_gazou/kantan/dog.gif" alt="犬の顔の紙" width="62" height="43" border="0" /><font size="+2">いぬ</font></a>
+                                                                    </td>
+                                                                </tr>
+                                                            </tbody>
+                                                        </table>
+                                                    </td>
+                                                </tr>
+                                            </tbody>
+                                        </table>
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                        <table width="100%" border="0">
+                            <tbody>
+                                <tr>
+                                    <td align="center">
+                                        ♪音声配信(html5)
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td align="center">
+                                        <audio src="http://ohanashi2.up.seesaa.net/mp3/ae_0101.mp3" controls=""></audio>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td align="center">
+                                        <a href="http://www.voiceblog.jp/onokuboaki/"><font size="-1">亜姫の朗読☆　イソップ童話より</font></a>
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                        <p>
+                            　肉をくわえたイヌが、橋を渡っていました。 　ふと下を見ると、川の中にも肉をくわえたイヌがいます。　イヌはそれを見て、思いました。（あいつの肉の方が、大きそうだ） 　イヌは、くやしくてたまりません。 （そうだ、あいつをおどかして、あの肉を取ってやろう） 　そこでイヌは、川の中のイヌに向かって思いっきり吠えました。 「ウゥー、ワン！！」 　そのとたん、くわえていた肉はポチャンと川の中に落ちてしまいました。 「ああー、ぁぁー」 　川の中には、がっかりしたイヌの顔がうつっています。 　さっきの川の中のイヌは、水にうつった自分の顔だったのです。 　同じ物を持っていても、人が持っている物の方が良く見え、また、欲張るとけっきょく損をするというお話しです。
+                        </p>
+                        <p align="center">
+                            おしまい
+                        </p>
+                        <p align="center">
+                            <a href="javascript:history.back();" onmouseout="MM_swapImgRestore()" onmouseover="MM_swapImage('Image10','','../../../gazou/pc_gazou/all/top_bana/back_logo_b.gif',1)"><img src="../../../gazou/pc_gazou/all/top_bana/back_logo_r.gif" alt="前のページへ戻る" name="Image10" width="175" height="32" border="0" id="Image10" /></a><br />
+                            <br />
+                            <br />
+                            <br />
+                            <script type="text/javascript">
+                            //<![CDATA[
+
+                            <!--
+                            google_ad_client = "ca-pub-2746615155806331";
+                            /* 1月336x280 */
+                            google_ad_slot = "6046482409";
+                            google_ad_width = 336;
+                            google_ad_height = 280;
+                            //-->
+                            //]]>
+                            </script> 
+                            <script type="text/javascript" src="http://pagead2.googlesyndication.com/pagead/show_ads.js"></script>
+                        </p>
+                    </td>
+                    <td width="10">
+                        <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" />
+                    </td>
+                    <td width="166" valign="top">
+                        <table width="100%" border="0" cellpadding="0" cellspacing="0" bgcolor="#C8FFC8">
+                            <tbody>
+                                <tr>
+                                    <td height="7" valign="top">
+                                        <img src="../../../../366/logo_bana/corner_1.gif" width="7" height="7" />
+                                    </td>
+                                    <td></td>
+                                    <td align="right" valign="top">
+                                        <img src="../../../../366/logo_bana/corner_2.gif" width="7" height="7" />
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                        <table width="166" border="0" bgcolor="#C8FFC8">
+                            <tbody>
+                                <tr>
+                                    <td width="156" height="50">
+                                        &#160;&#160;&#160;&#160; <font size="-1"><b>1月 1日の豆知識</b></font><br />
+                                        <br />
+                                        <font size="-2"><u><br />
+                                        <br />
+                                        366日への旅</u></font>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97" width="1" height="1" /><b><font size="-1">きょうの記念日</font></b><br />
+                                        <br />
+                                        <a href="../../../../366/kinenbi/pc/01gatu/1_01.htm"><font size="-1">元旦</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /><b><font size="-1">きょうの誕生花</font></b><br />
+                                        <br />
+                                        <a href="../../../../366/hana/pc/01gatu/1_01.htm"><font size="-1">松(まつ)</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <b><font size="-1">きょうの誕生日・出来事</font></b><br />
+                                        <br />
+                                        <a href="../../../../366/birthday/pc/01gatu/1_01.htm"><font size="-1">1949年　Mr.マリック(マジシャン)</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <b><font size="-1">恋の誕生日占い</font></b><br />
+                                        <br />
+                                        <a href="../../../../sakura/uranai/birthday/01/01.html"><font size="-1">自分の考えをしっかりと持った女の子。</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <b><font size="-1">なぞなぞ小学校</font></b><br />
+                                        <br />
+                                        <a href="../../../../nazonazo/new/2012/04/02.html"><font size="-1">○(丸)を取ったらお母さんになってしまう男の人は？</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <b><font size="-1">あこがれの職業紹介</font></b><br />
+                                        <br />
+                                        <a href="../../../../sakura/navi/work/2017/041.html"><font size="-1">歌手</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <b><font size="-1">恋の魔法とおまじない</font></b> 001<br />
+                                        <br />
+                                        <a href="../../../../omajinai/new/2012/00/re01.html"><font size="-1">両思いになれる おまじない</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#C8FFC8">
+                                        <font size="-1">　　<b>1月 1日の童話・昔話</b><br />
+                                        <br />
+                                        <u><font size="-2"><br />
+                                        <br />
+                                        福娘童話集</font></u></font>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <b><font size="-1">きょうの日本昔話</font></b><br />
+                                        <br />
+                                        <a href="../../../../douwa/pc/jap/01/01.htm"><font size="-1">ネコがネズミを追いかける訳</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <b><font size="-1">きょうの世界昔話<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /></font></b><br />
+                                        <br />
+                                        <a href="../../../../douwa/pc/world/01/01a.htm"><font size="-1">モンゴルの十二支話</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /><b><font size="-1">きょうの日本民話</font></b><br />
+                                        <br />
+                                        <a href="../../../../douwa/pc/minwa/01/01c.html"><font size="-1">仕事の取替えっこ</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <b><font size="-1">きょうのイソップ童話</font></b><br />
+                                        <br />
+                                        <a href="../../../../douwa/pc/aesop/01/01.htm"><font size="-1">欲張りなイヌ</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <b><font size="-1">きょうの江戸小話</font></b><br />
+                                        <br />
+                                        <a href="../../../../douwa/pc/kobanashi/01/01.htm"><font size="-1">ぞうきんとお年玉</font></a>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="50" bgcolor="#ECFFEC">
+                                        <b><font size="-1">きょうの百物語</font></b><br />
+                                        <br />
+                                        <a href="../../../../douwa/pc/kaidan/01/01.htm"><font size="-1">百物語の幽霊</font></a>
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                        <table width="100%" border="0" bgcolor="#C8FFC8">
+                            <tbody>
+                                <tr>
+                                    <td height="30" align="center" bgcolor="#C8FFC8">
+                                        <b><font size="-1">福娘のサイト</font></b>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="60" bgcolor="#ECFFEC">
+                                        <font size="-1"><b>366日への旅</b><br />
+                                        <br />
+                                        <a href="http://hukumusume.com/366/">毎日の記念日・誕生花 ・有名人の誕生日と性格判断</a></font>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="60" bgcolor="#ECFFEC">
+                                        <font size="-1"><b>福娘童話集</b><br />
+                                        <br />
+                                        <a href="http://hukumusume.com/douwa/">世界と日本の童話と昔話</a></font>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="60" bgcolor="#ECFFEC">
+                                        <font size="-1"><b>女の子応援サイト -さくら-</b><br />
+                                        <br />
+                                        <a href="http://hukumusume.com/sakura/index.html">誕生日占い、お仕事紹介、おまじない、など</a></font>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="60" bgcolor="#ECFFEC">
+                                        <font size="-1"><b>子どもの病気相談所</b><br />
+                                        <br />
+                                        <a href="http://hukumusume.com/my_baby/sick/">病気検索と対応方法、症状から検索するWEB問診</a></font>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td height="60" bgcolor="#ECFFEC">
+                                        <font size="-1"><b>世界60秒巡り</b><br />
+                                        <br />
+                                        <a href="http://hukumusume.com/366/world/">国旗国歌や世界遺産など、世界の国々の豆知識</a></font>
+                                    </td>
+                                </tr>
+                            </tbody>
+                        </table>
+                    </td>
+                </tr>
+            </tbody>
+        </table>
+    </body>
+</html>

article_scraper/resources/tests/readability/iab-1/expected.html

@@ -0,0 +1,22 @@
+<article><div id="readability-page-1">
+			<p>We messed up. As technologists, tasked with delivering content and services to users, we lost track of the user experience.</p>
+<p>Twenty years ago we saw an explosion of websites, built by developers around the world, providing all forms of content. This was the beginning of an age of enlightenment, the intersection of content and technology. Many of us in the technical field felt compelled, and even empowered, to produce information as the distribution means for mass communication were no longer restricted by a high barrier to entry.</p>
+<p>In 2000, the dark ages came when the dot-com bubble burst. We were told that our startups were gone or that our divisions sustained by corporate parent companies needed to be in the black. It was a wakeup call that led to a renaissance age. Digital advertising became the foundation of an economic engine that, still now, sustains the free and democratic World Wide Web. In digital publishing, we strived to balance content, commerce, and technology. The content management systems and communication gateways we built to inform and entertain populations around the world disrupted markets and in some cases governments, informed communities of imminent danger, and liberated new forms of art and entertainment—all while creating a digital middle class of small businesses.</p>
+<p>We engineered not just the technical, but also the social and economic foundation that users around the world came to lean on for access to real time information. And users came to expect this information whenever and wherever they needed it. And more often than not, for anybody with a connected device, it was free.</p>
+<p>This was choice—powered by digital advertising—and premised on user experience.</p>
+<p>But we messed up.</p>
+<p>Through our pursuit of further automation and maximization of margins during the industrial age of media technology, we built advertising technology to optimize publishers’ yield of marketing budgets that had eroded after the last recession. Looking back now, our scraping of dimes may have cost us dollars in consumer loyalty. The fast, scalable systems of targeting users with ever-heftier advertisements have slowed down the public internet and drained more than a few batteries. We were so clever and so good at it that we over-engineered the capabilities of the plumbing laid down by, well, ourselves. This steamrolled the users, depleted their devices, and tried their patience.</p>
+<p>The rise of ad blocking poses a threat to the internet and could potentially drive users to an enclosed platform world dominated by a few companies. We have let the fine equilibrium of content, commerce, and technology get out of balance in the open web. We had, and still do have, a responsibility to educate the business side, and in some cases to push back. We lost sight of our social and ethical responsibility to provide a safe, usable experience for anyone and everyone wanting to consume the content of their choice.</p>
+<p>We need to bring that back into alignment, starting right now.</p>
+<p><a href="http://www.iab.com/wp-content/uploads/2015/10/getting-lean-with-digital-ad-ux.jpg" target="_blank"><img width="300" height="250" alt="Getting LEAN with Digital Ad UX" src="http://www.iab.com/wp-content/uploads/2015/10/getting-lean-with-digital-ad-ux-300x250.jpg"/></a>Today, the IAB Tech Lab is launching the L.E.A.N. Ads program. Supported by the Executive Committee of the IAB Tech Lab Board, IABs around the world, and hundreds of member companies, L.E.A.N. stands for Light, Encrypted, Ad choice supported, Non-invasive ads. These are principles that will help guide the next phases of advertising technical standards for the global digital advertising supply chain.</p>
+<p>As with any other industry, standards should be created by non-profit standards-setting bodies, with many diverse voices providing input. We will invite all parties for public comment, and make sure consumer interest groups have the opportunity to provide input.</p>
+<p>L.E.A.N. Ads do not replace the current advertising standards many consumers still enjoy and engage with while consuming content on our sites across all IP enabled devices. Rather, these principles will guide an alternative set of standards that provide choice for marketers, content providers, and consumers.</p>
+<p>Among the many areas of concentration, we must also address frequency capping on retargeting in Ad Tech and make sure a user is targeted appropriately before, but never AFTER they make a purchase. If we are so good at reach and scale, we can be just as good, if not better, at moderation. Additionally, we must address volume of ads per page as well as continue on the path to viewability. The dependencies here are critical to an optimized user experience.</p>
+<p>The consumer is demanding these actions, challenging us to do better, and we must respond.</p>
+<p>The IAB Tech Lab will continue to provide the tools for publishers in the digital supply chain to have a dialogue with users about their choices so that content providers can generate revenue while creating value. Publishers should have the opportunity to provide rich advertising experiences, L.E.A.N. advertising experiences, and subscription services. Or publishers can simply deny their service to users who choose to keep on blocking ads. That is all part of elasticity of consumer tolerance and choice.</p>
+<p>Finally, we must do this in an increasingly fragmented market, across screens. We must do this in environments where entire sites are blocked, purposefully or not. Yes, it is disappointing that our development efforts will have to manage with multiple frameworks while we work to supply the economic engine to sustain an open internet. However, our goal is still to provide diverse content and voices to as many connected users as possible around the world.</p>
+<p>That is user experience.</p>
+
+<P>IAB Tech Lab Members can join the IAB Tech Lab Ad Blocking Working Group, please email <a href="mailto:adblocking@iab.com" target="_blank">adblocking@iab.com</a> for more information.</P>
+<p>Read <a target="_blank" href="http://www.iab.com/insights/ad-blocking/">more about ad blocking here</a>.</p>
+					</div></article>