misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-07 16:15:32 +02:00
Code Releases Activity

update deps & fix some flaky tests

Browse source
This commit is contained in:
Jan Lukas Gernert 2024-02-13 17:00:45 +01:00
parent f9812b556c
commit ed8a83708b
3 changed files with 13 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

12
article_scraper/Cargo.toml
View file

@ -14,18 +14,18 @@ exclude = ["resources/tests"]
thiserror = "1.0" thiserror = "1.0"
libxml = "0.3" libxml = "0.3"
reqwest = { version = "0.11", features = ["json", "native-tls", "gzip", "brotli", "stream"] } reqwest = { version = "0.11", features = ["json", "native-tls", "gzip", "brotli", "stream"] }
tokio = { version = "1.28", features = ["macros", "fs", "io-util"] } tokio = { version = "1", features = ["macros", "fs", "io-util"] }
url = "2.3" url = "2.5"
regex = "1.8" regex = "1.10"
encoding_rs = "0.8" encoding_rs = "0.8"
chrono = "0.4" chrono = "0.4"
base64 = "0.21" base64 = "0.21"
image = "0.24" image = "0.24"
log = "0.4" log = "0.4"
rust-embed="6.6" rust-embed="8.2"
once_cell = "1.17" once_cell = "1.19"
escaper = "0.1" escaper = "0.1"
futures = "0.3" futures = "0.3"
[dev-dependencies] [dev-dependencies]
env_logger = "0.10" env_logger = "0.11"

10
article_scraper/resources/tests/readability/hukumusume/source.html
View file

@ -175,7 +175,7 @@
</p> </p>
</td> </td>
<td width="10"> <td width="10">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /> <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" />
</td> </td>
<td width="166" valign="top"> <td width="166" valign="top">
<table width="100%" border="0" cellpadding="0" cellspacing="0" bgcolor="#C8FFC8"> <table width="100%" border="0" cellpadding="0" cellspacing="0" bgcolor="#C8FFC8">
@ -204,14 +204,14 @@
</tr> </tr>
<tr> <tr>
<td height="50" bgcolor="#ECFFEC"> <td height="50" bgcolor="#ECFFEC">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97" width="1" height="1" /><b><font size="-1">きょうの記念日</font></b><br /> <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97" width="1" height="1" /><b><font size="-1">きょうの記念日</font></b><br />
<br /> <br />
<a href="../../../../366/kinenbi/pc/01gatu/1_01.htm"><font size="-1">元旦</font></a> <a href="../../../../366/kinenbi/pc/01gatu/1_01.htm"><font size="-1">元旦</font></a>
</td> </td>
</tr> </tr>
<tr> <tr>
<td height="50" bgcolor="#ECFFEC"> <td height="50" bgcolor="#ECFFEC">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /><b><font size="-1">きょうの誕生花</font></b><br /> <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /><b><font size="-1">きょうの誕生花</font></b><br />
<br /> <br />
<a href="../../../../366/hana/pc/01gatu/1_01.htm"><font size="-1">松(まつ)</font></a> <a href="../../../../366/hana/pc/01gatu/1_01.htm"><font size="-1">松(まつ)</font></a>
</td> </td>
@ -269,14 +269,14 @@
</tr> </tr>
<tr> <tr>
<td height="50" bgcolor="#ECFFEC"> <td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうの世界昔話<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /></font></b><br /> <b><font size="-1">きょうの世界昔話<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /></font></b><br />
<br /> <br />
<a href="../../../../douwa/pc/world/01/01a.htm"><font size="-1">モンゴルの十二支話</font></a> <a href="../../../../douwa/pc/world/01/01a.htm"><font size="-1">モンゴルの十二支話</font></a>
</td> </td>
</tr> </tr>
<tr> <tr>
<td height="50" bgcolor="#ECFFEC"> <td height="50" bgcolor="#ECFFEC">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /><b><font size="-1">きょうの日本民話</font></b><br /> <img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /><b><font size="-1">きょうの日本民話</font></b><br />
<br /> <br />
<a href="../../../../douwa/pc/minwa/01/01c.html"><font size="-1">仕事の取替えっこ</font></a> <a href="../../../../douwa/pc/minwa/01/01c.html"><font size="-1">仕事の取替えっこ</font></a>
</td> </td>

6
article_scraper/src/clean.rs
View file

@ -145,9 +145,7 @@ mod tests {
let url = let url =
Url::parse("https://blogs.gnome.org/tbernard/2023/07/26/rethinking-window-management/") Url::parse("https://blogs.gnome.org/tbernard/2023/07/26/rethinking-window-management/")
.unwrap(); .unwrap();
let res = clean_html_fragment(html, &url).unwrap(); _ = clean_html_fragment(html, &url).unwrap();
std::fs::write("/home/jeanluc/result.html", res.html).unwrap();
} }
#[test] #[test]
@ -156,7 +154,7 @@ mod tests {
let url = Url::parse("https://finshots.in").unwrap(); let url = Url::parse("https://finshots.in").unwrap();
let res = clean_html_fragment(html, &url).unwrap(); let res = clean_html_fragment(html, &url).unwrap();
assert_eq!(res.html.len(), 11989); assert!(res.html.len().abs_diff(12_000) < 200);
assert_eq!( assert_eq!(
res.thumbnail.as_deref(), res.thumbnail.as_deref(),
Some("https://cdn.finshots.app/images/2023/03/Design-8-Amul.jpg") Some("https://cdn.finshots.app/images/2023/03/Design-8-Amul.jpg")