misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-07 16:15:32 +02:00
Code Releases Activity

fix negative regex & fmt

Browse source
This commit is contained in:
Jan Lukas Gernert 2023-03-12 11:42:37 +01:00
parent 1e71aa2bfb
commit 58a799b096
2 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/constants.rs
View file

@ -41,7 +41,7 @@ pub static POSITIVE: Lazy<Regex> =
.expect("POSITIVE regex")
});
pub static NEGATIVE: Lazy<Regex> = Lazy::new(|| {
Regex::new(r#"-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget"#).expect("NEGATIVE regex")
RegexBuilder::new(r#"-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget"#).case_insensitive(true).build().expect("NEGATIVE regex")
});
pub static TITLE_SEPARATOR: Lazy<Regex> =

4
src/full_text_parser/readability/mod.rs
View file

@ -355,7 +355,9 @@ impl Readability {
// The scores shouldn't get too low.
let score_threshold = last_score / 3.0;
while parent_of_top_candidate.is_some() && !Util::has_tag_name(parent_of_top_candidate.as_ref(), "BODY") {
while parent_of_top_candidate.is_some()
&& !Util::has_tag_name(parent_of_top_candidate.as_ref(), "BODY")
{
if parent_of_top_candidate
.as_ref()
.map(|n| Self::get_content_score(n).is_none())