misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-08 00:19:59 +02:00
Code Releases Activity

only go for single page link if xpath res isn't empty

Browse source
This commit is contained in:
Jan Lukas Gernert 2020-01-27 01:54:37 +01:00
parent e58acf828c
commit afe661fe6c
1 changed files with 8 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/lib.rs
View file

@ -171,6 +171,7 @@ impl ArticleScraper {
xpath_single_page_link
);
if let Ok(result) = xpath_ctx.findvalue(&xpath_single_page_link, None) {
if !result.trim().is_empty() {
// parse again with single page url
debug!("Single page link found '{}'", result);
let single_page_url = url::Url::parse(&result).context(ScraperErrorKind::Url)?;
@ -179,6 +180,7 @@ impl ArticleScraper {
.await;
}
}
}
ArticleScraper::extract_metadata(&xpath_ctx, config, article);
ArticleScraper::strip_junk(&xpath_ctx, config, &url);