misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-08 00:19:59 +02:00
Code Releases Activity

only go for single page link if xpath res isn't empty

Browse source
This commit is contained in:
Jan Lukas Gernert 2020-01-27 01:54:37 +01:00
parent e58acf828c
commit afe661fe6c
1 changed files with 8 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/lib.rs
View file

@ -171,12 +171,14 @@ impl ArticleScraper {
xpath_single_page_link
);
if let Ok(result) = xpath_ctx.findvalue(&xpath_single_page_link, None) {
// parse again with single page url
debug!("Single page link found '{}'", result);
let single_page_url = url::Url::parse(&result).context(ScraperErrorKind::Url)?;
return self
.parse_single_page(article, &single_page_url, root, config)
.await;
if !result.trim().is_empty() {
// parse again with single page url
debug!("Single page link found '{}'", result);
let single_page_url = url::Url::parse(&result).context(ScraperErrorKind::Url)?;
return self
.parse_single_page(article, &single_page_url, root, config)
.await;
}
}
}