1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-08 00:19:59 +02:00

only go for single page link if xpath res isn't empty

This commit is contained in:
Jan Lukas Gernert 2020-01-27 01:54:37 +01:00
parent e58acf828c
commit afe661fe6c

View file

@ -171,12 +171,14 @@ impl ArticleScraper {
xpath_single_page_link
);
if let Ok(result) = xpath_ctx.findvalue(&xpath_single_page_link, None) {
// parse again with single page url
debug!("Single page link found '{}'", result);
let single_page_url = url::Url::parse(&result).context(ScraperErrorKind::Url)?;
return self
.parse_single_page(article, &single_page_url, root, config)
.await;
if !result.trim().is_empty() {
// parse again with single page url
debug!("Single page link found '{}'", result);
let single_page_url = url::Url::parse(&result).context(ScraperErrorKind::Url)?;
return self
.parse_single_page(article, &single_page_url, root, config)
.await;
}
}
}