mirror of
https://gitlab.com/news-flash/article_scraper.git
synced 2025-07-08 00:19:59 +02:00
only go for single page link if xpath res isn't empty
This commit is contained in:
parent
e58acf828c
commit
afe661fe6c
1 changed files with 8 additions and 6 deletions
14
src/lib.rs
14
src/lib.rs
|
@ -171,12 +171,14 @@ impl ArticleScraper {
|
|||
xpath_single_page_link
|
||||
);
|
||||
if let Ok(result) = xpath_ctx.findvalue(&xpath_single_page_link, None) {
|
||||
// parse again with single page url
|
||||
debug!("Single page link found '{}'", result);
|
||||
let single_page_url = url::Url::parse(&result).context(ScraperErrorKind::Url)?;
|
||||
return self
|
||||
.parse_single_page(article, &single_page_url, root, config)
|
||||
.await;
|
||||
if !result.trim().is_empty() {
|
||||
// parse again with single page url
|
||||
debug!("Single page link found '{}'", result);
|
||||
let single_page_url = url::Url::parse(&result).context(ScraperErrorKind::Url)?;
|
||||
return self
|
||||
.parse_single_page(article, &single_page_url, root, config)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue