misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-07 16:15:32 +02:00
Code Releases Activity

check if final url differs from original even without redirect status

Browse source
This commit is contained in:
Jan Lukas Gernert 2020-06-06 05:18:25 +02:00
parent 3bb8485f40
commit a42ececb2a
1 changed files with 8 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/lib.rs
View file

@ -69,11 +69,12 @@ impl ArticleScraper {
.context(ScraperErrorKind::Http)?; .context(ScraperErrorKind::Http)?;
// check if url redirects and we need to pick up the new url // check if url redirects and we need to pick up the new url
let mut url = url; let url = if let Some(new_url) = ArticleScraper::check_redirect(&response, &url) {
if let Some(new_url) = ArticleScraper::check_redirect(&response) {
debug!("Url '{}' redirects to '{}'", url.as_str(), new_url.as_str()); debug!("Url '{}' redirects to '{}'", url.as_str(), new_url.as_str());
url = new_url; new_url
} } else {
url
};
// check if we are dealing with text/html // check if we are dealing with text/html
if !ArticleScraper::check_content_type(&response)? { if !ArticleScraper::check_content_type(&response)? {
@ -381,10 +382,12 @@ impl ArticleScraper {
Err(ScraperErrorKind::Http.into()) Err(ScraperErrorKind::Http.into())
} }
fn check_redirect(response: &Response) -> Option<url::Url> { fn check_redirect(response: &Response, original_url: &url::Url) -> Option<url::Url> {
if response.status() == reqwest::StatusCode::PERMANENT_REDIRECT { if response.status() == reqwest::StatusCode::PERMANENT_REDIRECT {
debug!("Article url redirects to '{}'", response.url().as_str()); debug!("Article url redirects to '{}'", response.url().as_str());
return Some(response.url().clone()); return Some(response.url().clone());
} else if response.url() != original_url {
return Some(response.url().clone());
} }
None None