misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-07 16:15:32 +02:00
Code Releases Activity

better error messages

Browse source
This commit is contained in:
Jan Lukas Gernert 2025-04-05 15:45:41 +02:00
parent 0978335d3b
commit b92500fca2
2 changed files with 50 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

6
article_scraper/src/error.rs
View file

@ -6,10 +6,10 @@ use thiserror::Error;
#[derive(Error, Debug)] #[derive(Error, Debug)]
pub enum ScraperError { pub enum ScraperError {
#[error("")] #[error("Configerror {0}")]
Config(#[from] ConfigError), Config(#[from] ConfigError),
#[error("")] #[error("ImageDownloadError {0}")]
Image(#[from] ImageDownloadError), Image(#[from] ImageDownloadError),
#[error("")] #[error("FullTextParserError {0}")]
Scrap(#[from] FullTextParserError), Scrap(#[from] FullTextParserError),
} }

24
article_scraper/src/full_text_parser/mod.rs
View file

@ -354,18 +354,19 @@ impl FullTextParser {
.send() .send()
.await .await
.map_err(|err| { .map_err(|err| {
log::error!( log::error!("Downloading HTML failed: GET '{url}' - '{err}'");
"Downloading HTML failed: GET '{}' - '{}'",
url.as_str(),
err
);
FullTextParserError::Http FullTextParserError::Http
})?; })?;
Ok(response) Ok(response)
} }
async fn get_body(response: Response) -> Result<String, FullTextParserError> { async fn get_body(response: Response) -> Result<String, FullTextParserError> {
if response.status().is_success() { let status = response.status();
if !status.is_success() {
log::error!("status code: {status}");
return Err(FullTextParserError::Http);
}
let headers = response.headers().clone(); let headers = response.headers().clone();
let bytes = response let bytes = response
.bytes() .bytes()
@ -375,14 +376,14 @@ impl FullTextParser {
match from_utf8(&bytes) { match from_utf8(&bytes) {
Ok(utf8_str) => { Ok(utf8_str) => {
log::debug!("Valid utf-8 string"); log::debug!("Valid utf-8 string");
return Ok(utf8_str.into()); Ok(utf8_str.into())
} }
Err(error) => { Err(error) => {
log::debug!("Invalid utf-8 string"); log::debug!("Invalid utf-8 string");
let lossy_string = std::string::String::from_utf8_lossy(&bytes); let lossy_string = std::string::String::from_utf8_lossy(&bytes);
if let Some(encoding) = Self::get_encoding_from_html(&lossy_string) { if let Some(encoding) = Self::get_encoding_from_html(&lossy_string) {
log::debug!("Encoding extracted from HTML: '{}'", encoding); log::debug!("Encoding extracted from HTML: '{encoding}'");
if let Some(decoded_html) = Self::decode_html(&bytes, encoding) { if let Some(decoded_html) = Self::decode_html(&bytes, encoding) {
let decoded_html = decoded_html.replacen( let decoded_html = decoded_html.replacen(
&format!("charset=\"{encoding}\""), &format!("charset=\"{encoding}\""),
@ -394,7 +395,7 @@ impl FullTextParser {
} }
if let Some(encoding) = Self::get_encoding_from_http_header(&headers) { if let Some(encoding) = Self::get_encoding_from_http_header(&headers) {
log::debug!("Encoding extracted from headers: '{}'", encoding); log::debug!("Encoding extracted from headers: '{encoding}'");
if let Some(decoded_html) = Self::decode_html(&bytes, encoding) { if let Some(decoded_html) = Self::decode_html(&bytes, encoding) {
let decoded_html = decoded_html.replacen( let decoded_html = decoded_html.replacen(
&format!("charset=\"{encoding}\""), &format!("charset=\"{encoding}\""),
@ -405,14 +406,11 @@ impl FullTextParser {
} }
} }
return Err(FullTextParserError::Utf8(error)); Err(FullTextParserError::Utf8(error))
} }
} }
} }
Err(FullTextParserError::Http)
}
pub async fn download( pub async fn download(
url: &url::Url, url: &url::Url,
client: &Client, client: &Client,