mirror of
https://gitlab.com/news-flash/article_scraper.git
synced 2025-07-07 08:05:31 +02:00
better error messages
This commit is contained in:
parent
0978335d3b
commit
b92500fca2
2 changed files with 50 additions and 52 deletions
|
@ -6,10 +6,10 @@ use thiserror::Error;
|
|||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum ScraperError {
|
||||
#[error("")]
|
||||
#[error("Configerror {0}")]
|
||||
Config(#[from] ConfigError),
|
||||
#[error("")]
|
||||
#[error("ImageDownloadError {0}")]
|
||||
Image(#[from] ImageDownloadError),
|
||||
#[error("")]
|
||||
#[error("FullTextParserError {0}")]
|
||||
Scrap(#[from] FullTextParserError),
|
||||
}
|
||||
|
|
|
@ -354,18 +354,19 @@ impl FullTextParser {
|
|||
.send()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
log::error!(
|
||||
"Downloading HTML failed: GET '{}' - '{}'",
|
||||
url.as_str(),
|
||||
err
|
||||
);
|
||||
log::error!("Downloading HTML failed: GET '{url}' - '{err}'");
|
||||
FullTextParserError::Http
|
||||
})?;
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn get_body(response: Response) -> Result<String, FullTextParserError> {
|
||||
if response.status().is_success() {
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
log::error!("status code: {status}");
|
||||
return Err(FullTextParserError::Http);
|
||||
}
|
||||
|
||||
let headers = response.headers().clone();
|
||||
let bytes = response
|
||||
.bytes()
|
||||
|
@ -375,14 +376,14 @@ impl FullTextParser {
|
|||
match from_utf8(&bytes) {
|
||||
Ok(utf8_str) => {
|
||||
log::debug!("Valid utf-8 string");
|
||||
return Ok(utf8_str.into());
|
||||
Ok(utf8_str.into())
|
||||
}
|
||||
Err(error) => {
|
||||
log::debug!("Invalid utf-8 string");
|
||||
let lossy_string = std::string::String::from_utf8_lossy(&bytes);
|
||||
|
||||
if let Some(encoding) = Self::get_encoding_from_html(&lossy_string) {
|
||||
log::debug!("Encoding extracted from HTML: '{}'", encoding);
|
||||
log::debug!("Encoding extracted from HTML: '{encoding}'");
|
||||
if let Some(decoded_html) = Self::decode_html(&bytes, encoding) {
|
||||
let decoded_html = decoded_html.replacen(
|
||||
&format!("charset=\"{encoding}\""),
|
||||
|
@ -394,7 +395,7 @@ impl FullTextParser {
|
|||
}
|
||||
|
||||
if let Some(encoding) = Self::get_encoding_from_http_header(&headers) {
|
||||
log::debug!("Encoding extracted from headers: '{}'", encoding);
|
||||
log::debug!("Encoding extracted from headers: '{encoding}'");
|
||||
if let Some(decoded_html) = Self::decode_html(&bytes, encoding) {
|
||||
let decoded_html = decoded_html.replacen(
|
||||
&format!("charset=\"{encoding}\""),
|
||||
|
@ -405,14 +406,11 @@ impl FullTextParser {
|
|||
}
|
||||
}
|
||||
|
||||
return Err(FullTextParserError::Utf8(error));
|
||||
Err(FullTextParserError::Utf8(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(FullTextParserError::Http)
|
||||
}
|
||||
|
||||
pub async fn download(
|
||||
url: &url::Url,
|
||||
client: &Client,
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue