misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-07 16:15:32 +02:00
Code Releases Activity

fix encoding

Browse source
This commit is contained in:
Jan Lukas Gernert 2023-04-26 07:44:10 +02:00
parent bd413a795c
commit dd958fe30f
1 changed files with 2 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
article_scraper/src/full_text_parser/mod.rs
View file

@ -381,6 +381,7 @@ impl FullTextParser {
if let Some(encoding) = Self::get_encoding_from_html(&lossy_string) { if let Some(encoding) = Self::get_encoding_from_html(&lossy_string) {
log::debug!("Encoding extracted from HTML: '{}'", encoding); log::debug!("Encoding extracted from HTML: '{}'", encoding);
if let Some(decoded_html) = Self::decode_html(&bytes, encoding) { if let Some(decoded_html) = Self::decode_html(&bytes, encoding) {
let decoded_html = decoded_html.replace(&format!("charset=\"{encoding}\""), "charset=\"utf-8\"");
return Ok(decoded_html); return Ok(decoded_html);
} }
} }
@ -388,6 +389,7 @@ impl FullTextParser {
if let Some(encoding) = Self::get_encoding_from_http_header(&headers) { if let Some(encoding) = Self::get_encoding_from_http_header(&headers) {
log::debug!("Encoding extracted from headers: '{}'", encoding); log::debug!("Encoding extracted from headers: '{}'", encoding);
if let Some(decoded_html) = Self::decode_html(&bytes, encoding) { if let Some(decoded_html) = Self::decode_html(&bytes, encoding) {
let decoded_html = decoded_html.replace(&format!("charset=\"{encoding}\""), "charset=\"utf-8\"");
return Ok(decoded_html); return Ok(decoded_html);
} }
} }