mirror of
https://gitlab.com/news-flash/article_scraper.git
synced 2025-07-07 16:15:32 +02:00
TIL: map_err
This commit is contained in:
parent
5beb25a575
commit
fab4306ed9
3 changed files with 60 additions and 96 deletions
|
@ -36,21 +36,15 @@ impl ImageDownloader {
|
|||
pub fn download_images_from_string(&self, html: &str, article_url: &url::Url) -> Result<String, ImageDownloadError> {
|
||||
|
||||
let parser = Parser::default_html();
|
||||
let doc = match parser.parse_string(html) {
|
||||
Ok(doc) => doc,
|
||||
Err(_) => {
|
||||
let doc = parser.parse_string(html).map_err(|_| {
|
||||
error!("Failed to parse HTML string");
|
||||
return Err(ImageDownloadErrorKind::HtmlParse)?
|
||||
}
|
||||
};
|
||||
ImageDownloadErrorKind::HtmlParse
|
||||
})?;
|
||||
|
||||
let xpath_ctx = match Context::new(&doc) {
|
||||
Ok(context) => context,
|
||||
Err(_) => {
|
||||
let xpath_ctx = Context::new(&doc).map_err(|()| {
|
||||
error!("Failed to create xpath context for document");
|
||||
return Err(ImageDownloadErrorKind::HtmlParse)?
|
||||
}
|
||||
};
|
||||
ImageDownloadErrorKind::HtmlParse
|
||||
})?;
|
||||
|
||||
self.download_images_from_context(&xpath_ctx, article_url)?;
|
||||
|
||||
|
@ -109,13 +103,11 @@ impl ImageDownloader {
|
|||
|
||||
fn save_image(&self, image_url: &url::Url, article_url: &url::Url) -> Result<PathBuf, ImageDownloadError> {
|
||||
|
||||
let mut response = match self.client.get(image_url.clone()).send() {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
error!("GET {} failed - {}", image_url.as_str(), error.description());
|
||||
Err(error).context(ImageDownloadErrorKind::Http)?
|
||||
}
|
||||
};
|
||||
let mut response = self.client.get(image_url.clone()).send().map_err(|err| {
|
||||
error!("GET {} failed - {}", image_url.as_str(), err.description());
|
||||
err
|
||||
}).context(ImageDownloadErrorKind::Http)?;
|
||||
|
||||
let content_type = ImageDownloader::check_image_content_type(&response)?;
|
||||
|
||||
if let Some(host) = article_url.host_str() {
|
||||
|
@ -126,13 +118,10 @@ impl ImageDownloader {
|
|||
if let Ok(()) = std::fs::create_dir_all(&path) {
|
||||
let file_name = ImageDownloader::extract_image_name(image_url, content_type)?;
|
||||
let path = path.join(file_name);
|
||||
let mut image_buffer = match std::fs::File::create(&path) {
|
||||
Ok(buffer) => buffer,
|
||||
Err(error) => {
|
||||
let mut image_buffer = std::fs::File::create(&path).map_err(|err| {
|
||||
error!("Failed to create file {}", path.display());
|
||||
Err(error).context(ImageDownloadErrorKind::IO)?
|
||||
}
|
||||
};
|
||||
err
|
||||
}).context(ImageDownloadErrorKind::IO)?;
|
||||
|
||||
response.copy_to(&mut image_buffer).context(ImageDownloadErrorKind::IO)?;
|
||||
let path = std::fs::canonicalize(&path).context(ImageDownloadErrorKind::IO)?;
|
||||
|
@ -252,13 +241,10 @@ impl ImageDownloader {
|
|||
|
||||
fn scale_image(image_path: &PathBuf, max_width: u32, max_height: u32) -> Result<PathBuf, ImageDownloadError> {
|
||||
|
||||
let image = match image::open(image_path) {
|
||||
Ok(image) => image,
|
||||
Err(error) => {
|
||||
let image = image::open(image_path).map_err(|err| {
|
||||
error!("Failed to open image to resize: {:?}", image_path);
|
||||
return Err(error).context(ImageDownloadErrorKind::ImageScale)?
|
||||
}
|
||||
};
|
||||
err
|
||||
}).context(ImageDownloadErrorKind::ImageScale)?;
|
||||
let image = image.resize(max_width, max_height, image::FilterType::Lanczos3);
|
||||
|
||||
if let Some(file_name) = image_path.file_name() {
|
||||
|
|
61
src/lib.rs
61
src/lib.rs
|
@ -69,15 +69,12 @@ impl ArticleScraper {
|
|||
pub fn parse(&self, url: url::Url) -> Result<Article, ScraperError> {
|
||||
|
||||
info!("Scraping article: {}", url.as_str());
|
||||
|
||||
// do a HEAD request to url
|
||||
let response = match self.client.head(url.clone()).send() {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
error!("Failed head request to: {} - {}", url.as_str(), error.description());
|
||||
Err(error).context(ScraperErrorKind::Http)?
|
||||
}
|
||||
};
|
||||
let response = self.client.head(url.clone()).send()
|
||||
.map_err(|err| {
|
||||
error!("Failed head request to: {} - {}", url.as_str(), err.description());
|
||||
err
|
||||
})
|
||||
.context(ScraperErrorKind::Http)?;
|
||||
|
||||
// check if url redirects and we need to pick up the new url
|
||||
let mut url = url;
|
||||
|
@ -102,16 +99,13 @@ impl ArticleScraper {
|
|||
html: None,
|
||||
};
|
||||
|
||||
// create empty document to hold the content
|
||||
let mut document = match Document::new() {
|
||||
Ok(doc) => doc,
|
||||
Err(()) => return Err(ScraperErrorKind::Xml)?
|
||||
};
|
||||
let mut document = Document::new().map_err(|()| {
|
||||
ScraperErrorKind::Xml
|
||||
})?;
|
||||
|
||||
let mut root = match Node::new("article", None, &document) {
|
||||
Ok(root) => root,
|
||||
Err(()) => return Err(ScraperErrorKind::Xml)?
|
||||
};
|
||||
let mut root = Node::new("article", None, &document).map_err(|()| {
|
||||
ScraperErrorKind::Xml
|
||||
})?;
|
||||
|
||||
document.set_root_element(&root);
|
||||
|
||||
|
@ -119,13 +113,10 @@ impl ArticleScraper {
|
|||
|
||||
self.parse_first_page(&mut article, &url, &mut root, config)?;
|
||||
|
||||
let context = match Context::new(&document) {
|
||||
Ok(context) => context,
|
||||
Err(_) => {
|
||||
let context = Context::new(&document).map_err(|()| {
|
||||
error!("Failed to create xpath context for extracted article");
|
||||
return Err(ScraperErrorKind::Xml)?
|
||||
}
|
||||
};
|
||||
ScraperErrorKind::Xml
|
||||
})?;
|
||||
|
||||
if let Err(error) = ArticleScraper::prevent_self_closing_tags(&context) {
|
||||
error!("Preventing self closing tags failed - {}", error);
|
||||
|
@ -197,13 +188,12 @@ impl ArticleScraper {
|
|||
|
||||
fn download(url: &url::Url, client: &reqwest::Client) -> Result<String, ScraperError> {
|
||||
|
||||
let mut response = match client.get(url.as_str()).send() {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
error!("Downloading HTML failed: GET {} - {}", url.as_str(), error.description());
|
||||
return Err(error).context(ScraperErrorKind::Http)?
|
||||
}
|
||||
};
|
||||
let mut response = client.get(url.as_str()).send()
|
||||
.map_err(|err| {
|
||||
error!("Downloading HTML failed: GET {} - {}", url.as_str(), err.description());
|
||||
err
|
||||
})
|
||||
.context(ScraperErrorKind::Http)?;
|
||||
|
||||
if response.status().is_success() {
|
||||
let text = response.text().context(ScraperErrorKind::Http)?;
|
||||
|
@ -386,13 +376,10 @@ impl ArticleScraper {
|
|||
if let Ok(()) = node.set_property("width", "100%") {
|
||||
if let Ok(()) = node.remove_property("height") {
|
||||
node.unlink();
|
||||
match video_wrapper.add_child(&mut node) {
|
||||
Ok(_) => continue,
|
||||
Err(_) => {
|
||||
video_wrapper.add_child(&mut node).map_err(|_| {
|
||||
error!("Failed to add iframe as child of video wrapper <div>");
|
||||
return Err(ScraperErrorKind::Xml)?
|
||||
}
|
||||
}
|
||||
ScraperErrorKind::Xml
|
||||
})?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -11,21 +11,15 @@ macro_rules! parse_html {
|
|||
|
||||
// parse html
|
||||
let parser = Parser::default_html();
|
||||
let doc = match parser.parse_string($html.as_str()) {
|
||||
Ok(doc) => doc,
|
||||
Err(_) => {
|
||||
error!("Parsing HTML failed for downloaded HTML");
|
||||
return Err(ScraperErrorKind::Xml)?
|
||||
}
|
||||
};
|
||||
let doc = parser.parse_string($html.as_str()).map_err(|err| {
|
||||
error!("Parsing HTML failed for downloaded HTML {:?}", err);
|
||||
ScraperErrorKind::Xml
|
||||
})?;
|
||||
|
||||
let $xpath_ctx = match Context::new(&doc) {
|
||||
Ok(context) => context,
|
||||
Err(_) => {
|
||||
let $xpath_ctx = Context::new(&doc).map_err(|()| {
|
||||
error!("Creating xpath context failed for downloaded HTML");
|
||||
return Err(ScraperErrorKind::Xml)?
|
||||
}
|
||||
};
|
||||
ScraperErrorKind::Xml
|
||||
})?;
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -35,13 +29,10 @@ macro_rules! evaluate_xpath {
|
|||
$xpath: ident,
|
||||
$node_vec: ident
|
||||
) => {
|
||||
let res = match $context.evaluate($xpath) {
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
let res = $context.evaluate($xpath).map_err(|()| {
|
||||
error!("Evaluation of xpath {} yielded no results", $xpath);
|
||||
return Err(ScraperErrorKind::Xml)?
|
||||
}
|
||||
};
|
||||
ScraperErrorKind::Xml
|
||||
})?;
|
||||
|
||||
let $node_vec = res.get_nodes_as_vec();
|
||||
};
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue