misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-07 16:15:32 +02:00
Code Releases Activity

TIL: map_err

Browse source
This commit is contained in:
Jan Lukas Gernert 2018-08-31 16:49:58 +02:00
parent 5beb25a575
commit fab4306ed9
3 changed files with 60 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

56
src/images/mod.rs
View file

@ -36,21 +36,15 @@ impl ImageDownloader {
pub fn download_images_from_string(&self, html: &str, article_url: &url::Url) -> Result<String, ImageDownloadError> {
let parser = Parser::default_html();
let doc = match parser.parse_string(html) {
Ok(doc) => doc,
Err(_) => {
error!("Failed to parse HTML string");
return Err(ImageDownloadErrorKind::HtmlParse)?
}
};
let doc = parser.parse_string(html).map_err(|_| {
error!("Failed to parse HTML string");
ImageDownloadErrorKind::HtmlParse
})?;
let xpath_ctx = match Context::new(&doc) {
Ok(context) => context,
Err(_) => {
error!("Failed to create xpath context for document");
return Err(ImageDownloadErrorKind::HtmlParse)?
}
};
let xpath_ctx = Context::new(&doc).map_err(|()| {
error!("Failed to create xpath context for document");
ImageDownloadErrorKind::HtmlParse
})?;
self.download_images_from_context(&xpath_ctx, article_url)?;
@ -109,13 +103,11 @@ impl ImageDownloader {
fn save_image(&self, image_url: &url::Url, article_url: &url::Url) -> Result<PathBuf, ImageDownloadError> {
let mut response = match self.client.get(image_url.clone()).send() {
Ok(response) => response,
Err(error) => {
error!("GET {} failed - {}", image_url.as_str(), error.description());
Err(error).context(ImageDownloadErrorKind::Http)?
}
};
let mut response = self.client.get(image_url.clone()).send().map_err(|err| {
error!("GET {} failed - {}", image_url.as_str(), err.description());
err
}).context(ImageDownloadErrorKind::Http)?;
let content_type = ImageDownloader::check_image_content_type(&response)?;
if let Some(host) = article_url.host_str() {
@ -126,13 +118,10 @@ impl ImageDownloader {
if let Ok(()) = std::fs::create_dir_all(&path) {
let file_name = ImageDownloader::extract_image_name(image_url, content_type)?;
let path = path.join(file_name);
let mut image_buffer = match std::fs::File::create(&path) {
Ok(buffer) => buffer,
Err(error) => {
error!("Failed to create file {}", path.display());
Err(error).context(ImageDownloadErrorKind::IO)?
}
};
let mut image_buffer = std::fs::File::create(&path).map_err(|err| {
error!("Failed to create file {}", path.display());
err
}).context(ImageDownloadErrorKind::IO)?;
response.copy_to(&mut image_buffer).context(ImageDownloadErrorKind::IO)?;
let path = std::fs::canonicalize(&path).context(ImageDownloadErrorKind::IO)?;
@ -252,13 +241,10 @@ impl ImageDownloader {
fn scale_image(image_path: &PathBuf, max_width: u32, max_height: u32) -> Result<PathBuf, ImageDownloadError> {
let image = match image::open(image_path) {
Ok(image) => image,
Err(error) => {
error!("Failed to open image to resize: {:?}", image_path);
return Err(error).context(ImageDownloadErrorKind::ImageScale)?
}
};
let image = image::open(image_path).map_err(|err| {
error!("Failed to open image to resize: {:?}", image_path);
err
}).context(ImageDownloadErrorKind::ImageScale)?;
let image = image.resize(max_width, max_height, image::FilterType::Lanczos3);
if let Some(file_name) = image_path.file_name() {

65
src/lib.rs
View file

@ -69,15 +69,12 @@ impl ArticleScraper {
pub fn parse(&self, url: url::Url) -> Result<Article, ScraperError> {
info!("Scraping article: {}", url.as_str());
// do a HEAD request to url
let response = match self.client.head(url.clone()).send() {
Ok(response) => response,
Err(error) => {
error!("Failed head request to: {} - {}", url.as_str(), error.description());
Err(error).context(ScraperErrorKind::Http)?
}
};
let response = self.client.head(url.clone()).send()
.map_err(|err| {
error!("Failed head request to: {} - {}", url.as_str(), err.description());
err
})
.context(ScraperErrorKind::Http)?;
// check if url redirects and we need to pick up the new url
let mut url = url;
@ -102,16 +99,13 @@ impl ArticleScraper {
html: None,
};
// create empty document to hold the content
let mut document = match Document::new() {
Ok(doc) => doc,
Err(()) => return Err(ScraperErrorKind::Xml)?
};
let mut document = Document::new().map_err(|()| {
ScraperErrorKind::Xml
})?;
let mut root = match Node::new("article", None, &document) {
Ok(root) => root,
Err(()) => return Err(ScraperErrorKind::Xml)?
};
let mut root = Node::new("article", None, &document).map_err(|()| {
ScraperErrorKind::Xml
})?;
document.set_root_element(&root);
@ -119,13 +113,10 @@ impl ArticleScraper {
self.parse_first_page(&mut article, &url, &mut root, config)?;
let context = match Context::new(&document) {
Ok(context) => context,
Err(_) => {
error!("Failed to create xpath context for extracted article");
return Err(ScraperErrorKind::Xml)?
}
};
let context = Context::new(&document).map_err(|()| {
error!("Failed to create xpath context for extracted article");
ScraperErrorKind::Xml
})?;
if let Err(error) = ArticleScraper::prevent_self_closing_tags(&context) {
error!("Preventing self closing tags failed - {}", error);
@ -197,13 +188,12 @@ impl ArticleScraper {
fn download(url: &url::Url, client: &reqwest::Client) -> Result<String, ScraperError> {
let mut response = match client.get(url.as_str()).send() {
Ok(response) => response,
Err(error) => {
error!("Downloading HTML failed: GET {} - {}", url.as_str(), error.description());
return Err(error).context(ScraperErrorKind::Http)?
}
};
let mut response = client.get(url.as_str()).send()
.map_err(|err| {
error!("Downloading HTML failed: GET {} - {}", url.as_str(), err.description());
err
})
.context(ScraperErrorKind::Http)?;
if response.status().is_success() {
let text = response.text().context(ScraperErrorKind::Http)?;
@ -386,13 +376,10 @@ impl ArticleScraper {
if let Ok(()) = node.set_property("width", "100%") {
if let Ok(()) = node.remove_property("height") {
node.unlink();
match video_wrapper.add_child(&mut node) {
Ok(_) => continue,
Err(_) => {
error!("Failed to add iframe as child of video wrapper <div>");
return Err(ScraperErrorKind::Xml)?
}
}
video_wrapper.add_child(&mut node).map_err(|_| {
error!("Failed to add iframe as child of video wrapper <div>");
ScraperErrorKind::Xml
})?;
}
}
}

35
src/macros.rs
View file

@ -11,21 +11,15 @@ macro_rules! parse_html {
// parse html
let parser = Parser::default_html();
let doc = match parser.parse_string($html.as_str()) {
Ok(doc) => doc,
Err(_) => {
error!("Parsing HTML failed for downloaded HTML");
return Err(ScraperErrorKind::Xml)?
}
};
let $xpath_ctx = match Context::new(&doc) {
Ok(context) => context,
Err(_) => {
error!("Creating xpath context failed for downloaded HTML");
return Err(ScraperErrorKind::Xml)?
}
};
let doc = parser.parse_string($html.as_str()).map_err(|err| {
error!("Parsing HTML failed for downloaded HTML {:?}", err);
ScraperErrorKind::Xml
})?;
let $xpath_ctx = Context::new(&doc).map_err(|()| {
error!("Creating xpath context failed for downloaded HTML");
ScraperErrorKind::Xml
})?;
};
}
@ -35,13 +29,10 @@ macro_rules! evaluate_xpath {
$xpath: ident,
$node_vec: ident
) => {
let res = match $context.evaluate($xpath) {
Ok(result) => result,
Err(_) => {
error!("Evaluation of xpath {} yielded no results", $xpath);
return Err(ScraperErrorKind::Xml)?
}
};
let res = $context.evaluate($xpath).map_err(|()| {
error!("Evaluation of xpath {} yielded no results", $xpath);
ScraperErrorKind::Xml
})?;
let $node_vec = res.get_nodes_as_vec();
};