misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-08 00:19:59 +02:00
Code Releases Activity

update for newer deps

Browse source
This commit is contained in:
Jan Lukas Gernert 2020-04-28 02:51:30 +02:00
parent 1fbce6413d
commit 1fd7173eac
3 changed files with 13 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Cargo.toml
View file

@ -3,6 +3,9 @@ name = "article_scraper"
version = "1.0.0" version = "1.0.0"
authors = ["Jan Lukas Gernert <jangernert@gmail.com>"] authors = ["Jan Lukas Gernert <jangernert@gmail.com>"]
edition = "2018" edition = "2018"
license = "GPL-3.0-or-later"
description = "Scrap article contents from the web. Powered by fivefilters full text feed configurations."
repository = "https://gitlab.com/news-flash/article_scraper"
[dependencies] [dependencies]
failure = "0.1" failure = "0.1"

14
src/images/mod.rs
View file

@ -8,8 +8,6 @@ use libxml::tree::{Node, SaveOptions};
use libxml::xpath::Context; use libxml::xpath::Context;
use log::{debug, error}; use log::{debug, error};
use reqwest::{Client, Response}; use reqwest::{Client, Response};
use std;
use std::error::Error;
use url; use url;
mod error; mod error;
@ -103,7 +101,7 @@ impl ImageDownloader {
.send() .send()
.await .await
.map_err(|err| { .map_err(|err| {
error!("GET {} failed - {}", image_url.as_str(), err.description()); error!("GET {} failed - {}", image_url.as_str(), err);
err err
}) })
.context(ImageDownloadErrorKind::Http)?; .context(ImageDownloadErrorKind::Http)?;
@ -218,7 +216,7 @@ impl ImageDownloader {
.context(ImageDownloadErrorKind::ImageScale)?; .context(ImageDownloadErrorKind::ImageScale)?;
image image
.write_to(&mut original_image, image::ImageOutputFormat::PNG) .write_to(&mut original_image, image::ImageOutputFormat::Png)
.map_err(|err| { .map_err(|err| {
error!("Failed to save resized image to resize"); error!("Failed to save resized image to resize");
err err
@ -230,11 +228,11 @@ impl ImageDownloader {
image = image.resize( image = image.resize(
max_dimensions.0, max_dimensions.0,
max_dimensions.1, max_dimensions.1,
image::FilterType::Lanczos3, image::imageops::FilterType::Lanczos3,
); );
let mut resized_buf: Vec<u8> = Vec::new(); let mut resized_buf: Vec<u8> = Vec::new();
image image
.write_to(&mut resized_buf, image::ImageOutputFormat::PNG) .write_to(&mut resized_buf, image::ImageOutputFormat::Png)
.map_err(|err| { .map_err(|err| {
error!("Failed to save resized image to resize"); error!("Failed to save resized image to resize");
err err
@ -249,9 +247,13 @@ impl ImageDownloader {
fn get_image_dimensions(image: &image::DynamicImage) -> (u32, u32) { fn get_image_dimensions(image: &image::DynamicImage) -> (u32, u32) {
match image { match image {
image::DynamicImage::ImageLuma8(image) => (image.width(), image.height()), image::DynamicImage::ImageLuma8(image) => (image.width(), image.height()),
image::DynamicImage::ImageLuma16(image) => (image.width(), image.height()),
image::DynamicImage::ImageLumaA8(image) => (image.width(), image.height()), image::DynamicImage::ImageLumaA8(image) => (image.width(), image.height()),
image::DynamicImage::ImageLumaA16(image) => (image.width(), image.height()),
image::DynamicImage::ImageRgb8(image) => (image.width(), image.height()), image::DynamicImage::ImageRgb8(image) => (image.width(), image.height()),
image::DynamicImage::ImageRgba8(image) => (image.width(), image.height()), image::DynamicImage::ImageRgba8(image) => (image.width(), image.height()),
image::DynamicImage::ImageRgb16(image) => (image.width(), image.height()),
image::DynamicImage::ImageRgba16(image) => (image.width(), image.height()),
image::DynamicImage::ImageBgr8(image) => (image.width(), image.height()), image::DynamicImage::ImageBgr8(image) => (image.width(), image.height()),
image::DynamicImage::ImageBgra8(image) => (image.width(), image.height()), image::DynamicImage::ImageBgra8(image) => (image.width(), image.height()),
} }

9
src/lib.rs
View file

@ -17,7 +17,6 @@ use log::{debug, error, info, warn};
use regex; use regex;
use reqwest::{Client, Response}; use reqwest::{Client, Response};
use std::collections; use std::collections;
use std::error::Error;
use std::path::PathBuf; use std::path::PathBuf;
use std::str::FromStr; use std::str::FromStr;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
@ -66,11 +65,7 @@ impl ArticleScraper {
.send() .send()
.await .await
.map_err(|err| { .map_err(|err| {
error!( error!("Failed head request to: '{}' - '{}'", url.as_str(), err);
"Failed head request to: '{}' - '{}'",
url.as_str(),
err.description()
);
err err
}) })
.context(ScraperErrorKind::Http)?; .context(ScraperErrorKind::Http)?;
@ -273,7 +268,7 @@ impl ArticleScraper {
error!( error!(
"Downloading HTML failed: GET '{}' - '{}'", "Downloading HTML failed: GET '{}' - '{}'",
url.as_str(), url.as_str(),
err.description() err
); );
err err
}) })