misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-08 16:40:00 +02:00
Code Releases Activity

move stuff around

Browse source
This commit is contained in:
Jan Lukas Gernert 2022-12-13 08:54:57 +01:00
parent 90383545e0
commit c08f5afa5d
6 changed files with 84 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

19
src/lib.rs
View file

@ -2,20 +2,17 @@ mod article;
mod error;
mod full_text_parser;
pub mod images;
mod readability;
mod util;
use article::Article;
use error::ScraperError;
use full_text_parser::FullTextParser;
use images::ImageDownloader;
use readability::Readability;
use reqwest::Client;
use std::path::Path;
pub struct ArticleScraper {
full_text_parser: FullTextParser,
readability: Readability,
image_downloader: ImageDownloader,
}
@ -23,7 +20,6 @@ impl ArticleScraper {
pub async fn new(user_configs: Option<&Path>) -> Self {
Self {
full_text_parser: FullTextParser::new(user_configs).await,
readability: Readability::new(),
image_downloader: ImageDownloader::new((2048, 2048)),
}
}
@ -37,13 +33,14 @@ impl ArticleScraper {
let res = self.full_text_parser.parse(url, client).await;
if download_images {
// if let Err(error) = self
// .image_downloader
// .download_images_from_context(&context, client)
// .await
// {
// log::error!("Downloading images failed: '{}'", error);
// }
if let Ok(res) = res {
if let Some(document) = res.document.as_ref() {
let _image_res = self
.image_downloader
.download_images_from_document(document, client)
.await;
}
}
}
unimplemented!()