misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-10 01:15:31 +02:00
Code Releases Activity

command to use readability extractor

Browse source
This commit is contained in:
Jan Lukas Gernert 2023-04-07 11:51:14 +02:00
parent 063996d62f
commit d978059709
5 changed files with 65 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

4
article_scraper/src/full_text_parser/mod.rs
View file

@ -9,7 +9,7 @@ mod tests;
use self::config::{ConfigCollection, ConfigEntry};
use self::error::FullTextParserError;
use self::readability::Readability;
pub use self::readability::Readability;
use crate::article::Article;
use crate::constants;
use crate::util::Util;
@ -269,7 +269,7 @@ impl FullTextParser {
Ok(())
}
async fn download(
pub async fn download(
url: &url::Url,
client: &Client,
headers: HeaderMap,

2
article_scraper/src/full_text_parser/readability/mod.rs
View file

@ -14,7 +14,7 @@ use crate::{constants, util::Util};
pub struct Readability;
impl Readability {
async fn extract_from_str(
pub async fn extract_from_str(
html: &str,
base_url: Option<url::Url>,
) -> Result<String, FullTextParserError> {

3
article_scraper/src/lib.rs
View file

@ -7,7 +7,8 @@ mod util;
use article::Article;
use error::ScraperError;
use full_text_parser::FullTextParser;
pub use full_text_parser::FullTextParser;
pub use full_text_parser::Readability;
use images::ImageDownloader;
use reqwest::Client;
use std::path::Path;