misc-mirrors/article_scraper
1
0
Fork 0
mirror of https://gitlab.com/news-flash/article_scraper.git synced 2025-07-09 00:45:31 +02:00
Code Releases Activity

add support for header values: fixes golem test

Browse source
This commit is contained in:
Jan Lukas Gernert 2022-10-07 07:14:39 +02:00
parent 0e3553b647
commit 7b1b027c6d
5 changed files with 94 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

24
src/util.rs
View file

@ -1,5 +1,9 @@
use failure::ResultExt;
use reqwest::header::{HeaderMap, HeaderValue, HeaderName};
use tokio::fs::DirEntry;
use crate::{config::ConfigEntry, error::{ScraperErrorKind, ScraperError}};
pub struct Util;
impl Util {
@ -34,4 +38,24 @@ impl Util {
global_rule
}
}
pub fn generate_headers(site_specific_rule: Option<&ConfigEntry>, global_rule: &ConfigEntry) -> Result<HeaderMap, ScraperError> {
let mut headers = HeaderMap::new();
if let Some(config) = site_specific_rule {
for header in &config.header {
let name = HeaderName::from_bytes(header.name.as_bytes()).context(ScraperErrorKind::Config)?;
let value = header.value.parse::<HeaderValue>().context(ScraperErrorKind::Config)?;
headers.insert(name, value);
}
}
for header in &global_rule.header {
let name = HeaderName::from_bytes(header.name.as_bytes()).context(ScraperErrorKind::Config)?;
let value = header.value.parse::<HeaderValue>().context(ScraperErrorKind::Config)?;
headers.insert(name, value);
}
Ok(headers)
}
}