frickel

Cargo.toml

@@ -7,7 +7,7 @@ edition = "2018"
 [dependencies]
 failure = "0.1"
 libxml = { git = "https://github.com/KWARC/rust-libxml.git" }
-reqwest = { version = "0.10.0-alpha.1", features = ["json"] }
+reqwest = { version = "0.10.0-alpha.2", features = ["json"] }
 tokio = { version = "=0.2.0-alpha.6" }
 url = "2.1"
 regex = "1.3"

src/images/mod.rs

@@ -16,7 +16,7 @@ use self::error::{ImageDownloadError, ImageDownloadErrorKind};
 use base64;
 use std;
 use image;
-use super::ScraperErrorKind;
+use crate::ArticleScraper;
 
 mod error;
 
@@ -63,7 +63,8 @@ impl ImageDownloader {
 
     pub async fn download_images_from_context(&self, context: &Context) -> Result<(), ImageDownloadError> {
         let xpath = "//img";
-        evaluate_xpath!(context, xpath, node_vec);
+        let node_vec = ArticleScraper::evaluate_xpath(context, xpath, false)
+            .context(ImageDownloadErrorKind::HtmlParse)?;
         for mut node in node_vec {
             if let Some(url) = node.get_property("src") {
                 if !url.starts_with("data:") {

src/lib.rs

@@ -1,5 +1,3 @@
-#[macro_use]
-mod macros;
 mod config;
 mod error;
 mod article;
@@ -147,11 +145,10 @@ impl ArticleScraper {
 
     async fn parse_pages(&self, article: &mut Article, url: &url::Url, root: &mut Node, config: &GrabberConfig) -> Result<(), ScraperError> {
 
-        let mut html = ArticleScraper::download(&url, &self.client).await?;
+        let html = ArticleScraper::download(&url, &self.client).await?;
-        parse_html!(html, config, xpath_ctx);
+        let mut xpath_ctx = Self::parse_html(html, config)?;
 
         // check for single page link
-        let mut xpath_ctx = xpath_ctx;
         if let Some(xpath_single_page_link) = config.single_page_link.clone() {
             debug!("Single page link xpath specified in config {}", xpath_single_page_link);
             if let Ok(result) = xpath_ctx.findvalue(&xpath_single_page_link, None) {
@@ -167,16 +164,12 @@ impl ArticleScraper {
         ArticleScraper::extract_body(&xpath_ctx, root, config)?;
 
         loop {
-            println!("loop");
             if let Some(url) = self.check_for_next_page(&xpath_ctx, config) {
-                println!("url {}", url);
+                let html = ArticleScraper::download(&url, &self.client).await?;
-                let mut html = ArticleScraper::download(&url, &self.client).await?;
+                xpath_ctx = Self::parse_html(html, config)?;
-                parse_html!(html, config, new_xpath_ctx);
+                ArticleScraper::strip_junk(&xpath_ctx, config, &url);
-                ArticleScraper::strip_junk(&new_xpath_ctx, config, &url);
+                ArticleScraper::extract_body(&xpath_ctx, root, config)?;
-                ArticleScraper::extract_body(&new_xpath_ctx, root, config)?;
-                xpath_ctx = new_xpath_ctx;
             } else {
-                println!("break");
                 break;
             }
         }
@@ -184,10 +177,51 @@ impl ArticleScraper {
         Ok(())
     }
 
+    fn parse_html(html: String, config: &GrabberConfig) -> Result<Context, ScraperError> {
+        // replace matches in raw html
+        
+        let mut html = html;
+        for replace in &config.replace {
+            html = html.replace(&replace.to_replace, &replace.replace_with);
+        }
+
+        // parse html
+        let parser = Parser::default_html();
+        let doc = parser.parse_string(html.as_str()).map_err(|err| {
+            error!("Parsing HTML failed for downloaded HTML {:?}", err);
+            ScraperErrorKind::Xml
+        })?;
+        
+        Ok(Context::new(&doc).map_err(|()| {
+            error!("Creating xpath context failed for downloaded HTML");
+            ScraperErrorKind::Xml
+        })?)
+    }
+
+    pub fn evaluate_xpath(xpath_ctx: &Context, xpath: &str, thorw_if_empty: bool) -> Result<Vec<Node>, ScraperError> {
+        let res = xpath_ctx.evaluate(xpath).map_err(|()| {
+            error!("Evaluation of xpath {} yielded no results", xpath);
+            println!("Evaluation of xpath {} yielded no results", xpath);
+            ScraperErrorKind::Xml
+        })?;
+
+        let node_vec = res.get_nodes_as_vec();
+
+        if node_vec.len() == 0 {
+            error!("Evaluation of xpath {} yielded no results", xpath);
+            println!("Evaluation of xpath {} yielded no results", xpath);
+            if thorw_if_empty {
+                return Err(ScraperErrorKind::Xml)?
+            }
+        }
+
+        Ok(node_vec)
+    }
+
     async fn parse_single_page(&self, article: &mut Article, url: &url::Url, root: &mut Node, config: &GrabberConfig) -> Result<(), ScraperError> {
         
-        let mut html = ArticleScraper::download(&url, &self.client).await?;
+        let html = ArticleScraper::download(&url, &self.client).await?;
-        parse_html!(html, config, xpath_ctx);
+        let xpath_ctx = Self::parse_html(html, config)?;
         ArticleScraper::extract_metadata(&xpath_ctx, config, article);
         ArticleScraper::strip_junk(&xpath_ctx, config, &url);
         ArticleScraper::extract_body(&xpath_ctx, root, config)?;
@@ -321,9 +355,7 @@ impl ArticleScraper {
     }
 
     fn extract_value(context: &Context, xpath: &str) -> Result<String, ScraperError> {
-
+        let node_vec = Self::evaluate_xpath(context, xpath, false)?;
-        evaluate_xpath!(context, xpath, node_vec);
-        xpath_result_empty!(node_vec, xpath);
         if let Some(val) = node_vec.get(0) {
             return Ok(val.get_content())
         }
@@ -332,9 +364,7 @@ impl ArticleScraper {
     }
 
     fn extract_value_merge(context: &Context, xpath: &str) -> Result<String, ScraperError> {
-
+        let node_vec = Self::evaluate_xpath(context, xpath, true)?;
-        evaluate_xpath!(context, xpath, node_vec);
-        xpath_result_empty!(node_vec, xpath);
         let mut val = String::new();
         for node in node_vec {
             val.push_str(&node.get_content());
@@ -344,14 +374,14 @@ impl ArticleScraper {
     }
 
     fn strip_node(context: &Context, xpath: &String) -> Result<(), ScraperError> {
-
         let mut ancestor = xpath.clone();
         if ancestor.starts_with("//") {
             ancestor = ancestor.chars().skip(2).collect();
         }
 
         let query = &format!("{}[not(ancestor::{})]", xpath, ancestor);
-        evaluate_xpath!(context, query, node_vec);
+        //println!("{}", query);
+        let node_vec = Self::evaluate_xpath(context, query, false)?;
         for mut node in node_vec {
             node.unlink();
         }
@@ -359,11 +389,11 @@ impl ArticleScraper {
     }
 
     fn strip_id_or_class(context: &Context, id_or_class: &String) -> Result<(), ScraperError> {
-
         let xpath = &format!("//*[contains(@class, '{}') or contains(@id, '{}')]", id_or_class, id_or_class);
-        evaluate_xpath!(context, xpath, node_vec);
+        //println!("{}", xpath);
+        let node_vec = Self::evaluate_xpath(context, xpath, false)?;
         for mut node in node_vec {
-            //node.unlink();
+            node.unlink();
         }
         Ok(())
     }
@@ -371,7 +401,7 @@ impl ArticleScraper {
     fn fix_lazy_images(context: &Context, class: &str, property_url: &str) -> Result<(), ScraperError> {
         
         let xpath = &format!("//img[contains(@class, '{}')]", class);
-        evaluate_xpath!(context, xpath, node_vec);
+        let node_vec = Self::evaluate_xpath(context, xpath, false)?;
         for mut node in node_vec {
             if let Some(correct_url) = node.get_property(property_url) {
                 if let Err(_) = node.set_property("src", &correct_url) {
@@ -385,7 +415,7 @@ impl ArticleScraper {
     fn fix_iframe_size(context: &Context, site_name: &str) -> Result<(), ScraperError> {
 
         let xpath = &format!("//iframe[contains(@src, '{}')]", site_name);
-        evaluate_xpath!(context, xpath, node_vec);
+        let node_vec = Self::evaluate_xpath(context, xpath, false)?;
         for mut node in node_vec {
             if let Some(mut parent) = node.get_parent() {
                 if let Ok(mut video_wrapper) = parent.new_child(None, "div") {
@@ -420,7 +450,7 @@ impl ArticleScraper {
         };
 
         let xpath = &format!("//{}[@{}]", xpath_tag, attribute);
-        evaluate_xpath!(context, xpath, node_vec);
+        let node_vec = Self::evaluate_xpath(context, xpath, false)?;
         for mut node in node_vec {
             if let Err(_) = node.remove_property(attribute) {
                 return Err(ScraperErrorKind::Xml)?
@@ -437,7 +467,7 @@ impl ArticleScraper {
         };
 
         let xpath = &format!("//{}", xpath_tag);
-        evaluate_xpath!(context, xpath, node_vec);
+        let node_vec = Self::evaluate_xpath(context, xpath, false)?;
         for mut node in node_vec {
             if let Err(_) = node.set_attribute(attribute, value) {
                 return Err(ScraperErrorKind::Xml)?
@@ -447,10 +477,7 @@ impl ArticleScraper {
     }
 
     fn get_attribute(context: &Context, xpath: &str, attribute: &str) -> Result<String, ScraperError> {
-        println!("get attribute {}", attribute);
+        let node_vec = Self::evaluate_xpath(context, xpath, false)?;
-        evaluate_xpath!(context, xpath, node_vec);
-        println!("found {}", node_vec.len());
-        xpath_result_empty!(node_vec, xpath);
         for node in node_vec {
             if let Some(value) = node.get_attribute(attribute) {
                 return Ok(value)
@@ -461,8 +488,7 @@ impl ArticleScraper {
     }
 
     fn repair_urls(context: &Context, xpath: &str, attribute: &str, article_url: &url::Url) -> Result<(), ScraperError> {
-
+        let node_vec = Self::evaluate_xpath(context, xpath, false)?;
-        evaluate_xpath!(context, xpath, node_vec);
         for mut node in node_vec {
             if let Some(val) = node.get_attribute(attribute) {
                 if let Err(url::ParseError::RelativeUrlWithoutBase) = url::Url::parse(&val) {
@@ -603,8 +629,7 @@ impl ArticleScraper {
 
         let mut found_something = false;
         {
-            evaluate_xpath!(context, xpath, node_vec);
+            let node_vec = Self::evaluate_xpath(context, xpath, false)?;
-            xpath_result_empty!(node_vec, xpath);
             for mut node in node_vec {
                 if node.get_property("style").is_some() {
                     if let Err(_) = node.remove_property("style") {
@@ -662,7 +687,7 @@ impl ArticleScraper {
         // this prevents libxml from self closing non void elements such as iframe
 
         let xpath = "//*[not(node())]";
-        evaluate_xpath!(context, xpath, node_vec);
+        let node_vec = Self::evaluate_xpath(context, xpath, false)?;
         for mut node in node_vec {
             if node.get_name() == "meta" {
                 continue
@@ -677,7 +702,7 @@ impl ArticleScraper {
     fn eliminate_noscrip_tag(context: &Context) -> Result<(), ScraperError> {
 
         let xpath = "//noscript";
-        evaluate_xpath!(context, xpath, node_vec);
+        let node_vec = Self::evaluate_xpath(context, xpath, false)?;
 
         for mut node in node_vec {
             if let Some(mut parent) = node.get_parent() {
@@ -700,19 +725,19 @@ impl ArticleScraper {
 mod tests {
     use crate::*;
     
-    #[tokio::test]
+    // #[tokio::test]
-    async fn golem() {
+    // async fn golem() {
-        let config_path = PathBuf::from(r"./resources/tests/golem");
+    //     let config_path = PathBuf::from(r"./resources/tests/golem");
-        let out_path = PathBuf::from(r"./test_output");
+    //     let out_path = PathBuf::from(r"./test_output");
-        let url = url::Url::parse("https://www.golem.de/news/http-error-418-fehlercode-ich-bin-eine-teekanne-darf-bleiben-1708-129460.html").unwrap();
+    //     let url = url::Url::parse("https://www.golem.de/news/http-error-418-fehlercode-ich-bin-eine-teekanne-darf-bleiben-1708-129460.html").unwrap();
 
-        let grabber = ArticleScraper::new(config_path).unwrap();
+    //     let grabber = ArticleScraper::new(config_path).unwrap();
-        let article = grabber.parse(url, true).await.unwrap();
+    //     let article = grabber.parse(url, true).await.unwrap();
-        article.save_html(&out_path).unwrap();
+    //     article.save_html(&out_path).unwrap();
 
-        assert_eq!(article.title, Some(String::from("HTTP Error 418: Fehlercode \"Ich bin eine Teekanne\" darf bleiben")));
+    //     assert_eq!(article.title, Some(String::from("HTTP Error 418: Fehlercode \"Ich bin eine Teekanne\" darf bleiben")));
-        assert_eq!(article.author, Some(String::from("Hauke Gierow")));
+    //     assert_eq!(article.author, Some(String::from("Hauke Gierow")));
-    }
+    // }
 
     #[tokio::test]
     async fn phoronix() {

src/macros.rs

@@ -1,51 +0,0 @@
-macro_rules! parse_html {
-    (
-        $html: ident,
-        $config: ident,
-        $xpath_ctx: ident
-    ) => {
-        // replace matches in raw html
-        for replace in &$config.replace {
-            $html = $html.replace(&replace.to_replace, &replace.replace_with);
-        }
-
-        // parse html
-        let parser = Parser::default_html();
-        let doc = parser.parse_string($html.as_str()).map_err(|err| {
-            error!("Parsing HTML failed for downloaded HTML {:?}", err);
-            ScraperErrorKind::Xml
-        })?;
-        
-        let $xpath_ctx = Context::new(&doc).map_err(|()| {
-            error!("Creating xpath context failed for downloaded HTML");
-            ScraperErrorKind::Xml
-        })?;
-    };
-}
-
-macro_rules! evaluate_xpath {
-    (
-        $context: ident,
-        $xpath: ident,
-        $node_vec: ident
-    ) => {
-        let res = $context.evaluate($xpath).map_err(|()| {
-            error!("Evaluation of xpath {} yielded no results", $xpath);
-            ScraperErrorKind::Xml
-        })?;
-
-        let $node_vec = res.get_nodes_as_vec();
-    };
-}
-
-macro_rules! xpath_result_empty {
-    (
-        $node_vec: ident,
-        $xpath: ident
-    ) => {
-        if $node_vec.len() == 0 {
-            error!("Evaluation of xpath {} yielded no results", $xpath);
-            return Err(ScraperErrorKind::Xml)?
-        }
-    };
-}