forked from becheran/mlc
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Fix benches * First teset * WIP * Fix unit tests * Working html support * Update docs * Fix markup types empty * Update readme
- Loading branch information
Showing
40 changed files
with
252 additions
and
30 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| <html> | ||
| <h1>Hello, world!</h1> | ||
| <p>bla bla <a hreflang="en" href="https://www.w3schools.com">Visit W3Schools.com!</a> bla bla </p> | ||
| <p>bla bla <a href = "https://www.w3schools.com">Visit W3Schools.com!</a> bla bla </p> | ||
| <p>multiline | ||
| <a | ||
| href= | ||
| "https://www.w3schools.com" >Visit W3Schools.com! | ||
| </> bla bla | ||
| </p> | ||
| sdjklf slfkj <!-- | ||
| <p>commented </p> | ||
| --> | ||
| </html> |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" | ||
| "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> | ||
|
|
||
| <html xmlns="http://www.w3.org/1999/xhtml"> | ||
|
|
||
| <head> | ||
| <title>Title of document</title> | ||
| <a href = "https://www.w3schools.com">Visit W3Schools.com!</a> | ||
| </head> | ||
|
|
||
| <body> | ||
| some content | ||
| </body> | ||
|
|
||
| </html> |
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Empty file.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| use crate::link_extractors::link_extractor::LinkExtractor; | ||
| use crate::link_extractors::link_extractor::MarkupLink; | ||
|
|
||
| pub struct HtmlLinkExtractor(); | ||
|
|
||
| enum ParserState { | ||
| Text, | ||
| Comment, | ||
| Anchor, | ||
| EqualSign, | ||
| Link, | ||
| } | ||
| impl LinkExtractor for HtmlLinkExtractor { | ||
| fn find_links(&self, text: &str) -> Vec<MarkupLink> { | ||
| let mut result: Vec<MarkupLink> = Vec::new(); | ||
| let mut state: ParserState = ParserState::Text; | ||
| for (line, line_str) in text.lines().enumerate() { | ||
| let line_chars: Vec<char> = line_str.chars().collect(); | ||
| let mut column: usize = 0; | ||
| while line_chars.get(column).is_some() { | ||
| match state { | ||
| ParserState::Comment => { | ||
| if line_chars.get(column) == Some(&'-') | ||
| && line_chars.get(column + 1) == Some(&'-') | ||
| && line_chars.get(column + 2) == Some(&'>') | ||
| { | ||
| column += 2; | ||
| state = ParserState::Text; | ||
| } | ||
| } | ||
| ParserState::Text => { | ||
| if line_chars.get(column) == Some(&'<') | ||
| && line_chars.get(column + 1) == Some(&'!') | ||
| && line_chars.get(column + 2) == Some(&'-') | ||
| && line_chars.get(column + 3) == Some(&'-') | ||
| { | ||
| column += 3; | ||
| state = ParserState::Comment; | ||
| } else if line_chars.get(column) == Some(&'<') | ||
| && line_chars.get(column + 1) == Some(&'a') | ||
| { | ||
| column += 1; | ||
| state = ParserState::Anchor; | ||
| } | ||
| } | ||
| ParserState::Anchor => { | ||
| if line_chars.get(column) == Some(&'h') | ||
| && line_chars.get(column + 1) == Some(&'r') | ||
| && line_chars.get(column + 2) == Some(&'e') | ||
| && line_chars.get(column + 3) == Some(&'f') | ||
| { | ||
| column += 3; | ||
| state = ParserState::EqualSign; | ||
| } | ||
| } | ||
| ParserState::EqualSign => { | ||
| match line_chars.get(column) { | ||
| Some(x) if x.is_whitespace() => {} | ||
| Some(x) if x == &'=' => state = ParserState::Link, | ||
| Some(_) => state = ParserState::Anchor, | ||
| None => {} | ||
| }; | ||
| } | ||
| ParserState::Link => { | ||
| match line_chars.get(column) { | ||
| Some(x) if !x.is_whitespace() && x != &'"' => { | ||
| let link_column = column; | ||
| while line_chars.get(column).is_some() | ||
| && !line_chars[column].is_whitespace() | ||
| && line_chars[column] != '"' | ||
| { | ||
| column += 1; | ||
| } | ||
| while let Some(c) = line_chars.get(column) { | ||
| if c.is_whitespace() || c == &'"'{ | ||
| break; | ||
| } | ||
| column += 1; | ||
| } | ||
| let link = (&line_chars[link_column..column]) | ||
| .iter() | ||
| .collect::<String>(); | ||
| result.push(MarkupLink { | ||
| column: link_column + 1, | ||
| line: line + 1, | ||
| target: link.to_string(), | ||
| }); | ||
| state = ParserState::Text; | ||
| } | ||
| Some(_) | None => {} | ||
| }; | ||
| } | ||
| } | ||
| column += 1; | ||
| } | ||
| } | ||
| result | ||
| } | ||
| } | ||
|
|
||
| #[cfg(test)] | ||
| mod tests { | ||
| use super::*; | ||
| use ntest::test_case; | ||
|
|
||
| #[test] | ||
| fn no_link() { | ||
| let le = HtmlLinkExtractor(); | ||
| let input = "]This is not a <has> no link <h1>Bla</h1> attribute."; | ||
| let result = le.find_links(&input); | ||
| assert!(result.is_empty()); | ||
| } | ||
|
|
||
| #[test] | ||
| fn commented() { | ||
| let le = HtmlLinkExtractor(); | ||
| let input = "df <!-- <a href=\"http://wiki.selfhtml.org\"> haha</a> -->"; | ||
| let result = le.find_links(&input); | ||
| assert!(result.is_empty()); | ||
| } | ||
|
|
||
| #[test_case( | ||
| "<a href=\"https://www.w3schools.com\">Visit W3Schools.com!</a>", | ||
| 1, | ||
| 10 | ||
| )] | ||
| #[test_case( | ||
| "<a\nhref\n=\n \"https://www.w3schools.com\">\nVisit W3Schools.com!\n</a>", | ||
| 4, | ||
| 4 | ||
| )] | ||
| #[test_case( | ||
| "<a hreflang=\"en\" href=\"https://www.w3schools.com\">Visit W3Schools.com!</a>", | ||
| 1, | ||
| 24 | ||
| )] | ||
| #[test_case( | ||
| "<!--comment--><a href=\"https://www.w3schools.com\">Visit W3Schools.com!</a>", | ||
| 1, | ||
| 24 | ||
| )] | ||
| fn links(input: &str, line: usize, column: usize) { | ||
| let le = HtmlLinkExtractor(); | ||
| let result = le.find_links(&input); | ||
| let expected = MarkupLink { | ||
| target: "https://www.w3schools.com".to_string(), | ||
| line: line, | ||
| column: column, | ||
| }; | ||
| assert_eq!(vec![expected], result); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,3 @@ | ||
| pub mod link_extractor; | ||
| mod markdown_link_extractor; | ||
| mod markdown_link_extractor; | ||
| mod html_link_extractor; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.