diff options
| -rw-r--r-- | Cargo.lock | 119 | ||||
| -rw-r--r-- | htmlua-parser/Cargo.toml | 1 | ||||
| -rw-r--r-- | htmlua-parser/src/helpers.rs | 7 | ||||
| -rw-r--r-- | htmlua-parser/src/htmlua_stdlib.rs | 119 | ||||
| -rw-r--r-- | htmlua-parser/src/lib.rs | 4 | ||||
| -rw-r--r-- | htmlua-parser/src/render.rs | 177 | ||||
| -rw-r--r-- | htmlua-parser/src/serve.rs | 3 |
7 files changed, 392 insertions, 38 deletions
@@ -18,6 +18,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] name = "anyhow" version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -84,6 +93,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", + "regex-automata", "serde", ] @@ -152,6 +162,21 @@ dependencies = [ ] [[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] name = "cssparser" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -330,6 +355,20 @@ dependencies = [ ] [[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] name = "futures-channel" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -352,6 +391,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] name = "futures-sink" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -369,8 +419,10 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -507,6 +559,7 @@ version = "0.1.0" dependencies = [ "anyhow", "html5ever 0.35.0", + "httptest", "kuchikiki", "markup5ever 0.11.0", "mlua", @@ -563,6 +616,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "httptest" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde82de3ef9bd882493c6a5edbc3363ad928925b30ccecc0f2ddeb42601b3021" +dependencies = [ + "bstr", + "bytes", + "crossbeam-channel", + "form_urlencoded", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", +] + +[[package]] name = "hyper" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -575,6 +658,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa 1.0.15", "pin-project-lite", "smallvec", @@ -1503,6 +1587,29 @@ dependencies = [ ] [[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] name = "regex-syntax" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2047,10 +2154,22 @@ dependencies = [ "pin-project-lite", "slab", "socket2", + "tokio-macros", "windows-sys 0.52.0", ] [[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] name = "tokio-native-tls" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/htmlua-parser/Cargo.toml b/htmlua-parser/Cargo.toml index 83769af..4ee6be3 100644 --- a/htmlua-parser/Cargo.toml +++ b/htmlua-parser/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] anyhow = "1.0.98" html5ever = "0.35.0" +httptest = "0.16.3" kuchikiki = "0.8.2" markup5ever = "0.11.0" mlua = { version = "0.10.5", features = ["lua54", "vendored", "serialize", "anyhow"] } diff --git a/htmlua-parser/src/helpers.rs b/htmlua-parser/src/helpers.rs index 90439f7..512ebb7 100644 --- a/htmlua-parser/src/helpers.rs +++ b/htmlua-parser/src/helpers.rs @@ -1,4 +1,8 @@ -use std::{fs::File, io::{BufRead, BufReader, Read, Seek, SeekFrom}, path::PathBuf}; +use std::{ + fs::File, + io::{BufRead, BufReader, Read, Seek, SeekFrom}, + path::PathBuf, +}; use anyhow::Result; use kuchikiki::NodeRef; @@ -9,7 +13,6 @@ use tendril::TendrilSink; pub fn read_doc_from_file(path: PathBuf) -> Result<NodeRef> { - let mut file = File::open(path)?; let mut reader = BufReader::new(&mut file); diff --git a/htmlua-parser/src/htmlua_stdlib.rs b/htmlua-parser/src/htmlua_stdlib.rs index 3c27f26..d36620e 100644 --- a/htmlua-parser/src/htmlua_stdlib.rs +++ b/htmlua-parser/src/htmlua_stdlib.rs @@ -1,10 +1,19 @@ -use std::{cell::{LazyCell, RefCell}, collections::HashMap, fmt::Write, rc::Rc}; +use std::{ + cell::{LazyCell, RefCell}, + collections::HashMap, + fmt::Write, + rc::Rc, + str::FromStr, + time::Duration, +}; -use anyhow::anyhow; -use mlua::{prelude::*, Error, Lua, Table}; -use reqwest::{blocking::{Client, Response}, header::HeaderMap}; +use mlua::{Error, Lua, Table, prelude::*}; +use reqwest::{ + Method, Url, + blocking::{Client, Response}, + header::HeaderMap, +}; use serde::{Deserialize, Serialize}; -use serde_json::to_value; pub fn create_htmlua_stdlib(l: &Lua, stdout: &Rc<RefCell<String>>) -> mlua::Result<Table> { @@ -34,13 +43,16 @@ pub fn create_htmlua_stdlib(l: &Lua, stdout: &Rc<RefCell<String>>) -> mlua::Resu Ok(t) } +#[allow(clippy::too_many_lines)] fn create_http_lib(l: &Lua) -> mlua::Result<Table> { let t = l.create_table()?; - let http_client: Rc<LazyCell<Client>> = Rc::new(LazyCell::new(|| Client::builder() + let http_client: Rc<LazyCell<Client>> = Rc::new(LazyCell::new(|| { + Client::builder() .timeout(std::time::Duration::from_secs(3)) .user_agent("htmlua/0.1.0") - .build().unwrap() - )); + .build() + .unwrap() + })); let client = http_client.clone(); @@ -50,56 +62,116 @@ fn create_http_lib(l: &Lua) -> mlua::Result<Table> { let res = client.get(url).send().map_err(|e| Error::RuntimeError(e.to_string()))?; let lua_res: LuaHttpResponse = TryFrom::try_from(res)?; Ok(lua_res) - })? + })?, )?; let client = http_client.clone(); t.set( "post", l.create_function(move |_, url: String| { - let res = client.post(url).send().map_err(|e| Error::RuntimeError(e.to_string()))?; + let res = client + .post(url) + .send() + .map_err(|e| Error::RuntimeError(e.to_string()))?; let lua_res: LuaHttpResponse = TryFrom::try_from(res)?; Ok(lua_res) - })? + })?, )?; let client = http_client.clone(); t.set( "get_with_data", l.create_function(move |_, (url, data): (String, HashMap<String, String>)| { - let res = client.get(url).query(&data).send().map_err(|e| Error::RuntimeError(e.to_string()))?; + let res = client + .get(url) + .query(&data) + .send() + .map_err(|e| Error::RuntimeError(e.to_string()))?; let lua_res: LuaHttpResponse = TryFrom::try_from(res)?; Ok(lua_res) - })? + })?, )?; let client = http_client.clone(); t.set( "post_with_data_form", l.create_function(move |_, (url, data): (String, HashMap<String, String>)| { - let res = client.post(url).form(&data).send().map_err(|e| Error::RuntimeError(e.to_string()))?; + let res = client + .post(url) + .form(&data) + .send() + .map_err(|e| Error::RuntimeError(e.to_string()))?; let lua_res: LuaHttpResponse = TryFrom::try_from(res)?; Ok(lua_res) - })? + })?, )?; let client = http_client.clone(); t.set( "post_with_data_json", l.create_function(move |_, (url, data): (String, HashMap<String, String>)| { - let res = client.post(url).json(&data).send().map_err(|e| Error::RuntimeError(e.to_string()))?; + let res = client + .post(url) + .json(&data) + .send() + .map_err(|e| Error::RuntimeError(e.to_string()))?; + let lua_res: LuaHttpResponse = TryFrom::try_from(res)?; + Ok(lua_res) + })?, + )?; + + let client = http_client.clone(); + t.set( + "request", + l.create_function(move |_, table: mlua::Table| { + let mut request = client.request( + Method::from_bytes(table.get::<String>("method")?.as_bytes()) + .map_err(|e| Error::RuntimeError(e.to_string()))?, + Url::from_str(table.get::<String>("url")?.as_str()).map_err(|e| Error::RuntimeError(e.to_string()))?, + ); + + if let Ok(header_tbl) = table.get::<mlua::Table>("headers") { + request = header_tbl + .pairs::<String, String>() + .filter_map(std::result::Result::ok) + .fold(request, |req, (k, v)| req.header(k, v)); + } + + if let Ok(basic_auth) = table.get::<mlua::Table>("basic_auth") { + request = request + .basic_auth(basic_auth.get::<String>("username")?, basic_auth.get::<String>("password").ok()); + } + + if let Ok(bearer_auth) = table.get::<mlua::Table>("bearer_auth") { + request = request.bearer_auth(bearer_auth.get::<String>("token")?); + } + + if let Ok(body) = table.get::<String>("body") { + request = request.body(body); + } + + if let Ok(json) = table.get::<String>("json") { + request = request.json(&json); + } + + if let Ok(timeout) = table.get::<u64>("timeout") { + request = request.timeout(Duration::from_secs(timeout)); + } + + + let res = request.send().map_err(|e| Error::RuntimeError(e.to_string()))?; let lua_res: LuaHttpResponse = TryFrom::try_from(res)?; Ok(lua_res) - })? + })?, )?; - //TODO: all http t.set( "decode_json", l.create_function(move |l, text: String| { - let table: serde_json::Value = serde_json::from_str(&text).map_err(|e| Error::RuntimeError(e.to_string()))?; + let table: serde_json::Value = + serde_json::from_str(&text).map_err(|e| Error::RuntimeError(e.to_string()))?; Ok(l.to_value(&table)) - })? + })?, )?; Ok(t) @@ -115,7 +187,11 @@ struct LuaHttpResponse { impl TryFrom<Response> for LuaHttpResponse { fn try_from(value: Response) -> Result<Self, Self::Error> { - Ok(LuaHttpResponse { headers: headermap_to_hashmap(value.headers()), status: value.status().as_u16(), body: value.text().map_err(|e| Error::RuntimeError(e.to_string()))?, }) + Ok(LuaHttpResponse { + headers: headermap_to_hashmap(value.headers()), + status: value.status().as_u16(), + body: value.text().map_err(|e| Error::RuntimeError(e.to_string()))?, + }) } type Error = Error; @@ -133,6 +209,7 @@ fn headermap_to_hashmap(headers: &HeaderMap) -> HashMap<String, String> { map } + impl IntoLua for LuaHttpResponse { fn into_lua(self, lua: &Lua) -> LuaResult<LuaValue> { let table = lua.create_table()?; diff --git a/htmlua-parser/src/lib.rs b/htmlua-parser/src/lib.rs index be272dd..062fcb4 100644 --- a/htmlua-parser/src/lib.rs +++ b/htmlua-parser/src/lib.rs @@ -1,4 +1,4 @@ -pub mod render; -pub mod serve; pub mod helpers; pub mod htmlua_stdlib; +pub mod render; +pub mod serve; diff --git a/htmlua-parser/src/render.rs b/htmlua-parser/src/render.rs index 1bb043b..a92e7c5 100644 --- a/htmlua-parser/src/render.rs +++ b/htmlua-parser/src/render.rs @@ -1,8 +1,13 @@ -use std::{cell::RefCell, fmt::Write, fs::read_to_string, path::PathBuf, rc::Rc}; +use std::{ + cell::{LazyCell, RefCell}, + fmt::Write, + path::PathBuf, + rc::Rc, +}; use anyhow::{Result, anyhow}; use kuchikiki::{NodeRef, traits::TendrilSink}; -use mlua::{Lua, Table}; +use mlua::Lua; use pulldown_cmark::{Options, Parser, html}; use syntect::{ easy::HighlightLines, @@ -15,20 +20,22 @@ use syntect::{ use crate::{helpers::read_doc_from_file, htmlua_stdlib::create_htmlua_stdlib}; - -pub fn execute_lua(document: NodeRef) -> Result<NodeRef> { +fn build_lua_with_stdout(stdout: &Rc<RefCell<String>>) -> Result<Lua> { let lua = Lua::new(); let globals = lua.globals(); - let stdout: Rc<RefCell<String>> = Rc::new(RefCell::new(String::new())); - - let htmlua_table = - create_htmlua_stdlib(&lua, &stdout).map_err(|e| anyhow!("Failed to create Lua stdlib: {}", e))?; + let htmlua_table = create_htmlua_stdlib(&lua, stdout).map_err(|e| anyhow!("Failed to create Lua stdlib: {}", e))?; globals .set("htmlua", htmlua_table) .map_err(|e| anyhow!("Failed to set global: {}", e))?; + Ok(lua) +} + +pub fn execute_lua(document: NodeRef) -> Result<NodeRef> { + let stdout: Rc<RefCell<String>> = Rc::new(RefCell::new(String::new())); + let lua = LazyCell::new(|| build_lua_with_stdout(&stdout).unwrap_or_default()); let lua_elements: Vec<_> = match document.select("lua") { Ok(e) => e.collect(), Err(()) => return Err(anyhow!("Unable to find Lua")), @@ -85,7 +92,11 @@ pub fn expand_template(document: NodeRef, component_path: &PathBuf, include_from let exported = from_node .select_first(format!("exportelement.{name}").as_str()) .map_err(|()| anyhow!("Error finding exportelement"))?; - exported.as_node().children().rev().for_each(|c| i.as_node().insert_after(c)); + exported + .as_node() + .children() + .rev() + .for_each(|c| i.as_node().insert_after(c)); } } while let Ok(i) = document.select_first("includeelement") { @@ -107,8 +118,13 @@ pub fn expand_template(document: NodeRef, component_path: &PathBuf, include_from item_path.push(include_path); let new_node = read_doc_from_file(item_path)?; let replaced_node = expand_template(new_node, component_path, Some(i.as_node()))?; - replaced_node.select_first("html").map_err(|()| anyhow!("Error finding html"))?.as_node().children().rev().for_each(|c| i.as_node().insert_after(c)); - + replaced_node + .select_first("html") + .map_err(|()| anyhow!("Error finding html"))? + .as_node() + .children() + .rev() + .for_each(|c| i.as_node().insert_after(c)); } } while let Ok(i) = document.select_first("include") { @@ -167,6 +183,8 @@ pub fn process_syntax_highlighting(document: NodeRef) -> Result<NodeRef> { #[cfg(test)] mod tests { + use httptest::{Expectation, ServerPool, matchers::*, responders::*}; + use super::*; #[test] @@ -413,4 +431,141 @@ fn main() { assert_eq!(attrs.get("data-lang"), Some("rust")); assert!(attrs.get("class").unwrap().contains("syntax-highlight")); } + + static SERVER_POOL: ServerPool = ServerPool::new(2); + + #[test] + fn basic_get() { + let server = SERVER_POOL.get_server(); + + server.expect( + Expectation::matching(request::method_path("GET", "/test/1")).respond_with(status_code(200).body("ret")), + ); + + let stdout = Rc::new(RefCell::new(String::new())); + let lua = build_lua_with_stdout(&stdout).unwrap(); + let code = format!("htmlua.print(htmlua.http.get(\"{}\").body)", server.url("/test/1")); + lua.load(code).exec().unwrap(); + assert_eq!(stdout.borrow().as_str(), "ret"); + } + + #[test] + fn basic_post() { + let server = SERVER_POOL.get_server(); + + server.expect( + Expectation::matching(request::method_path("POST", "/test/1")).respond_with(status_code(200).body("ret")), + ); + + let stdout = Rc::new(RefCell::new(String::new())); + let lua = build_lua_with_stdout(&stdout).unwrap(); + let code = format!("htmlua.print(htmlua.http.post(\"{}\").body)", server.url("/test/1")); + lua.load(code).exec().unwrap(); + assert_eq!(stdout.borrow().as_str(), "ret"); + } + + #[test] + fn get_with_data() { + let server = SERVER_POOL.get_server(); + + server.expect( + Expectation::matching(request::query(url_decoded(contains(("a", "b"))))) + .respond_with(status_code(200).body("ret")), + ); + + let stdout = Rc::new(RefCell::new(String::new())); + let lua = build_lua_with_stdout(&stdout).unwrap(); + let code = format!( + " + data = {{}} + data.a = \"b\" + htmlua.print(htmlua.http.get_with_data(\"{}\", data).body) + ", + server.url("/test/1") + ); + lua.load(code).exec().unwrap(); + assert_eq!(stdout.borrow().as_str(), "ret"); + } + + #[test] + fn post_with_data_form() { + let server = SERVER_POOL.get_server(); + + server.expect( + Expectation::matching(all_of![request::method_path("POST", "/test/1"), request::body("a=b")]) + .respond_with(status_code(200).body("ret")), + ); + + let stdout = Rc::new(RefCell::new(String::new())); + let lua = build_lua_with_stdout(&stdout).unwrap(); + let code = format!( + " + data = {{}} + data.a = \"b\" + htmlua.print(htmlua.http.post_with_data_form(\"{}\", data).body) + ", + server.url("/test/1") + ); + lua.load(code).exec().unwrap(); + assert_eq!(stdout.borrow().as_str(), "ret"); + } + + #[test] + fn post_with_data_json() { + let server = SERVER_POOL.get_server(); + + server.expect( + Expectation::matching(all_of![request::method_path("POST", "/test/1"), request::body(r#"{"a":"b"}"#)]) + .respond_with(status_code(200).body("ret")), + ); + + let stdout = Rc::new(RefCell::new(String::new())); + let lua = build_lua_with_stdout(&stdout).unwrap(); + let code = format!( + " + data = {{}} + data.a = \"b\" + htmlua.print(htmlua.http.post_with_data_json(\"{}\", data).body) + ", + server.url("/test/1") + ); + lua.load(code).exec().unwrap(); + assert_eq!(stdout.borrow().as_str(), "ret"); + } + + #[test] + fn table_request() { + let server = SERVER_POOL.get_server(); + + server.expect( + Expectation::matching(all_of![ + request::method_path("GET", "/test/1"), + request::body("cool"), + request::headers(contains(("testhdr", "val"))), + request::headers(contains(("authorization", "Basic dXNlcjpwYXNz"))), + ]) + .respond_with(status_code(200).body("ret")), + ); + + let stdout = Rc::new(RefCell::new(String::new())); + let lua = build_lua_with_stdout(&stdout).unwrap(); + let code = format!( + " + req = {{}} + req.url = \"{}\" + req.method = \"GET\" + req.headers = {{}} + req.headers.testhdr = \"val\" + req.basic_auth = {{}} + req.basic_auth.username = \"user\" + req.basic_auth.password = \"pass\" + req.body = \"cool\" + + htmlua.print(htmlua.http.request(req).body) + ", + server.url("/test/1") + ); + lua.load(code).exec().unwrap(); + assert_eq!(stdout.borrow().as_str(), "ret"); + } } diff --git a/htmlua-parser/src/serve.rs b/htmlua-parser/src/serve.rs index 06de0ea..cfb3811 100644 --- a/htmlua-parser/src/serve.rs +++ b/htmlua-parser/src/serve.rs @@ -2,8 +2,7 @@ use crate::helpers::read_doc_from_file; use super::render::{execute_lua, expand_template, process_markdown, process_syntax_highlighting}; use anyhow::Result; -use std::path::{Path, PathBuf} -; +use std::path::{Path, PathBuf}; pub fn serve_content(request_uri: &str) -> Result<String> { // TODO: read from config |