Merge pull request #11 from gyoder/httpstdlib

Added Basic HTTP stdlib. Closes #10

4 files changed, 389 insertions, 34 deletions

diff --git a/htmlua-parser/Cargo.toml b/htmlua-parser/Cargo.toml
index 9e5018e..8200513 100644
--- a/htmlua-parser/Cargo.toml
+++ b/htmlua-parser/Cargo.toml
@@ -7,11 +7,14 @@ edition = "2024"
 anyhow = "1.0.98"
 dirs = "6.0.0"
 html5ever = "0.35.0"
+httptest = "0.16.3"
 kuchikiki = "0.8.2"
 markup5ever = "0.11.0"
-mlua = { version = "0.10.5", features = ["lua54", "vendored", "serialize"] }
+mlua = { version = "0.10.5", features = ["lua54", "vendored", "serialize", "anyhow"] }
 pulldown-cmark = "0.13.0"
+reqwest = { version = "0.12.22", features = ["blocking", "json"] }
 serde = { version = "1.0.219", features = ["derive"] }
+serde_json = "1.0.140"
 syntect = "5.2.0"
 tendril = "0.4.3"
 toml = "0.9.2"
diff --git a/htmlua-parser/src/htmlua_stdlib.rs b/htmlua-parser/src/htmlua_stdlib.rs
new file mode 100644
index 0000000..d36620e
--- /dev/null
+++ b/htmlua-parser/src/htmlua_stdlib.rs
@@ -0,0 +1,228 @@
+use std::{
+    cell::{LazyCell, RefCell},
+    collections::HashMap,
+    fmt::Write,
+    rc::Rc,
+    str::FromStr,
+    time::Duration,
+};
+
+use mlua::{Error, Lua, Table, prelude::*};
+use reqwest::{
+    Method, Url,
+    blocking::{Client, Response},
+    header::HeaderMap,
+};
+use serde::{Deserialize, Serialize};
+
+
+pub fn create_htmlua_stdlib(l: &Lua, stdout: &Rc<RefCell<String>>) -> mlua::Result<Table> {
+    let t = l.create_table()?;
+
+    // This cannot be the best way to do this
+
+    let stdout_println = stdout.clone();
+    t.set(
+        "println",
+        l.create_function(move |_, text: String| {
+            let mut stdout_ref = stdout_println.borrow_mut();
+            writeln!(stdout_ref, "{text}").map_err(mlua::Error::external)
+        })?,
+    )?;
+
+    let stdout_print = stdout.clone();
+    t.set(
+        "print",
+        l.create_function(move |_, text: String| {
+            let mut stdout_ref = stdout_print.borrow_mut();
+            write!(stdout_ref, "{text}").map_err(mlua::Error::external)
+        })?,
+    )?;
+
+    t.set("http", create_http_lib(l)?)?;
+    Ok(t)
+}
+
+#[allow(clippy::too_many_lines)]
+fn create_http_lib(l: &Lua) -> mlua::Result<Table> {
+    let t = l.create_table()?;
+    let http_client: Rc<LazyCell<Client>> = Rc::new(LazyCell::new(|| {
+        Client::builder()
+            .timeout(std::time::Duration::from_secs(3))
+            .user_agent("htmlua/0.1.0")
+            .build()
+            .unwrap()
+    }));
+
+
+    let client = http_client.clone();
+    t.set(
+        "get",
+        l.create_function(move |_, url: String| {
+            let res = client.get(url).send().map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
+            Ok(lua_res)
+        })?,
+    )?;
+
+    let client = http_client.clone();
+    t.set(
+        "post",
+        l.create_function(move |_, url: String| {
+            let res = client
+                .post(url)
+                .send()
+                .map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
+            Ok(lua_res)
+        })?,
+    )?;
+
+    let client = http_client.clone();
+    t.set(
+        "get_with_data",
+        l.create_function(move |_, (url, data): (String, HashMap<String, String>)| {
+            let res = client
+                .get(url)
+                .query(&data)
+                .send()
+                .map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
+            Ok(lua_res)
+        })?,
+    )?;
+
+    let client = http_client.clone();
+    t.set(
+        "post_with_data_form",
+        l.create_function(move |_, (url, data): (String, HashMap<String, String>)| {
+            let res = client
+                .post(url)
+                .form(&data)
+                .send()
+                .map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
+            Ok(lua_res)
+        })?,
+    )?;
+
+    let client = http_client.clone();
+    t.set(
+        "post_with_data_json",
+        l.create_function(move |_, (url, data): (String, HashMap<String, String>)| {
+            let res = client
+                .post(url)
+                .json(&data)
+                .send()
+                .map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
+            Ok(lua_res)
+        })?,
+    )?;
+
+    let client = http_client.clone();
+    t.set(
+        "request",
+        l.create_function(move |_, table: mlua::Table| {
+            let mut request = client.request(
+                Method::from_bytes(table.get::<String>("method")?.as_bytes())
+                    .map_err(|e| Error::RuntimeError(e.to_string()))?,
+                Url::from_str(table.get::<String>("url")?.as_str()).map_err(|e| Error::RuntimeError(e.to_string()))?,
+            );
+
+            if let Ok(header_tbl) = table.get::<mlua::Table>("headers") {
+                request = header_tbl
+                    .pairs::<String, String>()
+                    .filter_map(std::result::Result::ok)
+                    .fold(request, |req, (k, v)| req.header(k, v));
+            }
+
+            if let Ok(basic_auth) = table.get::<mlua::Table>("basic_auth") {
+                request = request
+                    .basic_auth(basic_auth.get::<String>("username")?, basic_auth.get::<String>("password").ok());
+            }
+
+            if let Ok(bearer_auth) = table.get::<mlua::Table>("bearer_auth") {
+                request = request.bearer_auth(bearer_auth.get::<String>("token")?);
+            }
+
+            if let Ok(body) = table.get::<String>("body") {
+                request = request.body(body);
+            }
+
+            if let Ok(json) = table.get::<String>("json") {
+                request = request.json(&json);
+            }
+
+            if let Ok(timeout) = table.get::<u64>("timeout") {
+                request = request.timeout(Duration::from_secs(timeout));
+            }
+
+
+            let res = request.send().map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
+            Ok(lua_res)
+        })?,
+    )?;
+
+    t.set(
+        "decode_json",
+        l.create_function(move |l, text: String| {
+            let table: serde_json::Value =
+                serde_json::from_str(&text).map_err(|e| Error::RuntimeError(e.to_string()))?;
+            Ok(l.to_value(&table))
+        })?,
+    )?;
+
+    Ok(t)
+}
+
+
+#[derive(Serialize, Deserialize)]
+struct LuaHttpResponse {
+    headers: HashMap<String, String>,
+    body: String,
+    status: u16,
+}
+
+impl TryFrom<Response> for LuaHttpResponse {
+    fn try_from(value: Response) -> Result<Self, Self::Error> {
+        Ok(LuaHttpResponse {
+            headers: headermap_to_hashmap(value.headers()),
+            status: value.status().as_u16(),
+            body: value.text().map_err(|e| Error::RuntimeError(e.to_string()))?,
+        })
+    }
+
+    type Error = Error;
+}
+
+fn headermap_to_hashmap(headers: &HeaderMap) -> HashMap<String, String> {
+    let mut map = HashMap::new();
+
+    for (name, value) in headers {
+        if let Ok(value_str) = value.to_str() {
+            map.insert(name.as_str().to_string(), value_str.to_string());
+        }
+    }
+
+    map
+}
+
+
+impl IntoLua for LuaHttpResponse {
+    fn into_lua(self, lua: &Lua) -> LuaResult<LuaValue> {
+        let table = lua.create_table()?;
+
+        table.set("status", self.status)?;
+        table.set("body", self.body)?;
+
+        let headers_table = lua.create_table()?;
+        for (key, value) in self.headers {
+            headers_table.set(key, value)?;
+        }
+        table.set("headers", headers_table)?;
+
+        Ok(LuaValue::Table(table))
+    }
+}
diff --git a/htmlua-parser/src/lib.rs b/htmlua-parser/src/lib.rs
index 525486e..2dc831c 100644
--- a/htmlua-parser/src/lib.rs
+++ b/htmlua-parser/src/lib.rs
@@ -1,4 +1,5 @@
-pub mod config;
 pub mod helpers;
+pub mod htmlua_stdlib;
+pub mod config;
 pub mod render;
 pub mod serve;
diff --git a/htmlua-parser/src/render.rs b/htmlua-parser/src/render.rs
index 9360f16..011412f 100644
--- a/htmlua-parser/src/render.rs
+++ b/htmlua-parser/src/render.rs
@@ -1,8 +1,13 @@
-use std::{cell::RefCell, fmt::Write, path::PathBuf, rc::Rc};
+use std::{
+    cell::{LazyCell, RefCell},
+    fmt::Write,
+    path::PathBuf,
+    rc::Rc,
+};
 
 use anyhow::{Result, anyhow};
 use kuchikiki::{NodeRef, traits::TendrilSink};
-use mlua::{Lua, Table};
+use mlua::Lua;
 use pulldown_cmark::{Options, Parser, html};
 use syntect::{
     easy::HighlightLines,
@@ -12,46 +17,25 @@ use syntect::{
     util::LinesWithEndings,
 };
 
-use crate::{helpers::read_doc_from_file, serve::get_config};
-
-fn create_htmlua_stdlib(l: &Lua, stdout: &Rc<RefCell<String>>) -> mlua::Result<Table> {
-    let t = l.create_table()?;
-
-    // This cannot be the best way to do this
+use crate::{helpers::read_doc_from_file, htmlua_stdlib::create_htmlua_stdlib, serve::get_config};
 
-    let stdout_println = stdout.clone();
-    t.set(
-        "println",
-        l.create_function(move |_, text: String| {
-            let mut stdout_ref = stdout_println.borrow_mut();
-            writeln!(stdout_ref, "{text}").map_err(mlua::Error::external)
-        })?,
-    )?;
 
-    let stdout_print = stdout.clone();
-    t.set(
-        "print",
-        l.create_function(move |_, text: String| {
-            let mut stdout_ref = stdout_print.borrow_mut();
-            write!(stdout_ref, "{text}").map_err(mlua::Error::external)
-        })?,
-    )?;
-    Ok(t)
-}
-
-pub fn execute_lua(document: NodeRef) -> Result<NodeRef> {
+fn build_lua_with_stdout(stdout: &Rc<RefCell<String>>) -> Result<Lua> {
     let lua = Lua::new();
     let globals = lua.globals();
 
-    let stdout: Rc<RefCell<String>> = Rc::new(RefCell::new(String::new()));
-
-    let htmlua_table =
-        create_htmlua_stdlib(&lua, &stdout).map_err(|e| anyhow!("Failed to create Lua stdlib: {}", e))?;
+    let htmlua_table = create_htmlua_stdlib(&lua, stdout).map_err(|e| anyhow!("Failed to create Lua stdlib: {}", e))?;
 
     globals
         .set("htmlua", htmlua_table)
         .map_err(|e| anyhow!("Failed to set global: {}", e))?;
 
+    Ok(lua)
+}
+
+pub fn execute_lua(document: NodeRef) -> Result<NodeRef> {
+    let stdout: Rc<RefCell<String>> = Rc::new(RefCell::new(String::new()));
+    let lua = LazyCell::new(|| build_lua_with_stdout(&stdout).unwrap_or_default());
     let lua_elements: Vec<_> = match document.select("lua") {
         Ok(e) => e.collect(),
         Err(()) => return Err(anyhow!("Unable to find Lua")),
@@ -198,6 +182,8 @@ pub fn process_syntax_highlighting(document: NodeRef) -> Result<NodeRef> {
 
 #[cfg(test)]
 mod tests {
+    use httptest::{Expectation, ServerPool, matchers::*, responders::*};
+
     use super::*;
 
     #[test]
@@ -444,4 +430,141 @@ fn main() {
         assert_eq!(attrs.get("data-lang"), Some("rust"));
         assert!(attrs.get("class").unwrap().contains("syntax-highlight"));
     }
+
+    static SERVER_POOL: ServerPool = ServerPool::new(2);
+
+    #[test]
+    fn basic_get() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(request::method_path("GET", "/test/1")).respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!("htmlua.print(htmlua.http.get(\"{}\").body)", server.url("/test/1"));
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
+
+    #[test]
+    fn basic_post() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(request::method_path("POST", "/test/1")).respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!("htmlua.print(htmlua.http.post(\"{}\").body)", server.url("/test/1"));
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
+
+    #[test]
+    fn get_with_data() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(request::query(url_decoded(contains(("a", "b")))))
+                .respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!(
+            "
+                data = {{}}
+                data.a = \"b\"
+                htmlua.print(htmlua.http.get_with_data(\"{}\", data).body)
+            ",
+            server.url("/test/1")
+        );
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
+
+    #[test]
+    fn post_with_data_form() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(all_of![request::method_path("POST", "/test/1"), request::body("a=b")])
+                .respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!(
+            "
+                data = {{}}
+                data.a = \"b\"
+                htmlua.print(htmlua.http.post_with_data_form(\"{}\", data).body)
+            ",
+            server.url("/test/1")
+        );
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
+
+    #[test]
+    fn post_with_data_json() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(all_of![request::method_path("POST", "/test/1"), request::body(r#"{"a":"b"}"#)])
+                .respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!(
+            "
+                data = {{}}
+                data.a = \"b\"
+                htmlua.print(htmlua.http.post_with_data_json(\"{}\", data).body)
+            ",
+            server.url("/test/1")
+        );
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
+
+    #[test]
+    fn table_request() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(all_of![
+                request::method_path("GET", "/test/1"),
+                request::body("cool"),
+                request::headers(contains(("testhdr", "val"))),
+                request::headers(contains(("authorization", "Basic dXNlcjpwYXNz"))),
+            ])
+            .respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!(
+            "
+                req = {{}}
+                req.url = \"{}\"
+                req.method = \"GET\"
+                req.headers = {{}}
+                req.headers.testhdr = \"val\"
+                req.basic_auth = {{}}
+                req.basic_auth.username = \"user\"
+                req.basic_auth.password = \"pass\"
+                req.body = \"cool\"
+
+                htmlua.print(htmlua.http.request(req).body)
+            ",
+            server.url("/test/1")
+        );
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
 }