diff --git a/Cargo.lock b/Cargo.lock index 6df3533..34013b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,7 +116,7 @@ dependencies = [ "p256", "p384", "rand 0.8.5", - "reqwest 0.12.28", + "reqwest", "serde", "serde_ipld_dagcbor", "serde_json", @@ -136,7 +136,7 @@ dependencies = [ "async-trait", "atproto-identity", "futures", - "http 1.4.0", + "http", "serde", "serde_json", "thiserror 2.0.17", @@ -165,10 +165,10 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http 1.4.0", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-util", "itoa", "matchit", @@ -180,7 +180,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tower", "tower-layer", @@ -196,12 +196,12 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.4.0", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "mime", "pin-project-lite", - "sync_wrapper 1.0.2", + "sync_wrapper", "tower-layer", "tower-service", "tracing", @@ -229,12 +229,6 @@ dependencies = [ "match-lookup", ] -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -247,12 +241,6 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.10.0" @@ -307,6 +295,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" @@ -393,6 +387,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -512,6 +516,29 @@ dependencies = [ "typenum", ] +[[package]] +name = "cssparser" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.11.3", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.111", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -620,6 +647,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "digest" version = "0.10.7" @@ -643,12 +681,60 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "dom_query" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688b93023aba6768721b48ec5588308e45ac42d788c6dd974d1c2b9a1d04ea29" +dependencies = [ + "cssparser", + "foldhash", + "html5ever 0.29.1", + "precomputed-hash", + "selectors", + "tendril", +] + +[[package]] +name = "dom_smoothie" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d23bf500fc0a79f9bf12c38816574820929ecf4f6b39ec07743f7ed485439c31" +dependencies = [ + "dom_query", + "flagset", + "gjson", + "html-escape", + "once_cell", + "phf 0.11.3", + "regex", + "tendril", + "thiserror 2.0.17", + "unicode-segmentation", + "url", +] + [[package]] name = "dotenvy" version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + [[package]] name = "ecdsa" version = "0.16.9" @@ -783,6 +869,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "fnv" version = "1.0.7" @@ -918,6 +1010,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -956,6 +1057,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gjson" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43503cc176394dd30a6525f5f36e838339b8b5619be33ed9a7783841580a97b6" + [[package]] name = "group" version = "0.13.0" @@ -967,25 +1074,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.12" @@ -997,7 +1085,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.4.0", + "http", "indexmap", "slab", "tokio", @@ -1143,29 +1231,53 @@ dependencies = [ "digest", ] +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + +[[package]] +name = "html2md" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cff9891f2e0d9048927fbdfc28b11bf378f6a93c7ba70b23d0fbee9af6071b4" +dependencies = [ + "html5ever 0.27.0", + "jni", + "lazy_static", + "markup5ever_rcdom", + "percent-encoding", + "regex", +] + [[package]] name = "html5ever" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" +checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4" dependencies = [ "log", "mac", - "markup5ever", + "markup5ever 0.12.1", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.111", ] [[package]] -name = "http" -version = "0.2.12" +name = "html5ever" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c" dependencies = [ - "bytes", - "fnv", - "itoa", + "log", + "mac", + "markup5ever 0.14.1", + "match_token", ] [[package]] @@ -1178,17 +1290,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -1196,7 +1297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.0", + "http", ] [[package]] @@ -1207,8 +1308,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.0", - "http-body 1.0.1", + "http", + "http-body", "pin-project-lite", ] @@ -1224,30 +1325,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.8.1" @@ -1258,9 +1335,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.12", - "http 1.4.0", - "http-body 1.0.1", + "h2", + "http", + "http-body", "httparse", "httpdate", "itoa", @@ -1277,8 +1354,8 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http 1.4.0", - "hyper 1.8.1", + "http", + "hyper", "hyper-util", "rustls", "rustls-pki-types", @@ -1288,19 +1365,6 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper 0.14.32", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "hyper-tls" version = "0.6.0" @@ -1309,7 +1373,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-util", "native-tls", "tokio", @@ -1323,20 +1387,20 @@ version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-core", "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "hyper 1.8.1", + "http", + "http-body", + "hyper", "ipnet", "libc", "percent-encoding", "pin-project-lite", "socket2 0.6.1", - "system-configuration 0.6.1", + "system-configuration", "tokio", "tower-service", "tracing", @@ -1530,6 +1594,26 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "jni" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec" +dependencies = [ + "cesu8", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.34" @@ -1582,7 +1666,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags 2.10.0", + "bitflags", "libc", "redox_syscall 0.7.0", ] @@ -1656,9 +1740,12 @@ version = "0.1.0" dependencies = [ "chrono", "clap", + "dom_smoothie", "dotenvy", + "html2md", "malfestio-core", "malfestio-server", + "reqwest", "tokio", "tokio-postgres", ] @@ -1681,17 +1768,18 @@ dependencies = [ "async-trait", "atproto-jetstream", "axum", - "base64 0.22.1", + "base64", "chrono", "deadpool-postgres", + "dom_smoothie", "dotenvy", "ed25519-dalek", "getrandom 0.3.4", "hickory-resolver 0.24.4", + "html2md", "malfestio-core", - "readability", "regex", - "reqwest 0.12.28", + "reqwest", "serde", "serde_json", "serde_qs", @@ -1710,12 +1798,26 @@ dependencies = [ [[package]] name = "markup5ever" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" +checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45" dependencies = [ "log", - "phf 0.10.1", + "phf 0.11.3", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "markup5ever" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18" +dependencies = [ + "log", + "phf 0.11.3", "phf_codegen", "string_cache", "string_cache_codegen", @@ -1724,12 +1826,12 @@ dependencies = [ [[package]] name = "markup5ever_rcdom" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9521dd6750f8e80ee6c53d65e2e4656d7de37064f3a7a5d2d11d05df93839c2" +checksum = "edaa21ab3701bfee5099ade5f7e1f84553fd19228cf332f13cd6e964bf59be18" dependencies = [ - "html5ever", - "markup5ever", + "html5ever 0.27.0", + "markup5ever 0.12.1", "tendril", "xml5ever", ] @@ -1745,6 +1847,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "match_token" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "matchers" version = "0.2.0" @@ -1912,7 +2025,7 @@ version = "0.10.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ - "bitflags 2.10.0", + "bitflags", "cfg-if", "foreign-types", "libc", @@ -2014,11 +2127,12 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "phf" -version = "0.10.1" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_shared 0.10.0", + "phf_macros", + "phf_shared 0.11.3", ] [[package]] @@ -2033,22 +2147,12 @@ dependencies = [ [[package]] name = "phf_codegen" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" -dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", -] - -[[package]] -name = "phf_generator" -version = "0.10.0" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_shared 0.10.0", - "rand 0.8.5", + "phf_generator", + "phf_shared 0.11.3", ] [[package]] @@ -2062,12 +2166,16 @@ dependencies = [ ] [[package]] -name = "phf_shared" -version = "0.10.0" +name = "phf_macros" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" dependencies = [ - "siphasher 0.3.11", + "phf_generator", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn 2.0.111", ] [[package]] @@ -2076,7 +2184,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ - "siphasher 1.0.1", + "siphasher", ] [[package]] @@ -2085,7 +2193,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" dependencies = [ - "siphasher 1.0.1", + "siphasher", ] [[package]] @@ -2128,7 +2236,7 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbef655056b916eb868048276cfd5d6a7dea4f81560dfd047f97c8c6fe3fcfd4" dependencies = [ - "base64 0.22.1", + "base64", "byteorder", "bytes", "fallible-iterator", @@ -2332,27 +2440,13 @@ dependencies = [ "getrandom 0.3.4", ] -[[package]] -name = "readability" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56596e20a6d3cf715182d9b6829220621e6e985cec04d00410cee29821b4220" -dependencies = [ - "html5ever", - "lazy_static", - "markup5ever_rcdom", - "regex", - "reqwest 0.11.27", - "url", -] - [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.10.0", + "bitflags", ] [[package]] @@ -2361,7 +2455,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" dependencies = [ - "bitflags 2.10.0", + "bitflags", ] [[package]] @@ -2393,63 +2487,23 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" -[[package]] -name = "reqwest" -version = "0.11.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "base64 0.21.7", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper-tls 0.5.0", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper 0.1.2", - "system-configuration 0.5.1", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", -] - [[package]] name = "reqwest" version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "encoding_rs", "futures-core", - "h2 0.4.12", - "http 1.4.0", - "http-body 1.0.1", + "h2", + "http", + "http-body", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-rustls", - "hyper-tls 0.6.0", + "hyper-tls", "hyper-util", "js-sys", "log", @@ -2463,7 +2517,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tokio-native-tls", "tokio-rustls", @@ -2528,7 +2582,7 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ - "bitflags 2.10.0", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -2561,15 +2615,6 @@ dependencies = [ "security-framework 3.5.1", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pki-types" version = "1.13.2" @@ -2603,6 +2648,15 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.28" @@ -2639,7 +2693,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.10.0", + "bitflags", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -2652,7 +2706,7 @@ version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ - "bitflags 2.10.0", + "bitflags", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -2669,6 +2723,25 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "new_debug_unreachable", + "phf 0.11.3", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "semver" version = "1.0.27" @@ -2784,6 +2857,15 @@ dependencies = [ "serde", ] +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha2" version = "0.10.9" @@ -2836,12 +2918,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - [[package]] name = "siphasher" version = "1.0.1" @@ -2915,7 +2991,7 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" dependencies = [ - "phf_generator 0.11.3", + "phf_generator", "phf_shared 0.11.3", "proc-macro2", "quote", @@ -2966,12 +3042,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "sync_wrapper" version = "1.0.2" @@ -2992,36 +3062,15 @@ dependencies = [ "syn 2.0.111", ] -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation 0.9.4", - "system-configuration-sys 0.5.0", -] - [[package]] name = "system-configuration" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 2.10.0", + "bitflags", "core-foundation 0.9.4", - "system-configuration-sys 0.6.0", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", + "system-configuration-sys", ] [[package]] @@ -3263,11 +3312,11 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fcaf159b4e7a376b05b5bfd77bfd38f3324f5fce751b4213bfc7eaa47affb4e" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-core", "futures-sink", - "http 1.4.0", + "http", "httparse", "rand 0.9.2", "ring", @@ -3288,7 +3337,7 @@ dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tower-layer", "tower-service", @@ -3304,7 +3353,7 @@ dependencies = [ "axum-core", "cookie", "futures-util", - "http 1.4.0", + "http", "parking_lot", "pin-project-lite", "tower-layer", @@ -3317,11 +3366,11 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags 2.10.0", + "bitflags", "bytes", "futures-util", - "http 1.4.0", - "http-body 1.0.1", + "http", + "http-body", "iri-string", "pin-project-lite", "tower", @@ -3443,6 +3492,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unsigned-varint" version = "0.8.0" @@ -3479,6 +3534,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3521,6 +3582,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -3655,6 +3726,15 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -3971,13 +4051,13 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "xml5ever" -version = "0.17.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4034e1d05af98b51ad7214527730626f019682d797ba38b51689212118d8e650" +checksum = "9bbb26405d8e919bc1547a5aa9abc95cbfa438f04844f5fdd9dc7596b748bf69" dependencies = [ "log", "mac", - "markup5ever", + "markup5ever 0.12.1", ] [[package]] diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 297dbae..6c78435 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -6,8 +6,11 @@ edition = "2024" [dependencies] chrono = "0.4" clap = { version = "4.5.53", features = ["derive"] } +dom_smoothie = "0.4" dotenvy = "0.15.7" +html2md = "0.2.15" malfestio-core = { version = "0.1.0", path = "../core" } malfestio-server = { version = "0.1.0", path = "../server" } +reqwest = { version = "0.12", features = ["json"] } tokio = { version = "1.48.0", features = ["full"] } tokio-postgres = "0.7.13" diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index b72a646..7f1f717 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -28,6 +28,25 @@ enum Commands { /// Bluesky handle to test (e.g., alice.bsky.social) handle: String, }, + #[cfg(debug_assertions)] + /// [DEBUG ONLY] Debug utilities + Debug { + #[command(subcommand)] + command: DebugCommands, + }, +} + +#[cfg(debug_assertions)] +#[derive(Subcommand)] +enum DebugCommands { + /// Test article extraction and markdown conversion + Article { + /// Article URL to extract + url: String, + /// Save to file instead of printing to terminal + #[arg(short, long)] + output: Option, + }, } #[tokio::main] @@ -47,6 +66,12 @@ async fn main() -> malfestio_core::Result<()> { Commands::Check { handle } => { check_flow(handle).await?; } + #[cfg(debug_assertions)] + Commands::Debug { command } => match command { + DebugCommands::Article { url, output } => { + debug_article(url, output.as_deref()).await?; + } + }, } Ok(()) @@ -289,3 +314,101 @@ fn format_time_ago(timestamp: chrono::DateTime) -> String { format!("{} months ago", duration.num_days() / 30) } } + +#[cfg(debug_assertions)] +async fn debug_article(url: &str, output_file: Option<&str>) -> malfestio_core::Result<()> { + use dom_smoothie::Readability; + + println!("Fetching article from: {}", url); + + // Fetch HTML content with user-agent + let client = reqwest::Client::builder() + .user_agent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + .build() + .map_err(|e| malfestio_core::Error::Other(format!("Failed to build client: {}", e)))?; + + let response = client.get(url) + .send() + .await + .map_err(|e| malfestio_core::Error::Other(format!("Failed to fetch URL: {}", e)))?; + + let html_content = response + .text() + .await + .map_err(|e| malfestio_core::Error::Other(format!("Failed to read response: {}", e)))?; + + println!("Fetched {} bytes of HTML", html_content.len()); + + // Extract article using dom_smoothie + println!("Extracting article content..."); + let url_clone = url.to_string(); + let result = tokio::task::spawn_blocking( + move || -> Result<(String, String, Option, Option), String> { + let mut readability = Readability::new(html_content, Some(&url_clone), None) + .map_err(|e| format!("Readability error: {}", e))?; + let article = readability.parse().map_err(|e| format!("Parse error: {}", e))?; + Ok(( + article.title, + article.content.to_string(), + article.byline, + article.published_time, + )) + }, + ) + .await + .map_err(|e| malfestio_core::Error::Other(format!("Task join error: {}", e)))? + .map_err(malfestio_core::Error::Other)?; + + let (title, content, author, publish_date) = result; + + println!("✓ Extracted article:"); + println!(" Title: {}", title); + if let Some(author) = &author { + println!(" Author: {}", author); + } + if let Some(date) = &publish_date { + println!(" Published: {}", date); + } + println!(" Content length: {} bytes", content.len()); + + // Convert HTML to markdown + println!("\nConverting to markdown..."); + let markdown = html2md::parse_html(&content); + println!("✓ Converted to {} bytes of markdown", markdown.len()); + + // Output + if let Some(file_path) = output_file { + println!("\nSaving to file: {}", file_path); + + let mut output = String::new(); + output.push_str(&format!("# {}\n\n", title)); + if let Some(author) = author { + output.push_str(&format!("**Author:** {}\n", author)); + } + if let Some(date) = publish_date { + output.push_str(&format!("**Published:** {}\n", date)); + } + output.push_str(&format!("**Source:** {}\n\n", url)); + output.push_str("---\n\n"); + output.push_str(&markdown); + + fs::write(file_path, output) + .map_err(|e| malfestio_core::Error::Other(format!("Failed to write file: {}", e)))?; + + println!("✓ Saved to {}", file_path); + } else { + println!("\n{}", "=".repeat(80)); + println!("# {}", title); + if let Some(author) = author { + println!("\n**Author:** {}", author); + } + if let Some(date) = publish_date { + println!("**Published:** {}", date); + } + println!("**Source:** {}", url); + println!("{}", "=".repeat(80)); + println!("\n{}", markdown); + } + + Ok(()) +} diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index ec597e5..67d0614 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -13,11 +13,12 @@ chrono = { version = "0.4.42", features = ["serde"] } deadpool-postgres = "0.14.0" dotenvy = "0.15.7" ed25519-dalek = { version = "2.2.0", features = ["serde"] } +dom_smoothie = "0.4" getrandom = { version = "0.3", features = ["std"] } +html2md = "0.2.15" malfestio-core = { version = "0.1.0", path = "../core" } -readability = "0.3.0" regex = "1.12.2" -reqwest = { version = "0.12.28", features = ["json"] } +reqwest = { version = "0.12", features = ["json"] } serde = "1.0.228" serde_json = "1.0.148" serde_qs = "0.15" @@ -36,4 +37,4 @@ tower-http = { version = "0.6.8", features = ["cors", "trace"] } tracing = "0.1.44" tracing-subscriber = { version = "0.3.22", features = ["env-filter"] } uuid = { version = "1.19.0", features = ["v4", "fast-rng"] } -hickory-resolver = "0.24.0" +hickory-resolver = "0.24" diff --git a/crates/server/src/api/importer.rs b/crates/server/src/api/importer.rs index c7759b4..96388d7 100644 --- a/crates/server/src/api/importer.rs +++ b/crates/server/src/api/importer.rs @@ -1,6 +1,9 @@ -use axum::{Json, http::StatusCode, response::IntoResponse}; -use readability::extractor; -use serde::Deserialize; +use crate::middleware::auth::UserContext; +use crate::state::SharedState; +use axum::{Json, extract::Extension, http::StatusCode, response::IntoResponse}; +use dom_smoothie::Readability; +use malfestio_core::model::Visibility; +use serde::{Deserialize, Serialize}; use serde_json::json; #[derive(Deserialize)] @@ -8,27 +11,186 @@ pub struct ImportRequest { url: String, } +#[derive(Serialize)] +pub struct ArticleMetadata { + #[serde(skip_serializing_if = "Option::is_none")] + author: Option, + #[serde(skip_serializing_if = "Option::is_none")] + publish_date: Option, + source_url: String, +} + +#[derive(Serialize)] +pub struct ImportArticleResponse { + title: String, + markdown: String, + metadata: ArticleMetadata, +} + pub async fn import_article(Json(payload): Json) -> impl IntoResponse { if payload.url.trim().is_empty() { return (StatusCode::BAD_REQUEST, Json(json!({"error": "URL is required"}))).into_response(); } let url = payload.url.clone(); + + // Fetch HTML content + let html_result = reqwest::get(&url).await; + let html_content = match html_result { + Ok(response) => match response.text().await { + Ok(text) => text, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": format!("Failed to fetch content: {}", e)})), + ) + .into_response(); + } + }, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": format!("Failed to fetch URL: {}", e)})), + ) + .into_response(); + } + }; + + // Extract article using dom_smoothie let url_for_task = url.clone(); + let result = tokio::task::spawn_blocking( + move || -> Result<(String, String, Option, Option), String> { + let mut readability = Readability::new(html_content, Some(&url_for_task), None) + .map_err(|e| format!("Readability error: {}", e))?; + let article = readability.parse().map_err(|e| format!("Parse error: {}", e))?; + Ok(( + article.title, + article.content.to_string(), + article.byline, + article.published_time, + )) + }, + ) + .await; + + match result { + Ok(Ok((title, content, author, publish_date))) => { + // Convert HTML content to markdown + let markdown = html2md::parse_html(&content); + + let response = ImportArticleResponse { + title, + markdown, + metadata: ArticleMetadata { author, publish_date, source_url: payload.url }, + }; + + Json(response).into_response() + } + Ok(Err(e)) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": format!("Failed to extract article: {}", e)})), + ) + .into_response(), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": format!("Task join error: {}", e)})), + ) + .into_response(), + } +} + +#[derive(Deserialize)] +pub struct ImportArticleSaveRequest { + url: String, + #[serde(default)] + tags: Vec, + #[serde(default = "default_visibility")] + visibility: Visibility, +} - let result = tokio::task::spawn_blocking(move || extractor::scrape(&url_for_task)).await; +fn default_visibility() -> Visibility { + Visibility::Private +} + +pub async fn import_article_save( + Extension(user_ctx): Extension, axum::extract::State(state): axum::extract::State, + Json(payload): Json, +) -> impl IntoResponse { + if payload.url.trim().is_empty() { + return (StatusCode::BAD_REQUEST, Json(json!({"error": "URL is required"}))).into_response(); + } + + let url = payload.url.clone(); + + // Fetch HTML content + let html_result = reqwest::get(&url).await; + let html_content = match html_result { + Ok(response) => match response.text().await { + Ok(text) => text, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": format!("Failed to fetch content: {}", e)})), + ) + .into_response(); + } + }, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": format!("Failed to fetch URL: {}", e)})), + ) + .into_response(); + } + }; + + // Extract article using dom_smoothie + let url_for_task = url.clone(); + let result = tokio::task::spawn_blocking(move || -> Result<(String, String), String> { + let mut readability = Readability::new(html_content, Some(&url_for_task), None) + .map_err(|e| format!("Readability error: {}", e))?; + let article = readability.parse().map_err(|e| format!("Parse error: {}", e))?; + Ok((article.title, article.content.to_string())) + }) + .await; match result { - Ok(Ok(product)) => Json(json!({ - "title": product.title, - "content": product.content, - "text": product.text, - "url": url - })) - .into_response(), + Ok(Ok((title, content))) => { + // Convert HTML content to markdown + let markdown = html2md::parse_html(&content); + + // Merge auto-tags with user-provided tags + let mut tags = payload.tags.clone(); + if !tags.contains(&"imported".to_string()) { + tags.push("imported".to_string()); + } + if !tags.contains(&"article".to_string()) { + tags.push("article".to_string()); + } + + // Store source URL as first link + let links = vec![payload.url.clone()]; + + // Create note + match state + .note_repo + .create(&user_ctx.did, &title, &markdown, tags, payload.visibility, links) + .await + { + Ok(note) => (StatusCode::CREATED, Json(note)).into_response(), + Err(e) => { + tracing::error!("Failed to create note from import: {:?}", e); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": "Failed to save article"})), + ) + .into_response() + } + } + } Ok(Err(e)) => ( StatusCode::INTERNAL_SERVER_ERROR, - Json(json!({"error": format!("Readability error: {}", e)})), + Json(json!({"error": format!("Failed to extract article: {}", e)})), ) .into_response(), Err(e) => ( @@ -60,7 +222,18 @@ mod tests { let body_json: serde_json::Value = serde_json::from_slice(&body_bytes).unwrap(); let title = body_json["title"].as_str().unwrap(); assert!(title.contains("Rust")); - assert!(body_json["text"].as_str().unwrap().len() > 100); + // Verify markdown field exists and is non-empty + let markdown = body_json["markdown"].as_str().unwrap(); + assert!(markdown.len() > 100); + // Verify no HTML tags leak through + assert!(!markdown.contains("")); + // Verify metadata structure exists + assert!(body_json["metadata"].is_object()); + assert_eq!( + body_json["metadata"]["source_url"].as_str().unwrap(), + "https://www.rust-lang.org" + ); } #[tokio::test] diff --git a/crates/server/src/api/note.rs b/crates/server/src/api/note.rs index a1f81fa..e3ac735 100644 --- a/crates/server/src/api/note.rs +++ b/crates/server/src/api/note.rs @@ -36,6 +36,7 @@ pub async fn create_note( &payload.body, payload.tags, payload.visibility, + Vec::new(), ) .await; diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 5e88499..9a78847 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -57,6 +57,7 @@ pub async fn start() -> malfestio_core::Result<()> { .route("/decks/{id}/fork", post(api::deck::fork_deck)) .route("/notes", post(api::note::create_note)) .route("/cards", post(api::card::create_card)) + .route("/import/article/save", post(api::importer::import_article_save)) .route("/review/due", get(api::review::get_due_cards)) .route("/review/submit", post(api::review::submit_review)) .route("/review/stats", get(api::review::get_stats)) diff --git a/crates/server/src/repository/note.rs b/crates/server/src/repository/note.rs index 3c93793..dc6e198 100644 --- a/crates/server/src/repository/note.rs +++ b/crates/server/src/repository/note.rs @@ -12,7 +12,13 @@ pub enum NoteRepoError { #[async_trait] pub trait NoteRepository: Send + Sync { async fn create( - &self, owner_did: &str, title: &str, body: &str, tags: Vec, visibility: Visibility, + &self, + owner_did: &str, + title: &str, + body: &str, + tags: Vec, + visibility: Visibility, + links: Vec, ) -> Result; async fn list(&self, viewer_did: Option<&str>) -> Result, NoteRepoError>; async fn get(&self, id: &str, viewer_did: Option<&str>) -> Result; @@ -32,7 +38,13 @@ impl DbNoteRepository { #[async_trait] impl NoteRepository for DbNoteRepository { async fn create( - &self, owner_did: &str, title: &str, body: &str, tags: Vec, visibility: Visibility, + &self, + owner_did: &str, + title: &str, + body: &str, + tags: Vec, + visibility: Visibility, + links: Vec, ) -> Result { let client = self .pool @@ -46,9 +58,9 @@ impl NoteRepository for DbNoteRepository { client .execute( - "INSERT INTO notes (id, owner_did, title, body, tags, visibility) - VALUES ($1, $2, $3, $4, $5, $6)", - &[¬e_id, &owner_did, &title, &body, &tags, &visibility_json], + "INSERT INTO notes (id, owner_did, title, body, tags, visibility, links) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + &[¬e_id, &owner_did, &title, &body, &tags, &visibility_json, &links], ) .await .map_err(|e| NoteRepoError::DatabaseError(format!("Failed to insert note: {}", e)))?; @@ -61,7 +73,7 @@ impl NoteRepository for DbNoteRepository { tags, visibility, published_at: None, - links: Vec::new(), + links, language: None, }) } @@ -257,7 +269,13 @@ pub mod mock { #[async_trait] impl NoteRepository for MockNoteRepository { async fn create( - &self, owner_did: &str, title: &str, body: &str, tags: Vec, visibility: Visibility, + &self, + owner_did: &str, + title: &str, + body: &str, + tags: Vec, + visibility: Visibility, + links: Vec, ) -> Result { if *self.should_fail.lock().unwrap() { return Err(NoteRepoError::DatabaseError("Mock failure".to_string())); @@ -271,7 +289,7 @@ pub mod mock { tags, visibility, published_at: None, - links: Vec::new(), + links, language: None, }; diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 175dd6a..1cdf86b 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -58,12 +58,14 @@ export const api = { const deck = await res.json(); if (cards && cards.length > 0) { - await Promise.all(cards.map((c) => - apiFetch("/cards", { - method: "POST", - body: JSON.stringify({ deck_id: deck.id, front: c.front, back: c.back, media_url: c.mediaUrl }), - }) - )); + await Promise.all( + cards.map((c) => + apiFetch("/cards", { + method: "POST", + body: JSON.stringify({ deck_id: deck.id, front: c.front, back: c.back, media_url: c.mediaUrl }), + }) + ), + ); } return { ok: true, json: async () => deck }; @@ -99,4 +101,19 @@ export const api = { } return res; }, + // TODO: type visibility + saveImportedArticle: (payload: { url: string; tags?: string[]; visibility?: unknown }) => { + return apiFetch("/import/article/save", { method: "POST", body: JSON.stringify(payload) }); + }, + downloadNoteAsMarkdown: (note: { title: string; body: string }) => { + const blob = new Blob([note.body], { type: "text/markdown" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `${note.title.replace(/[^a-z0-9]/gi, "_").toLowerCase()}.md`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + }, }; diff --git a/web/src/pages/Import.tsx b/web/src/pages/Import.tsx index 5f4a561..3f7cb22 100644 --- a/web/src/pages/Import.tsx +++ b/web/src/pages/Import.tsx @@ -2,24 +2,30 @@ import { NoteEditor } from "$components/NoteEditor"; import { api } from "$lib/api"; import { toast } from "$lib/toast"; import { Button } from "$ui/Button"; +import { useNavigate } from "@solidjs/router"; import { createSignal, Show } from "solid-js"; export default function Import() { + const navigate = useNavigate(); const [url, setUrl] = createSignal(""); const [loading, setLoading] = createSignal(false); - const [importedData, setImportedData] = createSignal<{ title: string; content: string } | null>(null); + const [saving, setSaving] = createSignal(false); + const [showEditor, setShowEditor] = createSignal(false); + const [importedData, setImportedData] = createSignal< + { title: string; markdown: string; metadata: { author?: string; publish_date?: string; source_url: string } } | null + >(null); const handleImport = async (e: Event) => { e.preventDefault(); setLoading(true); setImportedData(null); + setShowEditor(false); try { const res = await api.post("/import/article", { url: url() }); if (res.ok) { const data = await res.json(); - const content = `Source: [${data.title}](${data.url})\n\n${data.text}`; - setImportedData({ title: data.title, content }); - toast.success("Article imported!"); + setImportedData({ title: data.title, markdown: data.markdown, metadata: data.metadata }); + toast.success("Article extracted!"); } else { const err = await res.json(); toast.error(err.error || "Failed to import"); @@ -32,6 +38,31 @@ export default function Import() { } }; + const handleQuickSave = async () => { + if (!importedData()) return; + setSaving(true); + try { + const res = await api.saveImportedArticle({ + url: importedData()!.metadata.source_url, + tags: ["imported", "article"], + visibility: { type: "Private" }, + }); + if (res.ok) { + const note = await res.json(); + toast.success("Article saved!"); + navigate(`/notes/${note.id}`); + } else { + const err = await res.json(); + toast.error(err.error || "Failed to save"); + } + } catch (e) { + console.error(e); + toast.error("Network error"); + } finally { + setSaving(false); + } + }; + return (
@@ -50,10 +81,39 @@ export default function Import() { - + +
+
+

{importedData()!.title}

+ +
+ + By {importedData()!.metadata.author} + + + Published: {new Date(importedData()!.metadata.publish_date!).toLocaleDateString()} + +
+
+
+ +
+

+ {importedData()!.markdown.slice(0, 500)}... +

+
+ +
+ + +
+
+
+ +
-

Create Note from Import

- +

Edit Before Saving

+
diff --git a/web/src/pages/NoteView.tsx b/web/src/pages/NoteView.tsx index e16e5b3..51d595c 100644 --- a/web/src/pages/NoteView.tsx +++ b/web/src/pages/NoteView.tsx @@ -5,14 +5,8 @@ import { WikilinksPanel } from "$components/notes/WikilinksPanel"; import { Button } from "$components/ui/Button"; import { api } from "$lib/api"; import type { Note } from "$lib/model"; -import { - extractHeadings, - findBacklinks, - type Heading, - parseWikilinks, - resolveWikilink, - type WikiLink, -} from "$lib/wikilink"; +import { extractHeadings, findBacklinks, parseWikilinks, resolveWikilink } from "$lib/wikilink"; +import type { Heading, WikiLink } from "$lib/wikilink"; import { Tag } from "$ui/Tag"; import rehypeShiki from "@shikijs/rehype"; import { A, useParams } from "@solidjs/router"; @@ -118,6 +112,7 @@ const NoteView: Component = () => { +