diff --git a/.dockerignore b/.dockerignore index ec5820d..930bfc0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,9 @@ -.vscode -/target -.git -.cache -Dockerfile -/tests \ No newline at end of file +.vscode +/target +.git +.cache +Dockerfile +/tests + +__pycache__/ +*.py[cod] diff --git a/.env b/.env index 5231ff6..5159700 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ -DATABASE_URL=sqlite://db.sqlite -RUST_LOG=neko_server +DATABASE_URL=sqlite://db.sqlite +RUST_LOG=neko_server NEKO_SERVER=http://34.149.0.8 \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..64db245 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +* text=auto eol=lf + +*.png binary +*.ttf binary diff --git a/.gitignore b/.gitignore index 3e791e6..f9d7bf9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ -/target -image.png -template/**.psd \ No newline at end of file +/target +image.png +template/**.psd + +__pycache__/ +*.py[cod] diff --git a/.vscode/launch.json b/.vscode/launch.json index fc197c1..3bb0ae2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,45 +1,45 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "type": "lldb", - "request": "launch", - "name": "Debug executable 'NekoServer'", - "cargo": { - "args": [ - "build", - "--bin=NekoServer", - "--package=NekoServer" - ], - "filter": { - "name": "NekoServer", - "kind": "bin" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "Debug unit tests in executable 'NekoServer'", - "cargo": { - "args": [ - "test", - "--no-run", - "--bin=NekoServer", - "--package=NekoServer" - ], - "filter": { - "name": "NekoServer", - "kind": "bin" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - } - ] +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'NekoServer'", + "cargo": { + "args": [ + "build", + "--bin=NekoServer", + "--package=NekoServer" + ], + "filter": { + "name": "NekoServer", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in executable 'NekoServer'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=NekoServer", + "--package=NekoServer" + ], + "filter": { + "name": "NekoServer", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + } + ] } \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index b5e753c..ade403e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,7 +95,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -106,7 +106,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -327,6 +327,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.44" @@ -625,7 +631,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -766,6 +772,19 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -773,9 +792,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasip2", + "wasm-bindgen", ] [[package]] @@ -918,6 +939,23 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", ] [[package]] @@ -926,13 +964,21 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64", "bytes", + "futures-channel", + "futures-util", "http", "http-body", "hyper", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", + "socket2", "tokio", "tower-service", + "tracing", ] [[package]] @@ -1123,6 +1169,12 @@ dependencies = [ "syn", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1183,7 +1235,7 @@ version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom", + "getrandom 0.3.4", "libc", ] @@ -1257,6 +1309,12 @@ dependencies = [ "imgref", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -1307,7 +1365,7 @@ checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1332,6 +1390,10 @@ dependencies = [ "lazy_static", "log", "redis", + "reqwest", + "serde", + "serde_json", + "sha2", "signal-hook", "tokio", "tokio-util", @@ -1642,6 +1704,61 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.45" @@ -1683,7 +1800,7 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ - "getrandom", + "getrandom 0.3.4", ] [[package]] @@ -1822,12 +1939,105 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + [[package]] name = "rgb" version = "0.8.53" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -1933,6 +2143,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1993,7 +2214,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2008,6 +2229,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.117" @@ -2019,6 +2246,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -2084,6 +2320,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.52.3" @@ -2098,7 +2349,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2112,6 +2363,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -2125,6 +2386,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -2151,6 +2451,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.20.0" @@ -2169,6 +2475,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -2220,6 +2532,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "warp" version = "0.4.3" @@ -2277,6 +2598,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.121" @@ -2319,6 +2650,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "weezl" version = "0.1.12" @@ -2347,7 +2697,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2415,6 +2765,24 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -2424,6 +2792,135 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -2512,6 +3009,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + [[package]] name = "zerotrie" version = "0.2.4" diff --git a/Cargo.toml b/Cargo.toml index a1e6390..8f47ca6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,13 @@ env_logger = "0.11" image = "0.25" # Caching lazy_static = "1.5.*" +# JSON API support +serde = { version = "1", features = ["derive"] } +serde_json = "1" +# Cache key hashing +sha2 = "0.10" +# HTTP client for gallery-dl worker +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } # Logging log = "0.4" # Interact with Redis Database diff --git a/Dockerfile b/Dockerfile index 7bfad90..19e8afb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,29 @@ -# Rust build environment -FROM rust:alpine AS builder - -# Install dependencies -RUN apk add --no-cache \ - musl-dev \ - build-base - -WORKDIR /usr/src/neko_server - -# Cache dependencies -RUN echo "fn main() {}" > dummy.rs -COPY Cargo.toml . -RUN sed -i 's#src/main.rs#dummy.rs#' Cargo.toml -RUN cargo build --release -RUN sed -i 's#dummy.rs#src/main.rs#' Cargo.toml - -# Build -COPY . . -RUN cargo install --path . - -# Run Server -FROM alpine:latest -EXPOSE 80 - -# Copy binary from builder -COPY --from=builder /usr/local/cargo/bin/neko_server /usr/local/bin - -CMD ["neko_server"] \ No newline at end of file +# Rust build environment +FROM rust:alpine AS builder + +# Install dependencies +RUN apk add --no-cache \ + musl-dev \ + build-base + +WORKDIR /usr/src/neko_server + +# Cache dependencies +RUN echo "fn main() {}" > dummy.rs +COPY Cargo.toml . +RUN sed -i 's#src/main.rs#dummy.rs#' Cargo.toml +RUN cargo build --release +RUN sed -i 's#dummy.rs#src/main.rs#' Cargo.toml + +# Build +COPY . . +RUN cargo install --path . + +# Run Server +FROM alpine:latest +EXPOSE 80 + +# Copy binary from builder +COPY --from=builder /usr/local/cargo/bin/neko_server /usr/local/bin + +CMD ["neko_server"] diff --git a/README.md b/README.md index eed8f9a..a044495 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,68 @@ -# Neko Fans Server -[![Better Uptime Badge](https://betteruptime.com/status-badges/v1/monitor/iomh.svg)](https://status.Nekofans.net) - -This Server keeps track of how many images were displayed with the [Neko Fans](https://github.com/Meisterlala/NekoFans) plugin. - -## API - -[![Run in Postman](https://run.pstmn.io/button.svg)](https://app.getpostman.com/run-collection/23047093-1b309b0f-b56c-404e-9ff0-7321b09ae2c2?action=collection%2Ffork&collection-url=entityId%3D23047093-1b309b0f-b56c-404e-9ff0-7321b09ae2c2%26entityType%3Dcollection%26workspaceId%3D3f8f09fb-fa15-4552-bd71-a6644cd4e11e) -[![Open API](https://img.shields.io/badge/Open%20API%203.0.0-try%20it%20out-green?style=for-the-badge&logo=swagger)](https://app.swaggerhub.com/apis-docs/Meisterlala/Neko-Server) - - -| `https://api.nekofans.net/count_total` Generates the total download count from every user | -| --- | -| ![total](https://api.nekofans.net/count_total) | - -| | `https://api.nekofans.net/count/123` Generates an image with that number | -| --- | --- | -| 420 | ![420](https://api.nekofans.net/count/420) | -| 69 | ![69](https://api.nekofans.net/count/69) | -| 1337 | ![1337](https://api.nekofans.net/count/1337) | -| 314159 | ![314159](https://api.nekofans.net/count/314159) | - -## The technology used to run the server - -Neko Server is written in [Rust](https://www.rust-lang.org/) with the asyc framework [Tokio](https://tokio.rs/) and using [Warp](https://github.com/seanmonstar/warp) as a web server. This is then all bundled in a [Docker container](Dockerfile), which gets automaticly [build](cloudbuild.yaml) with Google Cloud Build. Those images are then pushed to a private Container repository. +# Neko Fans Server +[![Better Uptime Badge](https://betteruptime.com/status-badges/v1/monitor/iomh.svg)](https://status.Nekofans.net) + +This Server keeps track of how many images were displayed with the [Neko Fans](https://github.com/Meisterlala/NekoFans) plugin. + +## API + +[![Run in Postman](https://run.pstmn.io/button.svg)](https://app.getpostman.com/run-collection/23047093-1b309b0f-b56c-404e-9ff0-7321b09ae2c2?action=collection%2Ffork&collection-url=entityId%3D23047093-1b309b0f-b56c-404e-9ff0-7321b09ae2c2%26entityType%3Dcollection%26workspaceId%3D3f8f09fb-fa15-4552-bd71-a6644cd4e11e) +[![Open API](https://img.shields.io/badge/Open%20API%203.0.0-try%20it%20out-green?style=for-the-badge&logo=swagger)](https://app.swaggerhub.com/apis-docs/Meisterlala/Neko-Server) + + +| `https://api.nekofans.net/count_total` Generates the total download count from every user | +| --- | +| ![total](https://api.nekofans.net/count_total) | + +| | `https://api.nekofans.net/count/123` Generates an image with that number | +| --- | --- | +| 420 | ![420](https://api.nekofans.net/count/420) | +| 69 | ![69](https://api.nekofans.net/count/69) | +| 1337 | ![1337](https://api.nekofans.net/count/1337) | +| 314159 | ![314159](https://api.nekofans.net/count/314159) | + +### Gallery query API + +`POST /gallery/query` runs a constrained `gallery-dl` JSON query through the internal worker and caches the worker result in Redis. + +Request body: + +```json +{ + "source": "danbooru", + "rating": "safe", + "tags": ["cat_girl"], + "limit": 25 +} +``` + +Fields: + +| Field | Required | Description | +| --- | --- | --- | +| `source` | yes | One of `danbooru`, `gelbooru`, `safebooru`, `konachan`, or `yandere`. | +| `rating` | no | Convenience field for a booru rating tag such as `safe`, `questionable`, or `explicit`. `rating:safe` is also accepted and normalized to `safe`. The worker receives this as a normal `rating:` search tag. | +| `tags` | no | Up to 20 tag strings. Tags may contain ASCII letters, numbers, `_`, `-`, and `:`. Existing clients may still pass rating filters as tags, for example `rating:safe`. | +| `limit` | no | Number of gallery-dl items to request. Defaults to `50`; maximum is `100`. | + +At least one query term is required: either `rating` or one `tags` entry. + +Response body: + +```json +[] +``` + +The response body is exactly the JSON value returned by `gallery-dl`. + +Caching behavior: + +| Setting | Default | Description | +| --- | --- | --- | +| `GALLERY_DL_CACHE_TTL_SECONDS` | `900` | Redis result cache TTL for normalized `source` + query terms + `limit`. | +| worker lock TTL | `30` | Short Redis lock used to avoid duplicate worker calls for the same cache key. Concurrent identical misses return HTTP `202`. | + +Successful responses are always backed by Redis: either `X-Server-Cache: HIT` or the worker result is written to Redis before returning `X-Server-Cache: MISS`. `X-Server-Cache-Ttl-Seconds` reports the remaining server-side TTL. HTTP responses include `Cache-Control: no-store` because this is a POST endpoint whose response varies by request body; caching is server-side in Redis, not browser/proxy caching. + +## The technology used to run the server + +Neko Server is written in [Rust](https://www.rust-lang.org/) with the asyc framework [Tokio](https://tokio.rs/) and using [Warp](https://github.com/seanmonstar/warp) as a web server. This is then all bundled in a [Docker container](Dockerfile), which gets automaticly [build](cloudbuild.yaml) with Google Cloud Build. Those images are then pushed to a private Container repository. diff --git a/charts/neko-server/templates/deployment.yaml b/charts/neko-server/templates/deployment.yaml index eb4932c..cb53cea 100644 --- a/charts/neko-server/templates/deployment.yaml +++ b/charts/neko-server/templates/deployment.yaml @@ -48,6 +48,10 @@ spec: secretKeyRef: name: redis-secret key: redis-url + - name: GALLERY_DL_WORKER_URL + value: "http://127.0.0.1:8090/query" + - name: GALLERY_DL_CACHE_TTL_SECONDS + value: {{ .Values.galleryDl.cacheTtlSeconds | quote }} resources: limits: memory: {{ .Values.resources.limits.memory | quote }} @@ -55,3 +59,19 @@ spec: requests: memory: {{ .Values.resources.requests.memory | quote }} cpu: {{ .Values.resources.requests.cpu | quote }} + - name: gallery-dl-worker + image: "{{ .Values.galleryDlWorker.image }}:{{ .Values.galleryDlWorker.tag }}" + imagePullPolicy: {{ .Values.galleryDlWorker.pullPolicy | quote }} + env: + - name: MAX_CONCURRENT_JOBS + value: {{ .Values.galleryDlWorker.maxConcurrentJobs | quote }} + ports: + - name: gallery-worker + containerPort: 8090 + resources: + limits: + memory: 256Mi + cpu: 500m + requests: + memory: 50Mi + cpu: 50m diff --git a/charts/neko-server/values.yaml b/charts/neko-server/values.yaml index addcbe6..fa7a43b 100644 --- a/charts/neko-server/values.yaml +++ b/charts/neko-server/values.yaml @@ -6,6 +6,15 @@ image: tag: latest pullPolicy: Always +galleryDlWorker: + image: ghcr.io/meisterlala/nekoserver-gallery-dl-worker + tag: latest + pullPolicy: Always + maxConcurrentJobs: 2 + +galleryDl: + cacheTtlSeconds: 900 + configMap: RUST_LOG: "info, neko_server=trace" @@ -23,4 +32,4 @@ resources: cpu: 500m requests: memory: 50Mi - cpu: 50m \ No newline at end of file + cpu: 50m diff --git a/docker-compose.yml b/docker-compose.yml index b2a1a9c..042675b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.9" services: nekoserver: build: . @@ -6,9 +5,18 @@ services: - "8080:80" environment: REDIS_URL: redis://redis_db:6379 + GALLERY_DL_WORKER_URL: http://gallery_dl_worker:8090/query + GALLERY_DL_CACHE_TTL_SECONDS: 900 RUST_LOG: info, neko_server=trace depends_on: - "redis_db" + - "gallery_dl_worker" + gallery_dl_worker: + build: ./gallery-dl-worker + ports: + - "8090:8090" + environment: + MAX_CONCURRENT_JOBS: 2 redis_db: image: "redis:alpine" ports: diff --git a/gallery-dl-worker/Dockerfile b/gallery-dl-worker/Dockerfile new file mode 100644 index 0000000..741f7ef --- /dev/null +++ b/gallery-dl-worker/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.13-alpine + +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +WORKDIR /app +COPY worker.py /app/worker.py + +EXPOSE 8090 +CMD ["uvicorn", "worker:app", "--host", "0.0.0.0", "--port", "8090"] diff --git a/gallery-dl-worker/requirements.txt b/gallery-dl-worker/requirements.txt new file mode 100644 index 0000000..0dfc175 --- /dev/null +++ b/gallery-dl-worker/requirements.txt @@ -0,0 +1,3 @@ +gallery-dl==1.32.1 +fastapi==0.136.3 +uvicorn==0.47.0 diff --git a/gallery-dl-worker/worker.py b/gallery-dl-worker/worker.py new file mode 100755 index 0000000..2277922 --- /dev/null +++ b/gallery-dl-worker/worker.py @@ -0,0 +1,82 @@ +import asyncio +import json +import os +from typing import Annotated + +from fastapi import FastAPI, HTTPException, Response +from pydantic import BaseModel, Field, model_validator + +MAX_ARGS = 20 +MAX_ARG_LENGTH = 2048 +MAX_OUTPUT_BYTES = 25 * 1024 * 1024 +TIMEOUT_SECONDS = 20 +MAX_CONCURRENT_JOBS = int(os.environ.get("MAX_CONCURRENT_JOBS", "8")) + +app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) +job_semaphore = asyncio.Semaphore(MAX_CONCURRENT_JOBS) + + +class QueryRequest(BaseModel): + url: Annotated[str, Field(max_length=2048)] + args: Annotated[list[str], Field(max_length=MAX_ARGS)] = Field(default_factory=list) + + @model_validator(mode="after") + def validate_request(self): + if not self.url.startswith("https://"): + raise ValueError("invalid url") + if any(len(arg) > MAX_ARG_LENGTH or "\x00" in arg for arg in self.args): + raise ValueError("invalid arg") + return self + + +@app.get("/health") +async def health(): + return {"status": "ok"} + + +@app.post("/query") +async def query(request: QueryRequest): + async with job_semaphore: + output = await run_gallery_dl(request) + + if len(output) > MAX_OUTPUT_BYTES: + raise HTTPException(status_code=502, detail="gallery-dl output too large") + + try: + json.loads(output) + except json.JSONDecodeError as exc: + raise HTTPException( + status_code=502, detail="invalid gallery-dl output" + ) from exc + + return Response(content=output, media_type="application/json") + + +async def run_gallery_dl(request: QueryRequest) -> bytes: + command = [ + "gallery-dl", + "--ignore-config", + "--quiet", + "-J", + *request.args, + request.url, + ] + process = await asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + + try: + stdout, _ = await asyncio.wait_for( + process.communicate(), timeout=TIMEOUT_SECONDS + ) + except asyncio.TimeoutError as exc: + process.kill() + await process.wait() + raise HTTPException(status_code=504, detail="gallery-dl timed out") from exc + + if process.returncode != 0: + raise HTTPException(status_code=502, detail="gallery-dl failed") + + return stdout diff --git a/src/const_image.rs b/src/const_image.rs index 41c11a5..e206597 100644 --- a/src/const_image.rs +++ b/src/const_image.rs @@ -1,56 +1,56 @@ -use std::path::Path; - -macro_rules! const_image { - ($path:literal) => {{ - let p: &'static str = $path; - let file_name = Path::new(p).file_name().unwrap().to_str().unwrap(); - image::load_from_memory(include_bytes!($path)) - .expect(&format!("Could not load image from: {}", &file_name)) - .to_rgba8() - }}; -} - -lazy_static::lazy_static! { - pub static ref NUMBERS: [image::RgbaImage; 10] = [ - const_image!("../template/numbers/0.png"), - const_image!("../template/numbers/1.png"), - const_image!("../template/numbers/2.png"), - const_image!("../template/numbers/3.png"), - const_image!("../template/numbers/4.png"), - const_image!("../template/numbers/5.png"), - const_image!("../template/numbers/6.png"), - const_image!("../template/numbers/7.png"), - const_image!("../template/numbers/8.png"), - const_image!("../template/numbers/9.png") - ]; - - pub static ref HEADER: image::RgbaImage = const_image!("../template/count.png"); - - pub static ref HEADER_HALLOWEEN: image::RgbaImage = const_image!("../template/count_halloween.png"); - - pub static ref HEADER_CHRISTMAS_DAYS: [image::RgbaImage; 22] = [ - const_image!("../template/padoru/1.png"), - const_image!("../template/padoru/2.png"), - const_image!("../template/padoru/3.png"), - const_image!("../template/padoru/4.png"), - const_image!("../template/padoru/5.png"), - const_image!("../template/padoru/6.png"), - const_image!("../template/padoru/7.png"), - const_image!("../template/padoru/8.png"), - const_image!("../template/padoru/9.png"), - const_image!("../template/padoru/10.png"), - const_image!("../template/padoru/11.png"), - const_image!("../template/padoru/12.png"), - const_image!("../template/padoru/13.png"), - const_image!("../template/padoru/14.png"), - const_image!("../template/padoru/15.png"), - const_image!("../template/padoru/16.png"), - const_image!("../template/padoru/17.png"), - const_image!("../template/padoru/18.png"), - const_image!("../template/padoru/19.png"), - const_image!("../template/padoru/20.png"), - const_image!("../template/padoru/21.png"), - const_image!("../template/padoru/22.png") - ]; - pub static ref HEADER_CHRISTMAS: image::RgbaImage = const_image!("../template/padoru/christmas.png"); -} +use std::path::Path; + +macro_rules! const_image { + ($path:literal) => {{ + let p: &'static str = $path; + let file_name = Path::new(p).file_name().unwrap().to_str().unwrap(); + image::load_from_memory(include_bytes!($path)) + .expect(&format!("Could not load image from: {}", &file_name)) + .to_rgba8() + }}; +} + +lazy_static::lazy_static! { + pub static ref NUMBERS: [image::RgbaImage; 10] = [ + const_image!("../template/numbers/0.png"), + const_image!("../template/numbers/1.png"), + const_image!("../template/numbers/2.png"), + const_image!("../template/numbers/3.png"), + const_image!("../template/numbers/4.png"), + const_image!("../template/numbers/5.png"), + const_image!("../template/numbers/6.png"), + const_image!("../template/numbers/7.png"), + const_image!("../template/numbers/8.png"), + const_image!("../template/numbers/9.png") + ]; + + pub static ref HEADER: image::RgbaImage = const_image!("../template/count.png"); + + pub static ref HEADER_HALLOWEEN: image::RgbaImage = const_image!("../template/count_halloween.png"); + + pub static ref HEADER_CHRISTMAS_DAYS: [image::RgbaImage; 22] = [ + const_image!("../template/padoru/1.png"), + const_image!("../template/padoru/2.png"), + const_image!("../template/padoru/3.png"), + const_image!("../template/padoru/4.png"), + const_image!("../template/padoru/5.png"), + const_image!("../template/padoru/6.png"), + const_image!("../template/padoru/7.png"), + const_image!("../template/padoru/8.png"), + const_image!("../template/padoru/9.png"), + const_image!("../template/padoru/10.png"), + const_image!("../template/padoru/11.png"), + const_image!("../template/padoru/12.png"), + const_image!("../template/padoru/13.png"), + const_image!("../template/padoru/14.png"), + const_image!("../template/padoru/15.png"), + const_image!("../template/padoru/16.png"), + const_image!("../template/padoru/17.png"), + const_image!("../template/padoru/18.png"), + const_image!("../template/padoru/19.png"), + const_image!("../template/padoru/20.png"), + const_image!("../template/padoru/21.png"), + const_image!("../template/padoru/22.png") + ]; + pub static ref HEADER_CHRISTMAS: image::RgbaImage = const_image!("../template/padoru/christmas.png"); +} diff --git a/src/count_image.rs b/src/count_image.rs index 2f68e55..abf8884 100644 --- a/src/count_image.rs +++ b/src/count_image.rs @@ -1,111 +1,110 @@ -use image::{ImageBuffer, Rgba}; -use std::io::Write; - -use crate::const_image; -use crate::season_images; - -#[derive(Debug, Clone)] -/// This struct holds the image in memory -pub struct CountImage { - body: Vec, -} - -impl CountImage { - /// Returns a Clone of the image - pub fn get_image(&self) -> Vec { - self.body.clone() - } - - /// Returns a new CountImage - pub fn total_new() -> Self { - let inital_data = match image::open("total.png") { - Ok(data) => data.to_rgba8(), - Err(_) => image::RgbaImage::new(128, 128), - }; - CountImage { - body: CountImage::img_to_string(&inital_data), - } - } - - /// Returns a new CountImage - pub fn total_from_count(count: u128) -> Self { - let data = CountImage::create_total_image(count); - let body = CountImage::img_to_string(&data); - CountImage { body } - } - - /// Returns a new CountImage - pub fn from_count(count: u128) -> Self { - let data = CountImage::create_count_image(count); - let body = CountImage::img_to_string(&data); - CountImage { body } - } - - /// Render the total image - fn create_total_image(count: u128) -> ImageBuffer, Vec> { - // Generate number - let number = CountImage::create_count_image(count); - - // Get the seasonal image - let mut base = season_images::seasonal_count_total(); - // Overlay the number - let pos = (560, ((base.height() - number.height()) / 2) as i64); - image::imageops::overlay(&mut base, &number, pos.0, pos.1); - - base - } - - fn create_count_image(count: u128) -> ImageBuffer, Vec> { - // Convert count to char[] - let count_str = count.to_string(); - let count_len = count_str.len(); - let count_arr = count_str.chars(); - - // Create a new image with the same dimensions as the template - let number_width = const_image::NUMBERS[0].width(); - let number_height = const_image::NUMBERS[0].height(); - let mut overlay = image::RgbaImage::new(number_width * count_len as u32, number_height); - - // Write the Count onto the image - for (i, digit) in count_arr.enumerate() { - let digit = digit.to_digit(10).unwrap() as usize; - let number = &const_image::NUMBERS[digit]; - image::imageops::overlay(&mut overlay, number, (i as u32 * number_width) as i64, 0); - } - overlay - } - - // Save to Image to the disk (not really needed but nice for debugging) - /* - pub fn save_image(&self) { - let file = std::fs::OpenOptions::new() - .write(true) - .create(true) - .open("image.png") - .expect("Cant open image to write to it"); - let mut bufferd = std::io::BufWriter::new(file); - self.data - .write_to(&mut bufferd, image::ImageOutputFormat::Png) - .expect("Cant write to image"); - bufferd.flush().expect("Cant flush image"); - } - */ - - /// Convert the image to a Vec - fn img_to_string(img: &ImageBuffer, Vec>) -> Vec { - // Setup Bufferwriter with a Cursor so we have Seek - let mut buffer = std::io::BufWriter::new(std::io::Cursor::new(Vec::new())); - // Write Image to Buffer - img.write_to(&mut buffer, image::ImageFormat::Png) - .unwrap(); - buffer.flush().unwrap(); - // Extract the written data - buffer.into_inner().unwrap().into_inner() - } -} - -impl Default for CountImage { - fn default() -> Self { - Self::total_new() - } -} +use image::{ImageBuffer, Rgba}; +use std::io::Write; + +use crate::const_image; +use crate::season_images; + +#[derive(Debug, Clone)] +/// This struct holds the image in memory +pub struct CountImage { + body: Vec, +} + +impl CountImage { + /// Returns a Clone of the image + pub fn get_image(&self) -> Vec { + self.body.clone() + } + + /// Returns a new CountImage + pub fn total_new() -> Self { + let inital_data = match image::open("total.png") { + Ok(data) => data.to_rgba8(), + Err(_) => image::RgbaImage::new(128, 128), + }; + CountImage { + body: CountImage::img_to_string(&inital_data), + } + } + + /// Returns a new CountImage + pub fn total_from_count(count: u128) -> Self { + let data = CountImage::create_total_image(count); + let body = CountImage::img_to_string(&data); + CountImage { body } + } + + /// Returns a new CountImage + pub fn from_count(count: u128) -> Self { + let data = CountImage::create_count_image(count); + let body = CountImage::img_to_string(&data); + CountImage { body } + } + + /// Render the total image + fn create_total_image(count: u128) -> ImageBuffer, Vec> { + // Generate number + let number = CountImage::create_count_image(count); + + // Get the seasonal image + let mut base = season_images::seasonal_count_total(); + // Overlay the number + let pos = (560, ((base.height() - number.height()) / 2) as i64); + image::imageops::overlay(&mut base, &number, pos.0, pos.1); + + base + } + + fn create_count_image(count: u128) -> ImageBuffer, Vec> { + // Convert count to char[] + let count_str = count.to_string(); + let count_len = count_str.len(); + let count_arr = count_str.chars(); + + // Create a new image with the same dimensions as the template + let number_width = const_image::NUMBERS[0].width(); + let number_height = const_image::NUMBERS[0].height(); + let mut overlay = image::RgbaImage::new(number_width * count_len as u32, number_height); + + // Write the Count onto the image + for (i, digit) in count_arr.enumerate() { + let digit = digit.to_digit(10).unwrap() as usize; + let number = &const_image::NUMBERS[digit]; + image::imageops::overlay(&mut overlay, number, (i as u32 * number_width) as i64, 0); + } + overlay + } + + // Save to Image to the disk (not really needed but nice for debugging) + /* + pub fn save_image(&self) { + let file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .open("image.png") + .expect("Cant open image to write to it"); + let mut bufferd = std::io::BufWriter::new(file); + self.data + .write_to(&mut bufferd, image::ImageOutputFormat::Png) + .expect("Cant write to image"); + bufferd.flush().expect("Cant flush image"); + } + */ + + /// Convert the image to a Vec + fn img_to_string(img: &ImageBuffer, Vec>) -> Vec { + // Setup Bufferwriter with a Cursor so we have Seek + let mut buffer = std::io::BufWriter::new(std::io::Cursor::new(Vec::new())); + // Write Image to Buffer + img.write_to(&mut buffer, image::ImageFormat::Png).unwrap(); + buffer.flush().unwrap(); + // Extract the written data + buffer.into_inner().unwrap().into_inner() + } +} + +impl Default for CountImage { + fn default() -> Self { + Self::total_new() + } +} diff --git a/src/gallery_dl.rs b/src/gallery_dl.rs new file mode 100644 index 0000000..0d6a818 --- /dev/null +++ b/src/gallery_dl.rs @@ -0,0 +1,355 @@ +use redis::aio::ConnectionManager; +use redis::AsyncCommands; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::env; +use std::time::Duration; +use tokio::time; +use warp::http::Response; +use warp::hyper::StatusCode; + +const DEFAULT_CACHE_TTL_SECONDS: u64 = 15 * 60; +const LOCK_TTL_SECONDS: u64 = 30; +const MAX_TAGS: usize = 20; +const MAX_TAG_LENGTH: usize = 64; +const MAX_RATING_LENGTH: usize = 32; +const MAX_LIMIT: u16 = 100; +const MAX_OUTPUT_BYTES: usize = 10 * 1024 * 1024; +const COMMAND_TIMEOUT: Duration = Duration::from_secs(20); + +#[derive(Deserialize)] +pub struct GalleryQuery { + source: String, + tags: Option>, + rating: Option, + limit: Option, +} + +#[derive(Serialize)] +struct ErrorResponse { + error: String, +} + +#[derive(Serialize)] +struct WorkerRequest<'a> { + url: &'a str, + args: Vec, +} + +pub async fn query( + auth: Option, + request: GalleryQuery, + mut redis: ConnectionManager, +) -> Result { + if !is_authorized(auth.as_deref()) { + return Ok(json_error(StatusCode::UNAUTHORIZED, "unauthorized")); + } + + let cache_ttl_seconds = cache_ttl_seconds(); + let limit = request.limit.unwrap_or(50); + let source = match normalize_source(request.source) { + Ok(source) => source, + Err(message) => return Ok(json_error(StatusCode::BAD_REQUEST, message)), + }; + let query = match normalize_query(request.tags, request.rating) { + Ok(query) => query, + Err(message) => return Ok(json_error(StatusCode::BAD_REQUEST, message)), + }; + if limit == 0 || limit > MAX_LIMIT { + return Ok(json_error(StatusCode::BAD_REQUEST, "invalid limit")); + } + + let url = match build_url(&source, &query.terms, limit) { + Some(url) => url, + None => return Ok(json_error(StatusCode::BAD_REQUEST, "unsupported source")), + }; + + let cache_key = cache_key(&source, &query.terms, limit); + let cached: Result, _> = redis.get(&cache_key).await; + let cached = match cached { + Ok(value) => value, + Err(_) => { + return Ok(json_error( + StatusCode::SERVICE_UNAVAILABLE, + "cache unavailable", + )) + } + }; + if let Some(value) = cached { + if let Ok(gallery_dl) = serde_json::from_str::(&value) { + let ttl = cache_ttl(&mut redis, &cache_key, cache_ttl_seconds).await; + return Ok(json_response( + StatusCode::OK, + &gallery_dl, + Some(("HIT", ttl)), + )); + } + let _: Result<(), _> = redis.del(&cache_key).await; + } + + let lock_key = cache_key.replace("gallery:v1:result:", "gallery:v1:lock:"); + let lock: Result, _> = redis::cmd("SET") + .arg(&lock_key) + .arg("1") + .arg("NX") + .arg("EX") + .arg(LOCK_TTL_SECONDS) + .query_async(&mut redis) + .await; + match lock { + Ok(Some(_)) => {} + Ok(None) => return Ok(json_error(StatusCode::ACCEPTED, "query already running")), + Err(_) => { + return Ok(json_error( + StatusCode::SERVICE_UNAVAILABLE, + "cache unavailable", + )) + } + } + + let output = match run_gallery_dl(&url, limit).await { + Ok(value) => value, + Err(message) => { + let _: Result<(), _> = redis.del(lock_key).await; + return Ok(json_error(StatusCode::BAD_GATEWAY, message)); + } + }; + + let serialized = match serde_json::to_string(&output) { + Ok(serialized) => serialized, + Err(_) => { + let _: Result<(), _> = redis.del(lock_key).await; + return Ok(json_error( + StatusCode::BAD_GATEWAY, + "invalid gallery-dl output", + )); + } + }; + if let Err(error) = redis + .set_ex::<_, _, ()>(&cache_key, serialized, cache_ttl_seconds) + .await + { + log::error!("failed to cache gallery-dl response: {}", error); + let _: Result<(), _> = redis.del(lock_key).await; + return Ok(json_error( + StatusCode::SERVICE_UNAVAILABLE, + "cache unavailable", + )); + } + let _: Result<(), _> = redis.del(lock_key).await; + + Ok(json_response( + StatusCode::OK, + &output, + Some(("MISS", cache_ttl_seconds)), + )) +} + +fn is_authorized(auth: Option<&str>) -> bool { + let Ok(token) = env::var("GALLERY_DL_TOKEN") else { + return true; + }; + + let expected = format!("Bearer {}", token); + auth == Some(expected.as_str()) +} + +struct NormalizedQuery { + terms: Vec, +} + +fn normalize_source(source: String) -> Result { + let source = source.trim().to_ascii_lowercase(); + if source.is_empty() { + return Err("invalid source"); + } + Ok(source) +} + +fn normalize_query( + tags: Option>, + rating: Option, +) -> Result { + let tags = normalize_tags(tags.unwrap_or_default())?; + let rating = normalize_rating(rating)?; + let mut terms = tags.clone(); + if let Some(rating) = &rating { + terms.push(format!("rating:{}", rating)); + } + terms.sort(); + terms.dedup(); + + if terms.is_empty() { + return Err("missing query terms"); + } + + Ok(NormalizedQuery { terms }) +} + +fn normalize_tags(tags: Vec) -> Result, &'static str> { + if tags.len() > MAX_TAGS { + return Err("invalid tag count"); + } + + let mut normalized = Vec::with_capacity(tags.len()); + for tag in tags { + let tag = tag.trim().to_ascii_lowercase(); + if tag.is_empty() || tag.len() > MAX_TAG_LENGTH || !tag.chars().all(is_safe_tag_char) { + return Err("invalid tag"); + } + normalized.push(tag); + } + normalized.sort(); + normalized.dedup(); + Ok(normalized) +} + +fn normalize_rating(rating: Option) -> Result, &'static str> { + let Some(rating) = rating else { + return Ok(None); + }; + + let rating = rating.trim().to_ascii_lowercase(); + let rating = rating + .strip_prefix("rating:") + .unwrap_or(&rating) + .to_string(); + if rating.is_empty() + || rating.len() > MAX_RATING_LENGTH + || !rating.chars().all(is_safe_rating_char) + { + return Err("invalid rating"); + } + + Ok(Some(rating)) +} + +fn is_safe_tag_char(c: char) -> bool { + c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':') +} + +fn is_safe_rating_char(c: char) -> bool { + c.is_ascii_alphanumeric() || matches!(c, '_' | '-') +} + +fn cache_ttl_seconds() -> u64 { + env::var("GALLERY_DL_CACHE_TTL_SECONDS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|ttl| *ttl > 0) + .unwrap_or(DEFAULT_CACHE_TTL_SECONDS) +} + +async fn cache_ttl(redis: &mut ConnectionManager, cache_key: &str, fallback: u64) -> u64 { + let ttl: Result = redis.ttl(cache_key).await; + ttl.ok() + .and_then(|ttl| u64::try_from(ttl).ok()) + .filter(|ttl| *ttl > 0) + .unwrap_or(fallback) +} + +fn build_url(source: &str, tags: &[String], limit: u16) -> Option { + let joined_tags = tags.join("+"); + match source { + "danbooru" => Some(format!( + "https://danbooru.donmai.us/posts?tags={}&limit={}", + joined_tags, limit + )), + "gelbooru" => Some(format!( + "https://gelbooru.com/index.php?page=post&s=list&tags={}", + joined_tags + )), + "safebooru" => Some(format!( + "https://safebooru.org/index.php?page=post&s=list&tags={}", + joined_tags + )), + "konachan" => Some(format!( + "https://konachan.com/post?tags={}&limit={}", + joined_tags, limit + )), + "yandere" => Some(format!( + "https://yande.re/post?tags={}&limit={}", + joined_tags, limit + )), + _ => None, + } +} + +fn cache_key(source: &str, tags: &[String], limit: u16) -> String { + let mut hasher = Sha256::new(); + hasher.update(source.as_bytes()); + hasher.update([0]); + hasher.update(tags.join("\n").as_bytes()); + hasher.update([0]); + hasher.update(limit.to_string().as_bytes()); + + let digest = hasher.finalize(); + let mut hex = String::with_capacity(digest.len() * 2); + for byte in digest { + hex.push_str(&format!("{:02x}", byte)); + } + format!("gallery:v1:result:{}", hex) +} + +async fn run_gallery_dl(url: &str, limit: u16) -> Result { + let worker_url = + env::var("GALLERY_DL_WORKER_URL").map_err(|_| "GALLERY_DL_WORKER_URL is not configured")?; + let client = reqwest::Client::new(); + let response = time::timeout( + COMMAND_TIMEOUT, + client + .post(worker_url) + .json(&WorkerRequest { + url, + args: vec!["--range".to_string(), format!("1-{}", limit)], + }) + .send(), + ) + .await + .map_err(|_| "gallery-dl worker timed out")? + .map_err(|_| "failed to call gallery-dl worker")?; + + if !response.status().is_success() { + return Err("gallery-dl worker failed"); + } + + let bytes = response + .bytes() + .await + .map_err(|_| "failed to read gallery-dl worker response")?; + if bytes.len() > MAX_OUTPUT_BYTES { + return Err("gallery-dl output too large"); + } + + serde_json::from_slice(&bytes).map_err(|_| "invalid gallery-dl output") +} + +fn json_response( + status: StatusCode, + value: &T, + server_cache: Option<(&str, u64)>, +) -> Response { + let body = serde_json::to_string(value).unwrap_or_else(|_| "{}".to_string()); + let mut builder = Response::builder() + .status(status) + .header("Content-Type", "application/json") + .header("Cache-Control", "no-store"); + + if let Some((cache_state, ttl)) = server_cache { + builder = builder + .header("X-Server-Cache", cache_state) + .header("X-Server-Cache-Ttl-Seconds", ttl.to_string()); + } + + builder.body(body).unwrap() +} + +fn json_error(status: StatusCode, message: &str) -> Response { + json_response( + status, + &ErrorResponse { + error: message.to_string(), + }, + None, + ) +} diff --git a/src/lib.rs b/src/lib.rs index 545b698..4f96358 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,26 +1,27 @@ -use log::info; -use redis::aio::ConnectionManager; -use redis::AsyncCommands; -use std::time::Duration; -use std::{ - env, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, -}; -use tokio::time; -use warp::{http::Response, hyper::StatusCode, reply, Filter}; - -mod count_image; -use count_image::CountImage; - -mod image_cache; -use image_cache::ImageCache; - -mod const_image; -mod season_images; - +use log::info; +use redis::aio::ConnectionManager; +use redis::AsyncCommands; +use std::time::Duration; +use std::{ + env, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; +use tokio::time; +use warp::{http::Response, hyper::StatusCode, reply, Filter}; + +mod count_image; +use count_image::CountImage; + +mod image_cache; +use image_cache::ImageCache; + +mod const_image; +mod gallery_dl; +mod season_images; + const IMAGE_SOURCES: [&str; 15] = [ "nekos.life", "nekos.best", @@ -30,319 +31,328 @@ const IMAGE_SOURCES: [&str; 15] = [ "purrbot", "shibe.online", "catboys", - "waifu.im", - "waifu.pics", - "dog_ceo", - "the_cat_api", - "twitter_search", - "twitter_user_timeline", - "testing", -]; - -lazy_static::lazy_static! { - // Cache of images - static ref IMAGE_CACHE: ImageCache = ImageCache::new(); -} - -pub async fn init(port: u16) { - // Create Logger - if env::var_os("RUST_LOG").is_none() { - env::set_var("RUST_LOG", "neko_server=info"); - } - if env::var_os("RUST_LOG_STYLE").is_none() { - env::set_var("RUST_LOG_STYLE", "never"); - } - env_logger::builder() - .format_timestamp(None) - .target(env_logger::Target::Stdout) - .init(); - - // Database connection - let redis_url: &str = &env::var("REDIS_URL").expect("REDIS_URL must be set"); - let redis_client = redis::Client::open(redis_url).expect("Incorrect Redis URL"); - let redis = ConnectionManager::new(redis_client) - .await - .unwrap_or_else(|_| panic!("Failed to connect to Redis at: {}", redis_url)); - { - let mut conn = redis.clone(); - let check: Result<(), _> = conn.set("auth_test", "success").await; - check.expect("Failed to execute redis commands"); - } - - // Update the image from the database - let mut redis_clone = redis.clone(); - let update_task = tokio::spawn(async move { - let mut interval = time::interval(image_cache::UPDATE_INTERVAL); - loop { - interval.tick().await; - - let mut pipe = redis::pipe(); - pipe.atomic(); - for source in IMAGE_SOURCES.iter() { - pipe.get(*source); - } - let results = pipe - .query_async::>>(&mut redis_clone) - .await; - - if results.is_err() { - log::error!("Failed to get image count from Redis"); - continue; - } - let sum: u64 = results.unwrap().into_iter().flatten().sum(); - - IMAGE_CACHE.update_total_image(sum as u128).await; - } - }); - - // Save historical data - let mut redis_clone = redis.clone(); - let history_task = tokio::spawn(async move { - let mut interval = time::interval(Duration::from_secs(60 * 60)); // 1 hours - loop { - interval.tick().await; - - // Get current sum - let mut pipe = redis::pipe(); - pipe.atomic(); - for source in IMAGE_SOURCES.iter() { - pipe.get(*source); - } - let results = pipe - .query_async::>>(&mut redis_clone) - .await; - - if results.is_err() { - log::error!(target: "history", "Failed to get image count from Redis"); - continue; - } - let sum: u64 = results.unwrap().into_iter().flatten().sum(); - - // Get Date and Key name - let date = chrono::Utc::now().format("%Y%m%d").to_string(); - let key_name = format!("history:{}", date); - - // Save to Redis - let result: Result<(), redis::RedisError> = redis_clone.set(key_name, sum).await; - if result.is_err() { - log::error!(target: "history", "Failed to save history to Redis"); - continue; - } - - log::info!(target: "history", "Saved history of {} downloads on {} to Redis", sum, date); - } - }); - - // Print current season - info!("Current Season: {}", season_images::seasonal_name()); - - // Create a filter for each Imagesource - let add_routes = IMAGE_SOURCES - .iter() - .map(|name| { - info!("Adding route for {}", name); - // Add the path /add//:count - warp::path("add") - .and(warp::path(name)) - .and(warp::post()) - .and(warp::path::param()) - .and(warp::header::optional::("User-Agent")) - .and(with_redis(redis.clone())) - .and_then(move |count, agent, redis| add(name, count, agent, redis)) - .or(warp::path("add") - .and(warp::path(name)) - .and(warp::post()) - .and_then(|| async { - Ok::<_, warp::Rejection>( - Response::builder() - .status(StatusCode::NOT_ACCEPTABLE) - .body("Number to large".to_string()) - .unwrap(), - ) - })) - .boxed() - }) - .reduce(|acc, item| acc.or(item).unify().boxed()) - .expect("No routes were created"); - - // Add error message - let add_routes = add_routes.or(warp::path("add").map(|| { - Response::builder() - .status(StatusCode::NOT_ACCEPTABLE) - .body("Unknown Source".to_string()) - .unwrap() - })); - - // Create a filter to get the total count image - let get_image = warp::path("count_total") - .and(warp::get()) - .and_then(get_total_image); - - // Create a filter to get the image for a specific count - let get_count = warp::path("count") - .and(warp::get()) - .and(warp::path::param()) - .and_then(get_count_image); - - // Create a filter for the favicon - let favicon = warp::path("favicon.ico") - .and(warp::get()) - .and_then(get_favicon); - - // Add a default route to display server verison - let index = warp::path::end().map(|| { - reply::html(format!( - r#" - - - Neko Server - - - -
-

Neko Server

-

version: {}

-
- -"#, - env!("CARGO_PKG_VERSION") - )) - }); - - // Combine all Filters - let routes = get_image.or(add_routes).or(get_count).or(index).or(favicon); - - // Add logger - let routes = routes.with(warp::log::custom(|info| { - // Look for the `x-forwarded-for` header, and if it's not present, fall back to x-real-ip, and if that's not present, fall back to the remote addr. - let ip: String = info - .request_headers() - .get("x-forwarded-for") - .or_else(|| info.request_headers().get("x-real-ip")) - .map(|ip| ip.to_str().unwrap_or("Invalid Header")) - .unwrap_or_else(|| "Unknown") - .to_owned(); - - log::info!( - "{} {} - Status: {} - IP: {} - Agent: {} - Time: {:?}", - info.method(), - info.path(), - info.status(), - ip, - info.user_agent().unwrap_or("no agent"), - info.elapsed() - ); - })); - - // Default route - let error_log = warp::log::custom(|info| { - // Header to list - let mut headers = String::new(); - for (key, value) in info.request_headers().iter() { - headers.push_str(&format!( - "{}: {}\n", - key, - value.to_str().unwrap_or("[empty]") - )); - } - log::info!( - "SUS REQUEST: {} {} - Status: {} - Agent: {} - Time: {:?} - Headers: \n{}", - info.method(), - info.path(), - info.status(), - info.user_agent().unwrap_or("no agent"), - info.elapsed(), - headers.trim_end() - ); - }); - let routes = routes.or(warp::any() - .map(|| { - Response::builder() - .status(StatusCode::NOT_FOUND) - .body("Not Found. This incident will be reported.".to_string()) - .unwrap() - }) - .with(error_log)); - - // Run the server - info!( - "Starting nekoserver version {} on port {}", - env!("CARGO_PKG_VERSION"), - port - ); - let server = tokio::spawn( - warp::serve(routes).run(([0, 0, 0, 0], port)) - ); - - // Wait for termination signal - let term = Arc::new(AtomicBool::new(false)); - for sig in signal_hook::consts::TERM_SIGNALS { - signal_hook::flag::register(*sig, Arc::clone(&term)) - .expect("Failed to register signal handler"); - } - while !term.load(Ordering::Relaxed) { - time::sleep(Duration::from_secs(1)).await; - } - - // Shutdown the server - info!("Shutting down"); - - // Cleanup - update_task.abort(); - history_task.abort(); - server.abort(); -} - -async fn add( - name: &str, - count: u8, - agent: Option, - mut redis: ConnectionManager, -) -> Result { - // Check for correct header - match agent { - Some(agent) if agent.contains("NekoFans") => (), - _ => { - return Ok(reply::with_status( - "Valid request, but unauthorized", - StatusCode::UNAUTHORIZED, - )); - } - } - - // Increment the count - let r: Result<(), redis::RedisError> = redis.incr(name, count.to_string()).await; - - match r { - Ok(_) => Ok(reply::with_status("OK", StatusCode::OK)), - Err(_) => Ok(reply::with_status("", StatusCode::NOT_MODIFIED)), - } -} - -async fn get_total_image() -> Result { - Ok(Response::builder() - .header("Content-Type", "image/png") - .header("Cache-Control", "no-cache") - .body(IMAGE_CACHE.get_total().await.get_image())) -} - -async fn get_favicon() -> Result { - Ok(Response::builder() - .header("Content-Type", "image/png") - .body(unsafe { - // SAFETY: We dont care that this is not a valid utf8 string, its binary data - String::from_utf8_unchecked(include_bytes!("../assets/favicon.png").to_vec()) - })) -} - -async fn get_count_image(count: u128) -> Result { - Ok(Response::builder() - .header("Content-Type", "image/png") - .body(IMAGE_CACHE.get_count(count).await.get_image())) -} - -fn with_redis( - connection: ConnectionManager, -) -> impl Filter + Clone { - warp::any().map(move || connection.clone()) -} + "waifu.im", + "waifu.pics", + "dog_ceo", + "the_cat_api", + "twitter_search", + "twitter_user_timeline", + "testing", +]; + +lazy_static::lazy_static! { + // Cache of images + static ref IMAGE_CACHE: ImageCache = ImageCache::new(); +} + +pub async fn init(port: u16) { + // Create Logger + if env::var_os("RUST_LOG").is_none() { + env::set_var("RUST_LOG", "neko_server=info"); + } + if env::var_os("RUST_LOG_STYLE").is_none() { + env::set_var("RUST_LOG_STYLE", "never"); + } + env_logger::builder() + .format_timestamp(None) + .target(env_logger::Target::Stdout) + .init(); + + // Database connection + let redis_url: &str = &env::var("REDIS_URL").expect("REDIS_URL must be set"); + let redis_client = redis::Client::open(redis_url).expect("Incorrect Redis URL"); + let redis = ConnectionManager::new(redis_client) + .await + .unwrap_or_else(|_| panic!("Failed to connect to Redis at: {}", redis_url)); + { + let mut conn = redis.clone(); + let check: Result<(), _> = conn.set("auth_test", "success").await; + check.expect("Failed to execute redis commands"); + } + + // Update the image from the database + let mut redis_clone = redis.clone(); + let update_task = tokio::spawn(async move { + let mut interval = time::interval(image_cache::UPDATE_INTERVAL); + loop { + interval.tick().await; + + let mut pipe = redis::pipe(); + pipe.atomic(); + for source in IMAGE_SOURCES.iter() { + pipe.get(*source); + } + let results = pipe.query_async::>>(&mut redis_clone).await; + + if results.is_err() { + log::error!("Failed to get image count from Redis"); + continue; + } + let sum: u64 = results.unwrap().into_iter().flatten().sum(); + + IMAGE_CACHE.update_total_image(sum as u128).await; + } + }); + + // Save historical data + let mut redis_clone = redis.clone(); + let history_task = tokio::spawn(async move { + let mut interval = time::interval(Duration::from_secs(60 * 60)); // 1 hours + loop { + interval.tick().await; + + // Get current sum + let mut pipe = redis::pipe(); + pipe.atomic(); + for source in IMAGE_SOURCES.iter() { + pipe.get(*source); + } + let results = pipe.query_async::>>(&mut redis_clone).await; + + if results.is_err() { + log::error!(target: "history", "Failed to get image count from Redis"); + continue; + } + let sum: u64 = results.unwrap().into_iter().flatten().sum(); + + // Get Date and Key name + let date = chrono::Utc::now().format("%Y%m%d").to_string(); + let key_name = format!("history:{}", date); + + // Save to Redis + let result: Result<(), redis::RedisError> = redis_clone.set(key_name, sum).await; + if result.is_err() { + log::error!(target: "history", "Failed to save history to Redis"); + continue; + } + + log::info!(target: "history", "Saved history of {} downloads on {} to Redis", sum, date); + } + }); + + // Print current season + info!("Current Season: {}", season_images::seasonal_name()); + + // Create a filter for each Imagesource + let add_routes = IMAGE_SOURCES + .iter() + .map(|name| { + info!("Adding route for {}", name); + // Add the path /add//:count + warp::path("add") + .and(warp::path(name)) + .and(warp::post()) + .and(warp::path::param()) + .and(warp::header::optional::("User-Agent")) + .and(with_redis(redis.clone())) + .and_then(move |count, agent, redis| add(name, count, agent, redis)) + .or(warp::path("add") + .and(warp::path(name)) + .and(warp::post()) + .and_then(|| async { + Ok::<_, warp::Rejection>( + Response::builder() + .status(StatusCode::NOT_ACCEPTABLE) + .body("Number to large".to_string()) + .unwrap(), + ) + })) + .boxed() + }) + .reduce(|acc, item| acc.or(item).unify().boxed()) + .expect("No routes were created"); + + // Add error message + let add_routes = add_routes.or(warp::path("add").map(|| { + Response::builder() + .status(StatusCode::NOT_ACCEPTABLE) + .body("Unknown Source".to_string()) + .unwrap() + })); + + // Create a filter to get the total count image + let get_image = warp::path("count_total") + .and(warp::get()) + .and_then(get_total_image); + + // Create a filter to get the image for a specific count + let get_count = warp::path("count") + .and(warp::get()) + .and(warp::path::param()) + .and_then(get_count_image); + + // Create a filter for the favicon + let favicon = warp::path("favicon.ico") + .and(warp::get()) + .and_then(get_favicon); + + // Create a guarded gallery-dl JSON query endpoint + let gallery_query = warp::path("gallery") + .and(warp::path("query")) + .and(warp::post()) + .and(warp::header::optional::("authorization")) + .and(warp::body::content_length_limit(16 * 1024)) + .and(warp::body::json()) + .and(with_redis(redis.clone())) + .and_then(gallery_dl::query); + + // Add a default route to display server verison + let index = warp::path::end().map(|| { + reply::html(format!( + r#" + + + Neko Server + + + +
+

Neko Server

+

version: {}

+
+ +"#, + env!("CARGO_PKG_VERSION") + )) + }); + + // Combine all Filters + let routes = get_image + .or(add_routes) + .or(get_count) + .or(gallery_query) + .or(index) + .or(favicon); + + // Add logger + let routes = routes.with(warp::log::custom(|info| { + // Look for the `x-forwarded-for` header, and if it's not present, fall back to x-real-ip, and if that's not present, fall back to the remote addr. + let ip: String = info + .request_headers() + .get("x-forwarded-for") + .or_else(|| info.request_headers().get("x-real-ip")) + .map(|ip| ip.to_str().unwrap_or("Invalid Header")) + .unwrap_or_else(|| "Unknown") + .to_owned(); + + log::info!( + "{} {} - Status: {} - IP: {} - Agent: {} - Time: {:?}", + info.method(), + info.path(), + info.status(), + ip, + info.user_agent().unwrap_or("no agent"), + info.elapsed() + ); + })); + + // Default route + let error_log = warp::log::custom(|info| { + // Header to list + let mut headers = String::new(); + for (key, value) in info.request_headers().iter() { + headers.push_str(&format!( + "{}: {}\n", + key, + value.to_str().unwrap_or("[empty]") + )); + } + log::info!( + "SUS REQUEST: {} {} - Status: {} - Agent: {} - Time: {:?} - Headers: \n{}", + info.method(), + info.path(), + info.status(), + info.user_agent().unwrap_or("no agent"), + info.elapsed(), + headers.trim_end() + ); + }); + let routes = routes.or(warp::any() + .map(|| { + Response::builder() + .status(StatusCode::NOT_FOUND) + .body("Not Found. This incident will be reported.".to_string()) + .unwrap() + }) + .with(error_log)); + + // Run the server + info!( + "Starting nekoserver version {} on port {}", + env!("CARGO_PKG_VERSION"), + port + ); + let server = tokio::spawn(warp::serve(routes).run(([0, 0, 0, 0], port))); + + // Wait for termination signal + let term = Arc::new(AtomicBool::new(false)); + for sig in signal_hook::consts::TERM_SIGNALS { + signal_hook::flag::register(*sig, Arc::clone(&term)) + .expect("Failed to register signal handler"); + } + while !term.load(Ordering::Relaxed) { + time::sleep(Duration::from_secs(1)).await; + } + + // Shutdown the server + info!("Shutting down"); + + // Cleanup + update_task.abort(); + history_task.abort(); + server.abort(); +} + +async fn add( + name: &str, + count: u8, + agent: Option, + mut redis: ConnectionManager, +) -> Result { + // Check for correct header + match agent { + Some(agent) if agent.contains("NekoFans") => (), + _ => { + return Ok(reply::with_status( + "Valid request, but unauthorized", + StatusCode::UNAUTHORIZED, + )); + } + } + + // Increment the count + let r: Result<(), redis::RedisError> = redis.incr(name, count.to_string()).await; + + match r { + Ok(_) => Ok(reply::with_status("OK", StatusCode::OK)), + Err(_) => Ok(reply::with_status("", StatusCode::NOT_MODIFIED)), + } +} + +async fn get_total_image() -> Result { + Ok(Response::builder() + .header("Content-Type", "image/png") + .header("Cache-Control", "no-cache") + .body(IMAGE_CACHE.get_total().await.get_image())) +} + +async fn get_favicon() -> Result { + Ok(Response::builder() + .header("Content-Type", "image/png") + .body(unsafe { + // SAFETY: We dont care that this is not a valid utf8 string, its binary data + String::from_utf8_unchecked(include_bytes!("../assets/favicon.png").to_vec()) + })) +} + +async fn get_count_image(count: u128) -> Result { + Ok(Response::builder() + .header("Content-Type", "image/png") + .body(IMAGE_CACHE.get_count(count).await.get_image())) +} + +fn with_redis( + connection: ConnectionManager, +) -> impl Filter + Clone { + warp::any().map(move || connection.clone()) +} diff --git a/tests/fill_cache.py b/tests/fill_cache.py old mode 100644 new mode 100755 index e62a4f7..4e6128e --- a/tests/fill_cache.py +++ b/tests/fill_cache.py @@ -1,19 +1,22 @@ -import requests -import random +#!/usr/bin/env python3 + import os -import time +import random -url = os.getenv('NEKO_SERVER', 'http://127.0.0.1') + "/count/" +import requests + +url = os.getenv("NEKO_SERVER", "http://127.0.0.1") + "/count/" timing = [] # Fill the cache for i in range(50000): response = requests.get(url + str(random.randint(0, 100000000))) - if (response.status_code != 200): + if response.status_code != 200: print("Error: " + response.text) - elif (i % 100 == 0): + elif i % 100 == 0: print(".", end="", flush=True) timing.append(response.elapsed.total_seconds()) del response -print("\nAverage response time: " + str(sum(timing) / len(timing) * 1000) + "ms") \ No newline at end of file +print("\nAverage response time: " + str(sum(timing) / len(timing) * 1000) + "ms") + diff --git a/tests/test_gallery_dl.py b/tests/test_gallery_dl.py new file mode 100755 index 0000000..0ba0ce9 --- /dev/null +++ b/tests/test_gallery_dl.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 + +import argparse +import json +import sys +import urllib.error +import urllib.request + +WORKER_QUERIES = [ + { + "name": "worker danbooru range", + "request": { + "url": "https://danbooru.donmai.us/posts?tags=rating:safe", + "args": ["--range", "1-1"], + }, + }, + { + "name": "worker safebooru range", + "request": { + "url": "https://safebooru.org/index.php?page=post&s=list&tags=cat", + "args": ["--range", "1-1"], + }, + }, +] + +SERVER_QUERIES = [ + { + "name": "server danbooru query", + "request": { + "source": "danbooru", + "rating": "safe", + "tags": [], + "limit": 1, + }, + }, +] + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Run smoke queries against local docker-compose gallery-dl services" + ) + parser.add_argument( + "--worker-url", + default="http://localhost:8090/query", + help="gallery-dl worker query endpoint", + ) + parser.add_argument( + "--server-url", + default="http://localhost:8080/gallery/query", + help="neko server gallery query endpoint", + ) + args = parser.parse_args() + + for query in WORKER_QUERIES: + response, _ = run_query(args.worker_url, query) + if response is None: + return 1 + if not isinstance(response, list): + print( + f"failed {query['name']}: expected gallery-dl JSON list", + file=sys.stderr, + ) + return 1 + print(f"ok {query['name']}: {len(response)} top-level items") + + for query in SERVER_QUERIES: + response, headers = run_query(args.server_url, query) + if response is None: + return 1 + if not isinstance(response, list): + print( + f"failed {query['name']}: expected gallery-dl JSON list", + file=sys.stderr, + ) + return 1 + if headers.get("Cache-Control") != "no-store": + print( + f"failed {query['name']}: expected no-store Cache-Control", + file=sys.stderr, + ) + return 1 + if headers.get("X-Server-Cache") not in {"HIT", "MISS"}: + print( + f"failed {query['name']}: expected server cache header", file=sys.stderr + ) + return 1 + print(f"ok {query['name']}: server-cache={headers.get('X-Server-Cache')}") + + return 0 + + +def run_query(url: str, query: dict): + print(f"running {query['name']}...") + try: + return post_json(url, query["request"]) + except urllib.error.HTTPError as error: + body = error.read().decode("utf-8", errors="replace") + print(f"failed {query['name']}: HTTP {error.code}: {body}", file=sys.stderr) + if error.code == 400 and "invalid tag count" in body: + print( + "server appears stale: rebuild/restart nekoserver so rating-only queries are accepted", + file=sys.stderr, + ) + except urllib.error.URLError as error: + print(f"failed {query['name']}: {error}", file=sys.stderr) + except json.JSONDecodeError as error: + print(f"failed {query['name']}: invalid JSON: {error}", file=sys.stderr) + return None, {} + + +def post_json(url: str, value: dict): + body = json.dumps(value).encode("utf-8") + request = urllib.request.Request( + url, + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=60) as response: + return json.loads(response.read()), response.headers + + +if __name__ == "__main__": + raise SystemExit(main())