From d16138036a0ae78c206851d1cc2ba24dd9d0c8b3 Mon Sep 17 00:00:00 2001 From: theBreadCompany Date: Fri, 27 Sep 2024 02:36:38 +0200 Subject: [PATCH 1/2] add database setup and select/insert statements --- .gitignore | 3 +- Cargo.lock | 532 ++++++++++++++++++++++++++++++++++++++--- Rocket.toml | 2 +- fxpixiv/Cargo.toml | 4 +- fxpixiv/src/main.rs | 153 +++++++++++- libpixiv/src/lib.rs | 2 +- libpixiv/src/models.rs | 10 +- 7 files changed, 656 insertions(+), 50 deletions(-) diff --git a/.gitignore b/.gitignore index 22faa77..9caccb6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target result -.vscode \ No newline at end of file +.vscode +fxpixiv.db \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index e7f921a..04de51c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -73,7 +79,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -84,7 +90,16 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.77", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] [[package]] @@ -129,6 +144,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -147,6 +168,15 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -227,6 +257,55 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "cssparser" version = "0.31.2" @@ -247,7 +326,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -267,7 +346,7 @@ checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -300,9 +379,25 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.77", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dtoa" version = "1.0.9" @@ -329,6 +424,9 @@ name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +dependencies = [ + "serde", +] [[package]] name = "encoding_rs" @@ -355,6 +453,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "fastrand" version = "2.1.1" @@ -375,6 +479,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -445,6 +560,28 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.30" @@ -493,10 +630,12 @@ dependencies = [ name = "fxpixiv" version = "0.1.0" dependencies = [ + "chrono", "json", "libpixiv", "maud", "rocket", + "rocket_db_pools", "scraper", "tl", ] @@ -514,6 +653,16 @@ dependencies = [ "windows", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getopts" version = "0.2.21" @@ -589,6 +738,28 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" @@ -602,6 +773,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "html5ever" version = "0.27.0" @@ -613,7 +790,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -738,7 +915,7 @@ dependencies = [ "http 1.1.0", "hyper 1.4.1", "hyper-util", - "rustls", + "rustls 0.23.13", "rustls-pki-types", "tokio", "tokio-rustls", @@ -892,6 +1069,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -977,7 +1165,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -998,6 +1186,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.0" @@ -1061,6 +1255,16 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1134,7 +1338,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -1184,6 +1388,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pear" version = "0.2.9" @@ -1204,7 +1414,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -1282,7 +1492,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -1382,7 +1592,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.77", "version_check", "yansi", ] @@ -1428,9 +1638,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0" +checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" dependencies = [ "bitflags", ] @@ -1452,7 +1662,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -1505,7 +1715,7 @@ version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -1526,7 +1736,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 2.1.3", "serde", "serde_json", "serde_urlencoded", @@ -1607,11 +1817,33 @@ dependencies = [ "proc-macro2", "quote", "rocket_http", - "syn", + "syn 2.0.77", "unicode-xid", "version_check", ] +[[package]] +name = "rocket_db_pools" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6578b2740ceee3e78bff63fe9299d964b7e68318446cdcb9af3b9cab46e1e9d" +dependencies = [ + "rocket", + "rocket_db_pools_codegen", + "sqlx", + "version_check", +] + +[[package]] +name = "rocket_db_pools_codegen" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "842e859f2e87a23efc0f81e25756c0fb43f18726e62daf99da7ea19fbc56cebd" +dependencies = [ + "devise", + "quote", +] + [[package]] name = "rocket_http" version = "0.5.1" @@ -1658,6 +1890,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.13" @@ -1666,18 +1909,27 @@ checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "once_cell", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ - "base64", + "base64 0.22.1", "rustls-pki-types", ] @@ -1687,6 +1939,16 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -1747,6 +2009,16 @@ dependencies = [ "tendril", ] +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -1806,7 +2078,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -1823,9 +2095,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -1851,6 +2123,17 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1911,6 +2194,131 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "sqlformat" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790" +dependencies = [ + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" +dependencies = [ + "ahash", + "atoi", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", + "serde", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", +] [[package]] name = "stable-pattern" @@ -1968,6 +2376,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.77" @@ -2033,6 +2452,26 @@ dependencies = [ "utf-8", ] +[[package]] +name = "thiserror" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "thread_local" version = "1.1.8" @@ -2121,7 +2560,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -2140,7 +2579,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls", + "rustls 0.23.13", "rustls-pki-types", "tokio", ] @@ -2215,6 +2654,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2228,7 +2668,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.77", ] [[package]] @@ -2276,6 +2716,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "ubyte" version = "0.10.4" @@ -2316,6 +2762,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unicode-width" version = "0.1.14" @@ -2328,6 +2780,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "untrusted" version = "0.9.0" @@ -2345,6 +2803,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -2406,7 +2870,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.77", "wasm-bindgen-shared", ] @@ -2440,7 +2904,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.77", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2461,6 +2925,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + [[package]] name = "winapi" version = "0.3.9" @@ -2672,9 +3142,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.19" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -2706,7 +3176,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.77", ] [[package]] diff --git a/Rocket.toml b/Rocket.toml index 21ab8be..9f2c7c0 100644 --- a/Rocket.toml +++ b/Rocket.toml @@ -7,4 +7,4 @@ address = "0.0.0.0" port = 5470 [default.databases] -illustrations = { url = "sqlite:fxpixiv.db" } \ No newline at end of file +pixiv = { url = "sqlite:fxpixiv.db" } \ No newline at end of file diff --git a/fxpixiv/Cargo.toml b/fxpixiv/Cargo.toml index ff2f36b..a6e2246 100644 --- a/fxpixiv/Cargo.toml +++ b/fxpixiv/Cargo.toml @@ -9,4 +9,6 @@ json = "0.12.4" scraper = "0.20.0" maud = "0.26.0" libpixiv = { path = "../libpixiv" } -rocket = { version = "0.5.1", features = ["json"] } \ No newline at end of file +rocket = { version = "0.5.1", features = ["json"] } +rocket_db_pools = { version = "0.2.0", features = ["sqlx_sqlite"] } +chrono = "0.4.38" \ No newline at end of file diff --git a/fxpixiv/src/main.rs b/fxpixiv/src/main.rs index 86949f7..b6510e6 100644 --- a/fxpixiv/src/main.rs +++ b/fxpixiv/src/main.rs @@ -1,10 +1,19 @@ #![deny(elided_lifetimes_in_paths)] -#[macro_use] extern crate rocket; +#[macro_use] +extern crate rocket; +use chrono::Local; use libpixiv::PixivAppClient; use maud::{html, DOCTYPE}; -use rocket::{http::Status, response::content::RawHtml, State}; +use rocket::{fairing::AdHoc, http::Status, response::content::RawHtml, State}; +use rocket_db_pools::{ + Database, + {sqlx, sqlx::Row}, +}; +#[derive(Database)] +#[database("pixiv")] +struct PixivDB(sqlx::SqlitePool); struct Metadata { pub image: String, @@ -13,27 +22,116 @@ struct Metadata { } #[get("/")] -async fn handle_route(state: &State>, path: std::path::PathBuf) -> Result, Status> { +async fn handle_route( + client: &State>, + db: &State, + path: std::path::PathBuf, +) -> Result, Status> { let target = format!("https://pixiv.net/{}", path.display()); if let Some(id) = path.file_name() { - let meta = fetch_illust(state, id.to_str().unwrap().parse::().unwrap()); - Ok(RawHtml(create_page(&target, &meta.await.unwrap()).await.unwrap())) + let meta = fetch_illust(client, &db, id.to_str().unwrap().parse::().unwrap()); + Ok(RawHtml( + create_page(&target, &meta.await.unwrap()).await.unwrap(), + )) } else { Err(Status::InternalServerError) } - } -async fn fetch_illust(client: &Option, illust_id: u32) -> Option { +async fn fetch_illust( + client: &Option, + db: &PixivDB, + illust_id: u32, +) -> Option { + if let Ok(row) = sqlx::query( + " + SELECT p.large, i.title, i.desc + FROM Illustrations i + JOIN IllustrationPages p ON p.illust_id = i.id + WHERE p.page_number = 0 AND i.id = ? + ", + ) + .bind(illust_id) + .fetch_one(&db.0) + .await + { + return Some(Metadata { + image: row.get(0), + title: row.get(1), + desc: row.get(2), + }); + }; + if let Some(client) = client { if let Ok(illust) = client.illust_details(illust_id).await { - let image = if illust.page_count == 1 { illust.meta_single_page.unwrap().original_image_url.unwrap() } else { illust.meta_pages[0].image_urls.large.clone() }; + let user_query = " + INSERT OR REPLACE INTO User (id, name) + VALUES ($1, $2) + RETURNING id + "; + let _ = sqlx::query(user_query) + .bind(illust.user.id) + .bind(illust.user.name) + .fetch_one(&db.0) + .await + .ok().expect("failed to insert user"); + + let illust_query = " + INSERT OR REPLACE INTO Illustrations (id, title, description, user, expires_on) + VALUES ($1, $2, $3, $4, $5) + "; + let _ = sqlx::query(&illust_query) + .bind(illust.id) + .bind(&illust.title) + .bind(&illust.caption) + .bind(illust.user.id) + .bind(Local::now().naive_utc().to_string()) + .execute(&db.0) + .await + .expect("failed to insert illustration"); + + let pages_query = " + INSERT OR REPLACE INTO IllustrationPages (square_medium, medium, large, original, page_number, illust_id) + VALUES ($1, $2, $3, $4, $5, $6) + "; + if illust.meta_pages.is_empty() { + let _ = sqlx::query(pages_query) + .bind(illust.image_urls.square_medium) + .bind(illust.image_urls.medium) + .bind(illust.image_urls.large) + .bind(&illust.meta_single_page.as_ref().unwrap().original_image_url.clone().unwrap()) + .bind(0) + .bind(illust.id) + .execute(&db.0) + .await + .expect("failed to insert page"); + } else { + let _ = illust.meta_pages.iter().enumerate().map(|page| async move { + let _ = sqlx::query(pages_query) + .bind(&page.1.image_urls.square_medium) + .bind(&page.1.image_urls.medium) + .bind(&page.1.image_urls.large) + .bind(&page.1.image_urls.original) + .bind(page.0.to_string()) + .bind(illust.id) + .execute(&db.0) + .await + .expect("failed to insert page"); + }); + } + + let image = if illust.page_count == 1 { + illust.meta_single_page.unwrap().original_image_url.unwrap() + } else { + illust.meta_pages[0].image_urls.large.clone() + }; + return Some(Metadata { image: image, title: illust.title, desc: illust.caption, - }) + }); } } @@ -77,7 +175,6 @@ async fn create_page(source: &str, meta: &Metadata) -> Result { #[launch] async fn launch() -> _ { - let mut pixiv_client: Option = None; if let Ok(token) = std::env::var("PIXIV_REFRESH_TOKEN") { @@ -88,5 +185,41 @@ async fn launch() -> _ { rocket::build() .manage(pixiv_client) + .attach(PixivDB::init()) + .attach(AdHoc::on_ignite("Database Init", |rocket| { + Box::pin(async { + let db = PixivDB::fetch(&rocket).expect("Database connection failed"); + sqlx::query( + " + CREATE TABLE IF NOT EXISTS User ( + id INTEGER UNIQUE PRIMARY KEY, + name TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS Illustrations ( + id INTEGER UNIQUE PRIMARY KEY, + title TEXT NOT NULL, + description TEXT NOT NULL, + user INTEGER NOT NULL, + expires_on TIMESTAMP, + FOREIGN KEY (user) REFERENCES User(id) + ); + CREATE TABLE IF NOT EXISTS IllustrationPages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + illust_id INTEGER NOT NULL, + page_number INTEGER NOT NULL, + square_medium TEXT NOT NULL, + medium TEXT NOT NULL, + large TEXT NOT NULL, + original TEXT NOT NULL, + FOREIGN KEY (illust_id) REFERENCES Illustrations(id) + ); + ", + ) + .execute(&db.0) + .await + .expect("Database init failed!"); + rocket + }) + })) .mount("/", routes![handle_route]) } diff --git a/libpixiv/src/lib.rs b/libpixiv/src/lib.rs index 4d2192a..860d7aa 100644 --- a/libpixiv/src/lib.rs +++ b/libpixiv/src/lib.rs @@ -79,7 +79,7 @@ impl PixivAppClient { pub async fn illust_details( &self, illust_id: u32, - ) -> Result> { + ) -> Result> { let url = format!("{}/v1/illust/detail", self.host); let illust_id_str = illust_id.to_string(); diff --git a/libpixiv/src/models.rs b/libpixiv/src/models.rs index 2d993ac..0a20575 100644 --- a/libpixiv/src/models.rs +++ b/libpixiv/src/models.rs @@ -32,11 +32,11 @@ pub struct Illustration { #[derive(Serialize, Deserialize, std::fmt::Debug)] pub struct User { - id: i32, - name: String, - account: String, - profile_image_urls: UserProfileImages, - is_followed: bool, + pub id: i32, + pub name: String, + pub account: String, + pub profile_image_urls: UserProfileImages, + pub is_followed: bool, } #[derive(Serialize, Deserialize, std::fmt::Debug)] From d3fa6abcf5e03cb881e7f8c2cd244ab318f082f8 Mon Sep 17 00:00:00 2001 From: theBreadCompany Date: Fri, 27 Sep 2024 03:31:17 +0200 Subject: [PATCH 2/2] add expiration mechanism that deletes entries older than one week when queried --- fxpixiv/src/main.rs | 45 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/fxpixiv/src/main.rs b/fxpixiv/src/main.rs index b6510e6..354051e 100644 --- a/fxpixiv/src/main.rs +++ b/fxpixiv/src/main.rs @@ -2,7 +2,9 @@ #[macro_use] extern crate rocket; -use chrono::Local; +use std::str::FromStr; + +use chrono::{DateTime, Duration, Utc}; use libpixiv::PixivAppClient; use maud::{html, DOCTYPE}; use rocket::{fairing::AdHoc, http::Status, response::content::RawHtml, State}; @@ -46,7 +48,7 @@ async fn fetch_illust( ) -> Option { if let Ok(row) = sqlx::query( " - SELECT p.large, i.title, i.desc + SELECT p.large, i.title, i.desc, i.expires_on FROM Illustrations i JOIN IllustrationPages p ON p.illust_id = i.id WHERE p.page_number = 0 AND i.id = ? @@ -56,11 +58,23 @@ async fn fetch_illust( .fetch_one(&db.0) .await { - return Some(Metadata { - image: row.get(0), - title: row.get(1), - desc: row.get(2), - }); + if Utc::now() >= DateTime::::from_str(row.get(0)).unwrap() { + return Some(Metadata { + image: row.get(0), + title: row.get(1), + desc: row.get(2), + }); + } else { + let _ = sqlx::query( + " + DELETE FROM Illustrations + WHERE id = $1 + ", + ) + .bind(illust_id) + .execute(&db.0) + .await; + } }; if let Some(client) = client { @@ -75,7 +89,8 @@ async fn fetch_illust( .bind(illust.user.name) .fetch_one(&db.0) .await - .ok().expect("failed to insert user"); + .ok() + .expect("failed to insert user"); let illust_query = " INSERT OR REPLACE INTO Illustrations (id, title, description, user, expires_on) @@ -86,7 +101,7 @@ async fn fetch_illust( .bind(&illust.title) .bind(&illust.caption) .bind(illust.user.id) - .bind(Local::now().naive_utc().to_string()) + .bind((Utc::now() + Duration::days(7)).to_string()) .execute(&db.0) .await .expect("failed to insert illustration"); @@ -100,9 +115,17 @@ async fn fetch_illust( .bind(illust.image_urls.square_medium) .bind(illust.image_urls.medium) .bind(illust.image_urls.large) - .bind(&illust.meta_single_page.as_ref().unwrap().original_image_url.clone().unwrap()) + .bind( + &illust + .meta_single_page + .as_ref() + .unwrap() + .original_image_url + .clone() + .unwrap(), + ) .bind(0) - .bind(illust.id) + .bind(illust.id) .execute(&db.0) .await .expect("failed to insert page");