add pixiv API stub

This commit is contained in:
theBreadCompany 2024-09-25 05:12:58 +02:00
parent 3a35b2100c
commit 129d60f07f
7 changed files with 109 additions and 204 deletions

122
Cargo.lock generated
View file

@ -78,9 +78,9 @@ dependencies = [
[[package]]
name = "async-trait"
version = "0.1.82"
version = "0.1.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1"
checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd"
dependencies = [
"proc-macro2",
"quote",
@ -355,18 +355,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "fallible-iterator"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fastrand"
version = "2.1.1"
@ -506,10 +494,9 @@ name = "fxpixiv"
version = "0.1.0"
dependencies = [
"json",
"libpixiv",
"maud",
"reqwest",
"rocket",
"rusqlite",
"scraper",
"tl",
]
@ -602,18 +589,6 @@ name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
dependencies = [
"ahash",
]
[[package]]
name = "hashlink"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
dependencies = [
"hashbrown",
]
[[package]]
name = "hermit-abi"
@ -788,9 +763,9 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.8"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba"
checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b"
dependencies = [
"bytes",
"futures-channel",
@ -801,7 +776,6 @@ dependencies = [
"pin-project-lite",
"socket2",
"tokio",
"tower",
"tower-service",
"tracing",
]
@ -902,9 +876,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "libc"
version = "0.2.158"
version = "0.2.159"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
[[package]]
name = "libpixiv"
@ -918,16 +892,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "libsqlite3-sys"
version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"
dependencies = [
"pkg-config",
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.14"
@ -1339,26 +1303,6 @@ dependencies = [
"siphasher",
]
[[package]]
name = "pin-project"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pin-project-lite"
version = "0.2.14"
@ -1373,9 +1317,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.30"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
[[package]]
name = "powerfmt"
@ -1484,9 +1428,9 @@ dependencies = [
[[package]]
name = "redox_syscall"
version = "0.5.4"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853"
checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0"
dependencies = [
"bitflags",
]
@ -1639,6 +1583,7 @@ dependencies = [
"rocket_codegen",
"rocket_http",
"serde",
"serde_json",
"state",
"tempfile",
"time",
@ -1694,20 +1639,6 @@ dependencies = [
"uncased",
]
[[package]]
name = "rusqlite"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e"
dependencies = [
"bitflags",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"smallvec",
]
[[package]]
name = "rustc-demangle"
version = "0.1.24"
@ -2261,9 +2192,9 @@ dependencies = [
[[package]]
name = "toml_edit"
version = "0.22.21"
version = "0.22.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf"
checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
dependencies = [
"indexmap",
"serde",
@ -2272,27 +2203,6 @@ dependencies = [
"winnow",
]
[[package]]
name = "tower"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
dependencies = [
"futures-core",
"futures-util",
"pin-project",
"pin-project-lite",
"tokio",
"tower-layer",
"tower-service",
]
[[package]]
name = "tower-layer"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
[[package]]
name = "tower-service"
version = "0.3.3"
@ -2762,9 +2672,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
version = "0.6.18"
version = "0.6.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587"
dependencies = [
"memchr",
]

View file

@ -5,3 +5,13 @@ The pixiv embed fixer
## DISCLAIMER
This project is still in development, please be patient until this project is actually usable.
## nix build
As this mainly runs on my NixOS server for now, so build and service files are included.
Run `nix-shell -E 'with import <nixpkgs> {}; callPackage ./default.nix {}'` to enter the development shell.
## nix options
There is some hope that this service will make its way to the NixOS options, but I need to disappoint you for now.
You can still just clone this project and manually import the service file of course, make sure to keep the `default.nix` reachable tho.

View file

@ -5,9 +5,8 @@ edition = "2021"
[dependencies]
tl = "0.7.8"
reqwest = "0.12.7"
json = "0.12.4"
rocket = "0.5.1"
scraper = "0.20.0"
maud = "0.26.0"
rusqlite = "0.32.1"
libpixiv = { path = "../libpixiv" }
rocket = { version = "0.5.1", features = ["json"] }

View file

@ -1,99 +1,65 @@
#[macro_use]
extern crate rocket;
#![deny(elided_lifetimes_in_paths)]
#[macro_use] extern crate rocket;
use libpixiv::PixivAppClient;
use maud::{html, DOCTYPE};
use reqwest::Client;
use rocket::http::Status;
use rocket::response::content::RawHtml;
use scraper::{Html, Selector};
use rocket::{http::Status, response::content::RawHtml, State};
struct Metadata {
pub image: String,
pub title: String,
pub desc: String,
}
#[get("/<path..>")]
async fn handle_route(path: std::path::PathBuf) -> Result<RawHtml<String>, Status> {
async fn handle_route(state: &State<Option<PixivAppClient>>, path: std::path::PathBuf) -> Result<RawHtml<String>, Status> {
let target = format!("https://pixiv.net/{}", path.display());
let html = match fetch_content(&target).await {
Ok(html) => html,
Err(err) => return Err(err),
};
if let Some(id) = path.file_name() {
let meta = fetch_illust(state, id.to_str().unwrap().parse::<u32>().unwrap());
Ok(RawHtml(create_page(&target, &meta.await.unwrap()).await.unwrap()))
} else {
Err(Status::InternalServerError)
}
let modified = create_page(&target, &html).await.unwrap();
Ok(RawHtml(modified))
}
async fn fetch_content(url: &String) -> Result<String, Status> {
println!("{}", url);
let client = Client::new();
let response = match client.get(url).send().await {
Ok(resp) => resp,
Err(_) => return Err(Status::BadGateway),
};
async fn fetch_illust(client: &Option<PixivAppClient>, illust_id: u32) -> Option<Metadata> {
if let Some(client) = client {
if let Ok(illust) = client.illust_details(illust_id).await {
let image = if illust.page_count == 1 { illust.meta_single_page.unwrap().original_image_url.unwrap() } else { illust.meta_pages[0].image_urls.large.clone() };
return Some(Metadata {
image: image,
title: illust.title,
desc: illust.caption,
})
}
}
let html = match response.text().await {
Ok(text) => text,
Err(_) => return Err(Status::InternalServerError),
};
Ok(html)
None
}
async fn create_page(source: &String, html: &String) -> Result<String, Status> {
let dom = Html::parse_document(html);
let data_selector = match Selector::parse(r#"meta[name="preload-data"]"#) {
Ok(sel) => sel,
Err(_) => return Err(Status::InternalServerError),
};
let data_meta = match dom.select(&data_selector).next() {
Some(meta) => meta,
None => return Err(Status::InternalServerError),
};
let illust = json::parse(data_meta.value().attr("content").unwrap()).unwrap();
let image = match illust["illust"].entries().next() {
Some(j) => j.1["urls"]["regular"]
.as_str()
.unwrap()
.replace("pximg.net", "fixiv.net"),
None => "https://http.cat/images/501.jpg".to_string(),
};
let title_selector = match Selector::parse(r#"meta[property="og:title"]"#) {
Ok(sel) => sel,
Err(_) => return Err(Status::InternalServerError),
};
let title_meta = match dom.select(&title_selector).next() {
Some(meta) => meta.attr("content").unwrap(),
None => "unknown title",
};
let desc_selector = match Selector::parse(r#"meta[property="og:description"]"#) {
Ok(sel) => sel,
Err(_) => return Err(Status::InternalServerError),
};
let desc_meta = match dom.select(&desc_selector).next() {
Some(meta) => meta.attr("content").unwrap(),
None => "unknown title",
};
async fn create_page(source: &str, meta: &Metadata) -> Result<String, Status> {
Ok(html! {
(DOCTYPE)
html lang="en" {
head {
meta charset="utf-8";
meta property="og:title" content=(title_meta);
meta property="og:image" content=(image);
meta property="og:title" content=(meta.title);
meta property="og:image" content=(meta.image);
meta property="og:url" content=(source);
meta property="og:type" content="article";
meta property="og:site_name" content="pixiv";
meta property="og:description" content=(desc_meta);
meta property="og:description" content=(meta.desc);
meta property="twitter:card" content="summary_large_image";
meta property="twitter:site" content="@pixiv";
meta property="twitter:url" content=(source);
meta property="twitter:title" content=(title_meta);
meta property="twitter:description" content=(desc_meta);
meta property="twitter:image" content=(image);
meta property="twitter:title" content=(meta.title);
meta property="twitter:description" content=(meta.desc);
meta property="twitter:image" content=(meta.image);
meta http-equiv="Refresh" content=(format!("0; url='{}'", source));
@ -110,6 +76,17 @@ async fn create_page(source: &String, html: &String) -> Result<String, Status> {
}
#[launch]
fn launch() -> _ {
rocket::build().mount("/", routes![handle_route])
async fn launch() -> _ {
let mut pixiv_client: Option<PixivAppClient> = None;
if let Ok(token) = std::env::var("PIXIV_REFRESH_TOKEN") {
let mut client = PixivAppClient::new(token);
client.refresh_token().await; // TODO: refresh token on a regular basis
pixiv_client = Some(client);
}
rocket::build()
.manage(pixiv_client)
.mount("/", routes![handle_route])
}

View file

@ -11,4 +11,8 @@ For now, no OAuth2 login will be provided, so please make sure to provide refres
You can run the following command to test existing features:
```bash
PIXIV_REFRESH_TOKEN= cargo test --lib
```
```
## Notes
This library may be available as a standalone crate in the future, depending on how many of the available endpoints get actually implemented.

View file

@ -5,11 +5,12 @@ extern crate serde_json;
use chrono::Local;
use reqwest::header::{HeaderName, AUTHORIZATION, CONTENT_TYPE, USER_AGENT};
use serde_json::Value;
use std::sync::{Arc, Mutex};
use tokio::sync::Mutex;
use std::sync::Arc;
mod models;
struct PixivAppClient {
pub struct PixivAppClient {
/// bearer token
access_token: Arc<Mutex<String>>,
refresh_token: Arc<Mutex<String>>,
@ -18,10 +19,10 @@ struct PixivAppClient {
}
impl PixivAppClient {
fn new(token: &str) -> Self {
pub fn new(token: String) -> Self {
let client = Self {
access_token: Arc::new(Mutex::new(String::new())),
refresh_token: Arc::new(Mutex::new(String::from(token))),
refresh_token: Arc::new(Mutex::new(token)),
http_client: reqwest::Client::new(),
host: String::from("https://app-api.pixiv.net"),
};
@ -37,7 +38,7 @@ impl PixivAppClient {
let time = Local::now().format("%y-%m-%dT%H:%m:%s+00:00");
let time_str = format!("{}", time);
let cloned_refresh_token = Arc::clone(&self.refresh_token);
let cloned_refresh_token_str = &cloned_refresh_token.lock().unwrap();
let cloned_refresh_token_str = &cloned_refresh_token.lock().await;
let client_id = "MOBrBDS8blbauoSck0ZfDbtuzpyT";
let client_secret = "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj";
@ -86,7 +87,7 @@ impl PixivAppClient {
.get(url)
.header(
AUTHORIZATION,
format!("Bearer {}", self.access_token.lock().unwrap()),
format!("Bearer {}", self.access_token.lock().await),
)
.header(HeaderName::from_lowercase(b"app-os").unwrap(), "ios")
.header(
@ -125,20 +126,18 @@ impl PixivAppClient {
#[cfg(test)]
mod client_tests {
use super::*;
use std::{assert, assert_eq, env, panic};
use std::{assert, assert_eq, env};
#[tokio::test]
async fn login() {
let token = env::var("PIXIV_REFRESH_TOKEN");
let mut client = PixivAppClient::new(
&token.expect("expecting PIXIV_REFRESH_TOKEN variable for testing!"),
token.expect("expecting PIXIV_REFRESH_TOKEN variable for testing!"),
);
client.refresh_token().await;
let cloned_access_token = Arc::clone(&client.access_token);
match cloned_access_token.lock() {
Ok(t) => assert!(!t.is_empty(), "Expected to receive token!"),
Err(_) => panic!("No token received!"),
};
let t = cloned_access_token.lock().await;
assert!(!t.is_empty(), "Expected to receive token!");
}
#[tokio::test]
@ -146,7 +145,7 @@ mod client_tests {
let illust_id = 122388293;
let token = env::var("PIXIV_REFRESH_TOKEN");
let mut client = PixivAppClient::new(
&token.expect("expecting PIXIV_REFRESH_TOKEN variable for testing!"),
token.expect("expecting PIXIV_REFRESH_TOKEN variable for testing!"),
);
client.refresh_token().await;
let illust = client.illust_details(illust_id).await;

View file

@ -17,6 +17,7 @@ pub struct Illustration {
pub restrict: i32,
pub x_restrict: i32,
pub image_urls: IllustrationURLs,
pub meta_single_page: Option<IllustrationMetaURL>,
pub meta_pages: Vec<IllustrationURLsWrapper>,
pub total_view: u32,
pub total_bookmarks: u32,
@ -40,26 +41,31 @@ pub struct User {
#[derive(Serialize, Deserialize, std::fmt::Debug)]
pub struct UserProfileImages {
medium: String,
pub medium: String,
}
#[derive(Serialize, Deserialize, std::fmt::Debug)]
pub struct IllustrationURLsWrapper {
image_urls: IllustrationURLs,
pub image_urls: IllustrationURLs,
}
#[derive(Serialize, Deserialize, std::fmt::Debug)]
pub struct IllustrationMetaURL {
pub original_image_url: Option<String>,
}
#[derive(Serialize, Deserialize, std::fmt::Debug)]
pub struct IllustrationURLs {
square_medium: String,
medium: String,
large: String,
original: Option<String>,
pub square_medium: String,
pub medium: String,
pub large: String,
pub original: Option<String>,
}
#[derive(Serialize, Deserialize, std::fmt::Debug)]
pub struct Tag {
name: String,
translated_name: Option<String>,
pub name: String,
pub translated_name: Option<String>,
}
#[derive(Serialize, Deserialize, std::fmt::Debug)]