init project with basic working example
This commit is contained in:
commit
17bd70e3bd
5 changed files with 1633 additions and 0 deletions
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
|
@ -0,0 +1 @@
|
||||||
|
/target
|
1476
Cargo.lock
generated
Normal file
1476
Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load diff
10
Cargo.toml
Normal file
10
Cargo.toml
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
[package]
|
||||||
|
name = "sfwikiscraper"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2021"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
reqwest = "0.12.9"
|
||||||
|
serde_json = "1.0.132"
|
||||||
|
tl = "0.7.8"
|
||||||
|
tokio = { version = "1.41.1", features = ["macros", "rt-multi-thread"] }
|
92
sample.html
Normal file
92
sample.html
Normal file
File diff suppressed because one or more lines are too long
54
src/main.rs
Normal file
54
src/main.rs
Normal file
|
@ -0,0 +1,54 @@
|
||||||
|
use tl;
|
||||||
|
use reqwest;
|
||||||
|
use serde_json::{self, Value, json};
|
||||||
|
use std::{fs::{write, File}, io::Write};
|
||||||
|
|
||||||
|
|
||||||
|
#[tokio::main]
|
||||||
|
async fn main() {
|
||||||
|
let url = "https://wiki.kmods.space/show/klib-res-timecrystal/itemDescriptor";
|
||||||
|
let webdata = reqwest::get(url).await.expect("request failed").text().await.expect("request failed");
|
||||||
|
|
||||||
|
|
||||||
|
let dom = tl::parse(webdata.as_str(), tl::ParserOptions::default()).expect("page is not html");
|
||||||
|
let parser = dom.parser();
|
||||||
|
let element = dom.get_element_by_id("__NUXT_DATA__").expect("__NUXT_DATA__ not found").get(parser).unwrap();
|
||||||
|
let data = element.inner_text(parser);
|
||||||
|
let mut json: Vec<Value> = serde_json::from_str(&data).unwrap();
|
||||||
|
//println!("{:#?}", json);
|
||||||
|
|
||||||
|
let mut v = json[0].clone();
|
||||||
|
recurse(&mut v, &mut json);
|
||||||
|
let result_b = v[1]["data"].as_object().unwrap().values().collect::<Vec<_>>();
|
||||||
|
let result = result_b.get(0).unwrap();
|
||||||
|
let mut file = File::create(format!("{}.json", &result["extraInformations"]["name"].as_str().unwrap())).unwrap();
|
||||||
|
file.write(serde_json::to_string_pretty(&result).unwrap().as_bytes()).expect("failed to write json");
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
fn recurse<'a>(val: &'a mut Value, src: &'a Vec<Value>) {
|
||||||
|
//println!("val before mutation: {}", val);
|
||||||
|
match val {
|
||||||
|
Value::Array(arr) => {
|
||||||
|
for val in arr.iter_mut() {
|
||||||
|
recurse(val, src);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Value::Object(map) => {
|
||||||
|
for (key, val) in map.iter_mut() {
|
||||||
|
recurse(val, src);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Value::Number(num) => {
|
||||||
|
let i = num.as_u64().unwrap();
|
||||||
|
let mut r = src.get(i as usize).unwrap().clone();
|
||||||
|
match r {
|
||||||
|
Value::Number(_) => {}
|
||||||
|
_ => { recurse(&mut r, src); }
|
||||||
|
}
|
||||||
|
*val = r;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
//println!("val after mutation: {}", val);
|
||||||
|
}
|
Loading…
Add table
Reference in a new issue