theBreadCompany/willdaten
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: theBreadCompany:6416cd5fd9f1c651e9141864521f66ee5cec60f0
Branches Tags
theBreadCompany:main
...
compare: theBreadCompany:16f369d73a6db2732b0f76daff6f6f934d255b8a
Branches Tags
theBreadCompany:main

2 commits
6416cd5fd9 ... 16f369d73a

Author SHA1 Message Date
theBreadCompany
16f369d73a async hell 2025-02-03 19:36:29 +01:00
theBreadCompany
e7cfc8217a things happened 2025-02-03 16:41:59 +01:00
20 changed files with 22275 additions and 588 deletions
Show stats Expand all files Collapse all files

6
.forgejo/workflows/pipeline.yml
View file

@ -1,8 +1,8 @@
name: Willdaten fetching pipeline name: Willdaten fetching pipeline
on: #on:
schedule: # schedule:
- cron: "0 * * * *" # run hourly # - cron: "0 * * * *" # run hourly
jobs: jobs:
process_data: process_data:

1041
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

3
datafetcher/Cargo.toml
View file

@ -8,4 +8,5 @@ chrono = "0.4.39"
reqwest = { version = "0.12.12", features = ["json"] } reqwest = { version = "0.12.12", features = ["json"] }
tokio = { version = "1.43.0", features = ["rt-multi-thread", "macros", "time"] } tokio = { version = "1.43.0", features = ["rt-multi-thread", "macros", "time"] }
dataworker = { path = "../dataworker" } dataworker = { path = "../dataworker" }
dotenvy = "0.15.7" dotenvy = "0.15.7"
sqlx = { version = "0.8.3", features = ["postgres"] }

96
datafetcher/src/main.rs
View file

@ -1,10 +1,13 @@
use chrono::Local; use chrono::Local;
use dataworker::diesel::{Connection, PgConnection};
use dataworker::dtos::Root; use dataworker::dtos::Root;
use dataworker::{insert_document, WilldatenError}; use dataworker::insert_document;
use sqlx::postgres::PgPoolOptions;
use dotenvy::dotenv; use dotenvy::dotenv;
use std::env; use std::env;
use std::io::Write; use std::io::Write;
use tokio::task;
use tokio::time::{self, sleep, Duration};
#[tokio::main] #[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> { async fn main() -> Result<(), Box<dyn std::error::Error>> {
@ -15,40 +18,75 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.collect(); .collect();
let token = env::var("VRN_TOKEN")?; let token = env::var("VRN_TOKEN")?;
let req = env::var("REQ")?; let req = env::var("REQ")?;
let db_url = env::var("DATABASE_URL")?; let url = env::var("DATABASE_URL")?;
let mut conn = PgConnection::establish(&db_url)?; let mut conn = PgPoolOptions::new()
.max_connections(5)
.connect(&url)
.await?;
let total = stations.len(); let total = stations.len();
let mut count = 0; let mut count = 0;
let client = reqwest::Client::new(); let client = reqwest::Client::new();
//let mut interval = time::interval(Duration::from_secs(1800));
for station in stations { loop {
count += 1; let mut handles = vec![];
let date = Local::now();
let req = req
.replace("TEMPLATE_ID", &station)
.replace("TEMPLATE_DATE", &date.format("%Y%m%d").to_string())
.replace("TEMPLATE_DATE", &date.format("%H%M").to_string());
print!("\x1B[2K\rFetching {}/{} ({}) ", count, total, station);
std::io::stdout().flush().unwrap(); std::io::stdout().flush().unwrap();
if let Ok(doc) = client //interval.tick().await;
.get(req) let date = Local::now();
.header("Authorization", &token) //println!("Ticked on {}", &date.format("%Y.%m.%d %H:%M"));
.send() for station in &stations {
.await? let station = station.clone();
.json::<Root>() let req = req.clone();
.await { let client = client.clone();
let place_name = doc.dm.itd_odv_assigned_stops.name_with_place.clone(); let mut conn = conn.clone();
match insert_document(doc, handles.push(task::spawn(async move {
&mut conn, count += 1;
) { let req = req
Ok(_) => {} .replace("TEMPLATE_ID", &station)
Err(ref e) if e.downcast_ref::<WilldatenError>().is_some() => println!("\x1B[2K\rMissing arrivals for {} ({})!", station, place_name), .replace("TEMPLATE_DATE", &date.format("%Y%m%d").to_string())
Err(e) => println!("\x1B[2K\rError inserting document: {}", e), .replace("TEMPLATE_TIME", &date.format("%H%M").to_string());
print!("\x1B[2K\rFetching {}/{} ({}) ", count, total, station);
std::io::stdout().flush().unwrap();
if let Ok(doc) = client
.get(req.replace("TEMPLATE_DEPARR", "arr"))
//.header("Authorization", &token)
.send()
.await
.unwrap()
.json::<Root>()
.await
{
match insert_document(doc, &mut conn).await {
Ok(_) => {}
Err(e) => println!("\x1B[2K\rError inserting document: {}", e),
}
} else {
println!("\x1B[2K\rFailed to fetch arrivals for station {}", station);
}
if let Ok(doc) = client
.get(req.replace("TEMPLATE_DEPARR", "dep"))
//.header("Authorization", &token)
.send()
.await
.unwrap()
.json::<Root>()
.await
{
match insert_document(doc, &mut conn).await {
Ok(_) => {}
Err(e) => println!("\x1B[2K\rError inserting document: {}", e),
}
} else {
println!("\x1B[2K\rFailed to fetch departures for station {}", station);
}
}));
} }
} else { println!("\x1B[2K\rFailed to fetch station {}", station); } for handle in handles {
handle.await?;
sleep(Duration::from_millis(100)).await;
}
count = 0;
} }
println!();
Ok(())
} }

4
dataworker/Cargo.toml
View file

@ -5,10 +5,12 @@ edition = "2021"
[dependencies] [dependencies]
chrono = "0.4.39" chrono = "0.4.39"
diesel = { version = "2.2.6", features = ["64-column-tables", "chrono", "postgres", "postgres_backend", "uuid"] }
dotenvy = "0.15.7" dotenvy = "0.15.7"
serde = { version = "1.0.217", features = ["derive"] } serde = { version = "1.0.217", features = ["derive"] }
serde_json = "1.0.135" serde_json = "1.0.135"
sqlx = { version = "0.8.3", features = ["postgres", "derive", "uuid", "chrono", "runtime-tokio"] }
thiserror = "2.0.11" thiserror = "2.0.11"
tokio = { version = "1.43.0", features = ["full"] } tokio = { version = "1.43.0", features = ["full"] }
uuid = { version = "1.12.1", features = ["v5"] } uuid = { version = "1.12.1", features = ["v5"] }
futures-util = "0.3.31"
tokio-stream = { version = "0.1.17", features = ["fs"] }

334
dataworker/input/xslt_dm_request/10024717.json Normal file
View file

@ -0,0 +1,334 @@
{
"parameters": [
{
"name": "serverID",
"value": "TRIAS-3"
},
{
"name": "requestID",
"value": "0"
},
{
"name": "sessionID",
"value": "0"
},
{
"name": "calcTime",
"value": "272.106"
},
{
"name": "serverTime",
"value": "2024-12-18T19:00:42"
},
{
"name": "logRequestId",
"value": "f2ab9c9dadd0df2e"
}
],
"dm": {
"input": {
"input": "10024717"
},
"points": {
"point": {
"usage": "dm",
"type": "any",
"name": "Darmstadt, Eberstadt Walther-Rathenau-Straße",
"stateless": "10024717",
"anyType": "stop",
"sort": "2",
"quality": "100000",
"best": "1",
"object": "Eberstadt Walther-Rathenau-Straße",
"ref": {
"id": "10024717",
"gid": "de:06411:24717",
"omc": "6411000",
"placeID": "903",
"place": "Darmstadt",
"coords": "8.6353431,49.8167590"
},
"infos": null
}
},
"itdOdvAssignedStops": {
"stopID": "10024717",
"name": "Eberstadt Walther-Rathenau-Straße",
"x": "8.6353431",
"y": "49.8167590",
"mapName": "WGS84[DD.ddddd]",
"value": "10024717:Eberstadt Walther-Rathenau-Straße",
"place": "Darmstadt",
"nameWithPlace": "Darmstadt-Eberstadt Walther-Rathenau-Straße",
"distanceTime": "0",
"isTransferStop": "0",
"vm": "100",
"gid": "de:06411:24717"
}
},
"arr": {
"input": {
"input": ""
},
"points": null
},
"dateTime": {
"deparr": "dep",
"ttpFrom": "20241101",
"ttpTo": "20251213",
"year": "2024",
"month": "12",
"day": "20",
"hour": "12",
"minute": "0"
},
"dateRange": [
{
"day": "18",
"month": "12",
"year": "2024",
"weekday": "4"
},
{
"day": "19",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "20",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "21",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "22",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "23",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "24",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "25",
"month": "12",
"year": "2024",
"weekday": "4"
},
{
"day": "26",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "27",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "28",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "29",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "30",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "31",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "01",
"month": "01",
"year": "2025",
"weekday": "4"
}
],
"option": {
"ptOption": {
"active": "1",
"maxChanges": "9",
"maxTime": "360",
"maxWait": "120",
"routeType": "LEASTTIME",
"changeSpeed": "normal",
"lineRestriction": "403",
"useProxFootSearch": "0",
"useProxFootSearchOrigin": "0",
"useProxFootSearchDestination": "0",
"bike": "0",
"plane": "0",
"noCrowded": "0",
"noSolidStairs": "0",
"noEscalators": "0",
"noElevators": "0",
"lowPlatformVhcl": "0",
"wheelchair": "0",
"needElevatedPlt": "0",
"assistance": "0",
"SOSAvail": "0",
"noLonelyTransfer": "0",
"illumTransfer": "0",
"overgroundTransfer": "0",
"noInsecurePlaces": "0",
"privateTransport": "0",
"excludedMeans": [
{
"means": "Zug",
"value": "0",
"selected": "0"
},
{
"means": "S-Bahn",
"value": "1",
"selected": "0"
},
{
"means": "U-Bahn",
"value": "2",
"selected": "0"
},
{
"means": "Stadtbahn",
"value": "3",
"selected": "0"
},
{
"means": "Straßen-/Trambahn",
"value": "4",
"selected": "0"
},
{
"means": "Stadtbus",
"value": "5",
"selected": "0"
},
{
"means": "Regionalbus",
"value": "6",
"selected": "0"
},
{
"means": "Schnellbus",
"value": "7",
"selected": "0"
},
{
"means": "Seil-/Zahnradbahn",
"value": "8",
"selected": "0"
},
{
"means": "Schiff",
"value": "9",
"selected": "0"
},
{
"means": "AST (flächengebunde",
"value": "10",
"selected": "0"
},
{
"means": "Sonstige",
"value": "11",
"selected": "0"
},
{
"means": "Flugzeug",
"value": "12",
"selected": "0"
},
{
"means": "Zug (Nahverkehr)",
"value": "13",
"selected": "0"
},
{
"means": "Zug (Fernverkehr)",
"value": "14",
"selected": "0"
},
{
"means": "Zug Fernv m Zuschlag",
"value": "15",
"selected": "0"
},
{
"means": "Zug Fernv m spez Fpr",
"value": "16",
"selected": "0"
},
{
"means": "SEV Schienenersatzv",
"value": "17",
"selected": "0"
},
{
"means": "Zug Shuttle",
"value": "18",
"selected": "0"
},
{
"means": "Bürgerbus",
"value": "19",
"selected": "0"
}
],
"activeImp": "-1",
"activeCom": "-1",
"activeSec": "-1"
}
},
"message": [
{
"name": "code",
"value": "-4050"
},
{
"name": "error",
"value": "no serving lines found"
},
{
"name": "type",
"value": ""
},
{
"name": "module",
"value": "BROKER"
}
],
"servingLines": {
"lines": null
},
"departureList": null
}

1
dataworker/input/xslt_dm_request/24016923.json Normal file
View file

@ -0,0 +1 @@
{ "parameters": [ { "name":"serverID", "value":"TRIAS-4" }, { "name":"requestID", "value":"0" }, { "name":"sessionID", "value":"0" }, { "name":"calcTime", "value":"272.484" }, { "name":"serverTime", "value":"2024-12-19T01:35:42" }, { "name":"logRequestId", "value":"f7fda445e1c95263" } ], "dm": { "input": { "input":"24016923" }, "points": { "point": { "usage":"dm", "type":"any", "name":"Gronau (Westf), Spinnerei Deutschland", "stateless":"24016923", "anyType":"stop", "sort":"2", "quality":"100000", "best":"1", "object":"Spinnerei Deutschland", "ref": { "id":"24016923", "gid":"de:05554:16923", "omc":"5554020", "placeID":"9", "place":"Gronau (Westf)", "coords":"7.0121503,52.2230738" }, "infos": null } }, "itdOdvAssignedStops": { "stopID":"24016923", "name":"Spinnerei Deutschland", "x":"7.0121503", "y":"52.2230738", "mapName":"WGS84[DD.ddddd]", "value":"24016923:Spinnerei Deutschland", "place":"Gronau (Westf)", "nameWithPlace":"Gronau, Spinnerei Deutschland", "distanceTime":"0", "isTransferStop":"0", "vm":"100", "gid":"de:05554:16923" } }, "arr": { "input": { "input":"" }, "points": null }, "dateTime": { "deparr":"dep", "ttpFrom":"20241101", "ttpTo":"20251213", "year":"2024", "month":"12", "day":"20", "hour":"12", "minute":"0" }, "dateRange": [ { "day":"19", "month":"12", "year":"2024", "weekday":"5" }, { "day":"20", "month":"12", "year":"2024", "weekday":"6" }, { "day":"21", "month":"12", "year":"2024", "weekday":"7" }, { "day":"22", "month":"12", "year":"2024", "weekday":"1" }, { "day":"23", "month":"12", "year":"2024", "weekday":"2" }, { "day":"24", "month":"12", "year":"2024", "weekday":"3" }, { "day":"25", "month":"12", "year":"2024", "weekday":"4" }, { "day":"26", "month":"12", "year":"2024", "weekday":"5" }, { "day":"27", "month":"12", "year":"2024", "weekday":"6" }, { "day":"28", "month":"12", "year":"2024", "weekday":"7" }, { "day":"29", "month":"12", "year":"2024", "weekday":"1" }, { "day":"30", "month":"12", "year":"2024", "weekday":"2" }, { "day":"31", "month":"12", "year":"2024", "weekday":"3" }, { "day":"01", "month":"01", "year":"2025", "weekday":"4" }, { "day":"02", "month":"01", "year":"2025", "weekday":"5" } ], "option": { "ptOption": { "active":"1", "maxChanges":"9", "maxTime":"360", "maxWait":"120", "routeType":"LEASTTIME", "changeSpeed":"normal", "lineRestriction":"403", "useProxFootSearch":"0", "useProxFootSearchOrigin":"0", "useProxFootSearchDestination":"0", "bike":"0", "plane":"0", "noCrowded":"0", "noSolidStairs":"0", "noEscalators":"0", "noElevators":"0", "lowPlatformVhcl":"0", "wheelchair":"0", "needElevatedPlt":"0", "assistance":"0", "SOSAvail":"0", "noLonelyTransfer":"0", "illumTransfer":"0", "overgroundTransfer":"0", "noInsecurePlaces":"0", "privateTransport":"0", "excludedMeans": [ { "means":"Zug", "value":"0", "selected":"0" }, { "means":"S-Bahn", "value":"1", "selected":"0" }, { "means":"U-Bahn", "value":"2", "selected":"0" }, { "means":"Stadtbahn", "value":"3", "selected":"0" }, { "means":"Straßen-/Trambahn", "value":"4", "selected":"0" }, { "means":"Stadtbus", "value":"5", "selected":"0" }, { "means":"Regionalbus", "value":"6", "selected":"0" }, { "means":"Schnellbus", "value":"7", "selected":"0" }, { "means":"Seil-/Zahnradbahn", "value":"8", "selected":"0" }, { "means":"Schiff", "value":"9", "selected":"0" }, { "means":"AST (flächengebunde", "value":"10", "selected":"0" }, { "means":"Sonstige", "value":"11", "selected":"0" }, { "means":"Flugzeug", "value":"12", "selected":"0" }, { "means":"Zug (Nahverkehr)", "value":"13", "selected":"0" }, { "means":"Zug (Fernverkehr)", "value":"14", "selected":"0" }, { "means":"Zug Fernv m Zuschlag", "value":"15", "selected":"0" }, { "means":"Zug Fernv m spez Fpr", "value":"16", "selected":"0" }, { "means":"SEV Schienenersatzv", "value":"17", "selected":"0" }, { "means":"Zug Shuttle", "value":"18", "selected":"0" }, { "means":"Bürgerbus", "value":"19", "selected":"0" } ], "activeImp":"-1", "activeCom":"-1", "activeSec":"-1" } }, "message": [ { "name":"code", "value":"-4050" }, { "name":"error", "value":"no serving lines found" }, { "name":"type", "value":"" }, { "name":"module", "value":"BROKER" } ], "servingLines": { "lines": null }, "departureList": null }

5121
dataworker/input/xslt_dm_request/25000001.json Normal file
View file

File diff suppressed because it is too large Load diff

441
dataworker/input/xslt_dm_request/25205074.json Normal file
View file

@ -0,0 +1,441 @@
{
"parameters": [
{
"name": "serverID",
"value": "TRIAS-3"
},
{
"name": "requestID",
"value": "0"
},
{
"name": "sessionID",
"value": "0"
},
{
"name": "calcTime",
"value": "325.315"
},
{
"name": "serverTime",
"value": "2024-12-19T03:20:45"
},
{
"name": "logRequestId",
"value": "f2ab9c9dadd19ef7"
}
],
"dm": {
"input": {
"input": "25205074"
},
"points": {
"point": {
"usage": "dm",
"type": "any",
"name": "Schapen (Emsl), Schapen, Schwerdt",
"stateless": "25205074",
"anyType": "stop",
"sort": "2",
"quality": "100000",
"best": "1",
"object": "Schapen, Schwerdt",
"ref": {
"id": "25205074",
"gid": "de:03454:5074",
"omc": "3454046",
"placeID": "1",
"place": "Schapen (Emsl)"
},
"infos": null
}
},
"itdOdvAssignedStops": {
"stopID": "25205074",
"name": "Schapen, Schwerdt",
"value": "25205074:Schapen, Schwerdt",
"place": "Schapen (Emsl)",
"nameWithPlace": "Schapen, Schwerdt",
"distanceTime": "0",
"isTransferStop": "0",
"vm": "100",
"gid": "de:03454:5074"
}
},
"arr": {
"input": {
"input": ""
},
"points": null
},
"dateTime": {
"deparr": "dep",
"ttpFrom": "20241101",
"ttpTo": "20251213",
"year": "2024",
"month": "12",
"day": "20",
"hour": "12",
"minute": "0"
},
"dateRange": [
{
"day": "19",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "20",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "21",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "22",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "23",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "24",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "25",
"month": "12",
"year": "2024",
"weekday": "4"
},
{
"day": "26",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "27",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "28",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "29",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "30",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "31",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "01",
"month": "01",
"year": "2025",
"weekday": "4"
},
{
"day": "02",
"month": "01",
"year": "2025",
"weekday": "5"
}
],
"option": {
"ptOption": {
"active": "1",
"maxChanges": "9",
"maxTime": "360",
"maxWait": "120",
"routeType": "LEASTTIME",
"changeSpeed": "normal",
"lineRestriction": "403",
"useProxFootSearch": "0",
"useProxFootSearchOrigin": "0",
"useProxFootSearchDestination": "0",
"bike": "0",
"plane": "0",
"noCrowded": "0",
"noSolidStairs": "0",
"noEscalators": "0",
"noElevators": "0",
"lowPlatformVhcl": "0",
"wheelchair": "0",
"needElevatedPlt": "0",
"assistance": "0",
"SOSAvail": "0",
"noLonelyTransfer": "0",
"illumTransfer": "0",
"overgroundTransfer": "0",
"noInsecurePlaces": "0",
"privateTransport": "0",
"excludedMeans": [
{
"means": "Zug",
"value": "0",
"selected": "0"
},
{
"means": "S-Bahn",
"value": "1",
"selected": "0"
},
{
"means": "U-Bahn",
"value": "2",
"selected": "0"
},
{
"means": "Stadtbahn",
"value": "3",
"selected": "0"
},
{
"means": "Straßen-/Trambahn",
"value": "4",
"selected": "0"
},
{
"means": "Stadtbus",
"value": "5",
"selected": "0"
},
{
"means": "Regionalbus",
"value": "6",
"selected": "0"
},
{
"means": "Schnellbus",
"value": "7",
"selected": "0"
},
{
"means": "Seil-/Zahnradbahn",
"value": "8",
"selected": "0"
},
{
"means": "Schiff",
"value": "9",
"selected": "0"
},
{
"means": "AST (flächengebunde",
"value": "10",
"selected": "0"
},
{
"means": "Sonstige",
"value": "11",
"selected": "0"
},
{
"means": "Flugzeug",
"value": "12",
"selected": "0"
},
{
"means": "Zug (Nahverkehr)",
"value": "13",
"selected": "0"
},
{
"means": "Zug (Fernverkehr)",
"value": "14",
"selected": "0"
},
{
"means": "Zug Fernv m Zuschlag",
"value": "15",
"selected": "0"
},
{
"means": "Zug Fernv m spez Fpr",
"value": "16",
"selected": "0"
},
{
"means": "SEV Schienenersatzv",
"value": "17",
"selected": "0"
},
{
"means": "Zug Shuttle",
"value": "18",
"selected": "0"
},
{
"means": "Bürgerbus",
"value": "19",
"selected": "0"
}
],
"activeImp": "-1",
"activeCom": "-1",
"activeSec": "-1"
}
},
"servingLines": {
"lines": {
"line": {
"mode": {
"name": "Stadtbus 133",
"number": "133",
"product": "Stadtbus",
"productId": "1",
"type": "5",
"code": "3",
"destination": "Schapen, Huilmanns Möörken",
"destID": "25209129",
"desc": "Spelle, Bahnhof - Lingen, ZOB 5",
"timetablePeriod": "WEB (VBN) Fahrpläne 2025",
"diva": {
"branch": "66",
"line": "66133",
"supplement": " ",
"dir": "R",
"project": "j25",
"network": "vbx",
"stateless": "vbx:66133: :R:j25",
"globalId": "de:VBN-VGE:133:4",
"tripCode": "0",
"operator": "Hülsmann Reisen",
"opCode": "2480",
"vF": "20241101",
"vTo": "20251213",
"lineDisplay": "line",
"attrs": [
{
"name": "OperatorURL",
"value": "https://www.reise-freunde.de/"
}
]
}
},
"index": "5:0"
}
}
},
"departureList": [
{
"stopID": "25205074",
"area": "0",
"platform": "",
"platformName": "",
"stopName": "Schapen, Schwerdt",
"nameWO": "Schapen, Schwerdt",
"countdown": "64",
"dateTime": {
"year": "2024",
"month": "12",
"day": "20",
"weekday": "6",
"hour": "13",
"minute": "4"
},
"servingLine": {
"key": "4",
"code": "3",
"number": "133",
"symbol": "133",
"motType": "5",
"mtSubcode": "0",
"realtime": "0",
"direction": "Schapen, Huilmanns Möörken",
"directionFrom": "Spelle, Bahnhof",
"name": "Stadtbus",
"liErgRiProj": {
"line": "66133",
"project": "j25",
"direction": "R",
"supplement": " ",
"network": "vbx",
"gid": "de:VBN-VGE:133:4"
},
"destID": "25209129",
"stateless": "vbx:66133: :R:j25",
"lineDisplay": "line"
},
"operator": {
"code": "2480",
"name": "Hülsmann Reisen",
"publicCode": ""
}
},
{
"stopID": "25205074",
"area": "0",
"platform": "",
"platformName": "",
"stopName": "Schapen, Schwerdt",
"nameWO": "Schapen, Schwerdt",
"countdown": "111",
"dateTime": {
"year": "2024",
"month": "12",
"day": "20",
"weekday": "6",
"hour": "13",
"minute": "51"
},
"servingLine": {
"key": "3",
"code": "3",
"number": "133",
"symbol": "133",
"motType": "5",
"mtSubcode": "0",
"realtime": "0",
"direction": "Schapen, Huilmanns Möörken",
"directionFrom": "Spelle, Rathaus",
"name": "Stadtbus",
"liErgRiProj": {
"line": "66133",
"project": "j25",
"direction": "R",
"supplement": " ",
"network": "vbx",
"gid": "de:VBN-VGE:133:4"
},
"destID": "25209129",
"stateless": "vbx:66133: :R:j25",
"lineDisplay": "line"
},
"operator": {
"code": "2480",
"name": "Hülsmann Reisen",
"publicCode": ""
}
}
]
}

398
dataworker/input/xslt_dm_request/31201864.json Normal file
View file

@ -0,0 +1,398 @@
{
"parameters": [
{
"name": "serverID",
"value": "TRIAS-1"
},
{
"name": "requestID",
"value": "0"
},
{
"name": "sessionID",
"value": "0"
},
{
"name": "calcTime",
"value": "345.166"
},
{
"name": "serverTime",
"value": "2024-12-19T09:12:55"
},
{
"name": "logRequestId",
"value": "acdae4a9972a52f5"
}
],
"dm": {
"input": {
"input": "31201864"
},
"points": {
"point": {
"usage": "dm",
"type": "any",
"name": "Tanna, Zollgrün, Schülerhaltestelle",
"stateless": "31201864",
"anyType": "stop",
"sort": "2",
"quality": "100000",
"best": "1",
"object": "Zollgrün, Schülerhaltestelle",
"ref": {
"id": "31201864",
"gid": "de:16075:2862061",
"omc": "16075132",
"placeID": "1",
"place": "Tanna",
"coords": "11.8304260,50.5149650"
},
"infos": null
}
},
"itdOdvAssignedStops": {
"stopID": "31201864",
"name": "Zollgrün, Schülerhaltestelle",
"x": "11.8304260",
"y": "50.5149650",
"mapName": "WGS84[DD.ddddd]",
"value": "31201864:Zollgrün, Schülerhaltestelle",
"place": "Tanna",
"nameWithPlace": "Zollgrün, Schülerhaltestelle",
"distanceTime": "0",
"isTransferStop": "0",
"vm": "100",
"gid": "de:16075:2862061"
}
},
"arr": {
"input": {
"input": ""
},
"points": null
},
"dateTime": {
"deparr": "dep",
"ttpFrom": "20241101",
"ttpTo": "20251213",
"year": "2024",
"month": "12",
"day": "20",
"hour": "12",
"minute": "0"
},
"dateRange": [
{
"day": "19",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "20",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "21",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "22",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "23",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "24",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "25",
"month": "12",
"year": "2024",
"weekday": "4"
},
{
"day": "26",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "27",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "28",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "29",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "30",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "31",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "01",
"month": "01",
"year": "2025",
"weekday": "4"
},
{
"day": "02",
"month": "01",
"year": "2025",
"weekday": "5"
}
],
"option": {
"ptOption": {
"active": "1",
"maxChanges": "9",
"maxTime": "360",
"maxWait": "120",
"routeType": "LEASTTIME",
"changeSpeed": "normal",
"lineRestriction": "403",
"useProxFootSearch": "0",
"useProxFootSearchOrigin": "0",
"useProxFootSearchDestination": "0",
"bike": "0",
"plane": "0",
"noCrowded": "0",
"noSolidStairs": "0",
"noEscalators": "0",
"noElevators": "0",
"lowPlatformVhcl": "0",
"wheelchair": "0",
"needElevatedPlt": "0",
"assistance": "0",
"SOSAvail": "0",
"noLonelyTransfer": "0",
"illumTransfer": "0",
"overgroundTransfer": "0",
"noInsecurePlaces": "0",
"privateTransport": "0",
"excludedMeans": [
{
"means": "Zug",
"value": "0",
"selected": "0"
},
{
"means": "S-Bahn",
"value": "1",
"selected": "0"
},
{
"means": "U-Bahn",
"value": "2",
"selected": "0"
},
{
"means": "Stadtbahn",
"value": "3",
"selected": "0"
},
{
"means": "Straßen-/Trambahn",
"value": "4",
"selected": "0"
},
{
"means": "Stadtbus",
"value": "5",
"selected": "0"
},
{
"means": "Regionalbus",
"value": "6",
"selected": "0"
},
{
"means": "Schnellbus",
"value": "7",
"selected": "0"
},
{
"means": "Seil-/Zahnradbahn",
"value": "8",
"selected": "0"
},
{
"means": "Schiff",
"value": "9",
"selected": "0"
},
{
"means": "AST (flächengebunde",
"value": "10",
"selected": "0"
},
{
"means": "Sonstige",
"value": "11",
"selected": "0"
},
{
"means": "Flugzeug",
"value": "12",
"selected": "0"
},
{
"means": "Zug (Nahverkehr)",
"value": "13",
"selected": "0"
},
{
"means": "Zug (Fernverkehr)",
"value": "14",
"selected": "0"
},
{
"means": "Zug Fernv m Zuschlag",
"value": "15",
"selected": "0"
},
{
"means": "Zug Fernv m spez Fpr",
"value": "16",
"selected": "0"
},
{
"means": "SEV Schienenersatzv",
"value": "17",
"selected": "0"
},
{
"means": "Zug Shuttle",
"value": "18",
"selected": "0"
},
{
"means": "Bürgerbus",
"value": "19",
"selected": "0"
}
],
"activeImp": "-1",
"activeCom": "-1",
"activeSec": "-1"
}
},
"servingLines": {
"lines": {
"line": {
"mode": {
"name": "Bus 730",
"number": "730",
"product": "Bus",
"productId": "1",
"type": "5",
"code": "3",
"destination": "Saalburg, Wartehalle",
"destID": "31201843",
"desc": "",
"timetablePeriod": "Jahresfahrplan 2024/2025",
"diva": {
"branch": "25",
"line": "25730",
"supplement": " ",
"dir": "H",
"project": "j25",
"network": "nth",
"stateless": "nth:25730: :H:j25",
"globalId": "de:VMT:16075132|Bus|730|730|kombusbt:",
"tripCode": "0",
"operator": "KomBus Verkehr GmbH (KomBus)",
"opCode": "96",
"vF": "20241101",
"vTo": "20251213",
"lineDisplay": "line",
"attrs": []
}
},
"index": "5:0"
}
}
},
"departureList": {
"departure": {
"stopID": "31201864",
"x": "11.8304260",
"y": "50.5149650",
"mapName": "WGS84[DD.ddddd]",
"area": "1",
"platform": "1",
"platformName": "",
"stopName": "Zollgrün, Schülerhaltestelle",
"nameWO": "Zollgrün, Schülerhaltestelle",
"countdown": "230",
"dateTime": {
"year": "2024",
"month": "12",
"day": "20",
"weekday": "6",
"hour": "15",
"minute": "50"
},
"servingLine": {
"key": "14",
"code": "3",
"number": "730",
"symbol": "730",
"motType": "5",
"mtSubcode": "0",
"realtime": "0",
"direction": "Saalburg, Wartehalle",
"directionFrom": "Tanna, Schule",
"name": "Bus",
"liErgRiProj": {
"line": "25730",
"project": "j25",
"direction": "H",
"supplement": " ",
"network": "nth",
"gid": "de:VMT:16075132|Bus|730|730|kombusbt:"
},
"destID": "31201843",
"stateless": "nth:25730: :H:j25",
"lineDisplay": "line"
},
"operator": {
"code": "96",
"name": "KomBus Verkehr GmbH (KomBus)",
"publicCode": ""
}
}
}
}

7800
dataworker/input/xslt_dm_request/6002417.json Normal file
View file

File diff suppressed because it is too large Load diff

617
dataworker/input/xslt_dm_request/71007418.json Normal file
View file

@ -0,0 +1,617 @@
{
"parameters": [
{
"name": "serverID",
"value": "TRIAS-2"
},
{
"name": "requestID",
"value": "0"
},
{
"name": "sessionID",
"value": "0"
},
{
"name": "calcTime",
"value": "318.042"
},
{
"name": "serverTime",
"value": "2024-12-19T19:06:31"
},
{
"name": "logRequestId",
"value": "e890028844c8a933"
}
],
"dm": {
"input": {
"input": "71007418"
},
"points": {
"point": {
"usage": "dm",
"type": "any",
"name": "Brehain la Ville, Bréhain-la-Cour Cabine téléphonique",
"stateless": "71007418",
"anyType": "stop",
"sort": "2",
"quality": "100000",
"best": "1",
"object": "Bréhain-la-Cour Cabine téléphonique",
"ref": {
"id": "71007418",
"gid": "fr:24054:7418",
"omc": "24054096",
"placeID": "1",
"place": "Brehain la Ville",
"coords": "5.8933795,49.4457553"
},
"infos": null
}
},
"itdOdvAssignedStops": {
"stopID": "71007418",
"name": "Bréhain-la-Cour Cabine téléphonique",
"x": "5.8933795",
"y": "49.4457553",
"mapName": "WGS84[DD.ddddd]",
"value": "71007418:Bréhain-la-Cour Cabine téléphonique",
"place": "Brehain la Ville",
"nameWithPlace": "Bréhain-la-Cour Cabine téléphonique",
"distanceTime": "0",
"isTransferStop": "0",
"vm": "100",
"gid": "fr:24054:7418"
}
},
"arr": {
"input": {
"input": ""
},
"points": null
},
"dateTime": {
"deparr": "dep",
"ttpFrom": "20241101",
"ttpTo": "20251213",
"year": "2024",
"month": "12",
"day": "20",
"hour": "12",
"minute": "0"
},
"dateRange": [
{
"day": "19",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "20",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "21",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "22",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "23",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "24",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "25",
"month": "12",
"year": "2024",
"weekday": "4"
},
{
"day": "26",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "27",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "28",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "29",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "30",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "31",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "01",
"month": "01",
"year": "2025",
"weekday": "4"
},
{
"day": "02",
"month": "01",
"year": "2025",
"weekday": "5"
}
],
"option": {
"ptOption": {
"active": "1",
"maxChanges": "9",
"maxTime": "360",
"maxWait": "120",
"routeType": "LEASTTIME",
"changeSpeed": "normal",
"lineRestriction": "403",
"useProxFootSearch": "0",
"useProxFootSearchOrigin": "0",
"useProxFootSearchDestination": "0",
"bike": "0",
"plane": "0",
"noCrowded": "0",
"noSolidStairs": "0",
"noEscalators": "0",
"noElevators": "0",
"lowPlatformVhcl": "0",
"wheelchair": "0",
"needElevatedPlt": "0",
"assistance": "0",
"SOSAvail": "0",
"noLonelyTransfer": "0",
"illumTransfer": "0",
"overgroundTransfer": "0",
"noInsecurePlaces": "0",
"privateTransport": "0",
"excludedMeans": [
{
"means": "Zug",
"value": "0",
"selected": "0"
},
{
"means": "S-Bahn",
"value": "1",
"selected": "0"
},
{
"means": "U-Bahn",
"value": "2",
"selected": "0"
},
{
"means": "Stadtbahn",
"value": "3",
"selected": "0"
},
{
"means": "Straßen-/Trambahn",
"value": "4",
"selected": "0"
},
{
"means": "Stadtbus",
"value": "5",
"selected": "0"
},
{
"means": "Regionalbus",
"value": "6",
"selected": "0"
},
{
"means": "Schnellbus",
"value": "7",
"selected": "0"
},
{
"means": "Seil-/Zahnradbahn",
"value": "8",
"selected": "0"
},
{
"means": "Schiff",
"value": "9",
"selected": "0"
},
{
"means": "AST (flächengebunde",
"value": "10",
"selected": "0"
},
{
"means": "Sonstige",
"value": "11",
"selected": "0"
},
{
"means": "Flugzeug",
"value": "12",
"selected": "0"
},
{
"means": "Zug (Nahverkehr)",
"value": "13",
"selected": "0"
},
{
"means": "Zug (Fernverkehr)",
"value": "14",
"selected": "0"
},
{
"means": "Zug Fernv m Zuschlag",
"value": "15",
"selected": "0"
},
{
"means": "Zug Fernv m spez Fpr",
"value": "16",
"selected": "0"
},
{
"means": "SEV Schienenersatzv",
"value": "17",
"selected": "0"
},
{
"means": "Zug Shuttle",
"value": "18",
"selected": "0"
},
{
"means": "Bürgerbus",
"value": "19",
"selected": "0"
}
],
"activeImp": "-1",
"activeCom": "-1",
"activeSec": "-1"
}
},
"servingLines": {
"lines": [
{
"mode": {
"name": "Bus inter-urbain 54S117",
"number": "54S117",
"product": "Bus inter-urbain",
"productId": "2",
"type": "6",
"code": "3",
"destination": "Mercy-le-Bas",
"destID": "71007546",
"desc": "",
"timetablePeriod": "CG54 Meurthe-et-Moselle Herbst 2024 (12.Dez.2024 bis 31.Dez.",
"diva": {
"branch": "54",
"line": "5454S",
"supplement": "P",
"dir": "R",
"project": "54r",
"network": "als",
"stateless": "als:5454S:P:R:54r",
"tripCode": "0",
"operator": "FLUO 54",
"opCode": "39",
"vF": "20241101",
"vTo": "20251213",
"lineDisplay": "line",
"specialFares": "ROUTE_IN_TRANSPORT_AUTHORITY_AREA_NO_FARE_CALCULATION",
"attrs": [
{
"name": "OperatorURL",
"value": "https://www.fluo.eu"
}
]
}
},
"index": "6:0"
},
{
"mode": {
"name": "Bus inter-urbain 54S130",
"number": "54S130",
"product": "Bus inter-urbain",
"productId": "2",
"type": "6",
"code": "3",
"destination": "Longwy",
"destID": "71007583",
"desc": "",
"timetablePeriod": "CG54 Meurthe-et-Moselle Herbst 2024 (12.Dez.2024 bis 31.Dez.",
"diva": {
"branch": "54",
"line": "5454S",
"supplement": "U",
"dir": "R",
"project": "54r",
"network": "als",
"stateless": "als:5454S:U:R:54r",
"tripCode": "0",
"operator": "FLUO 54",
"opCode": "39",
"vF": "20241101",
"vTo": "20251213",
"lineDisplay": "line",
"specialFares": "ROUTE_IN_TRANSPORT_AUTHORITY_AREA_NO_FARE_CALCULATION",
"attrs": [
{
"name": "OperatorURL",
"value": "https://www.fluo.eu"
}
]
}
},
"index": "6:1"
},
{
"mode": {
"name": "Bus inter-urbain 54S156",
"number": "54S156",
"product": "Bus inter-urbain",
"productId": "2",
"type": "6",
"code": "3",
"destination": "Audun-le-Roman",
"destID": "71055897",
"desc": "",
"timetablePeriod": "CG54 Meurthe-et-Moselle Herbst 2024 (12.Dez.2024 bis 31.Dez.",
"diva": {
"branch": "54",
"line": "5454T",
"supplement": "4",
"dir": "R",
"project": "54r",
"network": "als",
"stateless": "als:5454T:4:R:54r",
"tripCode": "0",
"operator": "FLUO 54",
"opCode": "39",
"vF": "20241101",
"vTo": "20251213",
"lineDisplay": "line",
"specialFares": "ROUTE_IN_TRANSPORT_AUTHORITY_AREA_NO_FARE_CALCULATION",
"attrs": [
{
"name": "OperatorURL",
"value": "https://www.fluo.eu"
}
]
}
},
"index": "6:2"
}
]
},
"departureList": [
{
"stopID": "71007418",
"x": "5.8933884",
"y": "49.4458254",
"mapName": "WGS84[DD.ddddd]",
"area": "0",
"platform": "2",
"platformName": "",
"stopName": "Bréhain-la-Cour Cabine téléphonique",
"nameWO": "Bréhain-la-Cour Cabine téléphonique",
"countdown": "307",
"dateTime": {
"year": "2024",
"month": "12",
"day": "20",
"weekday": "6",
"hour": "17",
"minute": "7"
},
"servingLine": {
"key": "2",
"code": "3",
"number": "54S156",
"symbol": "54S15",
"motType": "6",
"mtSubcode": "0",
"realtime": "0",
"direction": "Audun-le-Roman",
"directionFrom": "Villerupt Collège",
"name": "Bus inter-urbain",
"liErgRiProj": {
"line": "5454T",
"project": "54r",
"direction": "R",
"supplement": "4",
"network": "als"
},
"destID": "71055897",
"stateless": "als:5454T:4:R:54r",
"lineDisplay": "line"
},
"operator": {
"code": "39",
"name": "FLUO 54",
"publicCode": ""
}
},
{
"stopID": "71007418",
"x": "5.8933884",
"y": "49.4457261",
"mapName": "WGS84[DD.ddddd]",
"area": "0",
"platform": "1",
"platformName": "",
"stopName": "Bréhain-la-Cour Cabine téléphonique",
"nameWO": "Bréhain-la-Cour Cabine téléphonique",
"countdown": "337",
"dateTime": {
"year": "2024",
"month": "12",
"day": "20",
"weekday": "6",
"hour": "17",
"minute": "37"
},
"servingLine": {
"key": "2",
"code": "3",
"number": "54S117",
"symbol": "54S11",
"motType": "6",
"mtSubcode": "0",
"realtime": "0",
"direction": "Mercy-le-Bas",
"directionFrom": "Longwy Lycée Albert Mézières",
"name": "Bus inter-urbain",
"liErgRiProj": {
"line": "5454S",
"project": "54r",
"direction": "R",
"supplement": "P",
"network": "als"
},
"destID": "71007546",
"stateless": "als:5454S:P:R:54r",
"lineDisplay": "line"
},
"operator": {
"code": "39",
"name": "FLUO 54",
"publicCode": ""
}
},
{
"stopID": "71007418",
"x": "5.8933884",
"y": "49.4458254",
"mapName": "WGS84[DD.ddddd]",
"area": "0",
"platform": "2",
"platformName": "",
"stopName": "Bréhain-la-Cour Cabine téléphonique",
"nameWO": "Bréhain-la-Cour Cabine téléphonique",
"countdown": "347",
"dateTime": {
"year": "2024",
"month": "12",
"day": "20",
"weekday": "6",
"hour": "17",
"minute": "47"
},
"servingLine": {
"key": "2",
"code": "3",
"number": "54S130",
"symbol": "54S13",
"motType": "6",
"mtSubcode": "0",
"realtime": "0",
"direction": "Longwy",
"directionFrom": "Algrange LP Saint-Vincent-de-Paul",
"name": "Bus inter-urbain",
"liErgRiProj": {
"line": "5454S",
"project": "54r",
"direction": "R",
"supplement": "U",
"network": "als"
},
"destID": "71007583",
"stateless": "als:5454S:U:R:54r",
"lineDisplay": "line"
},
"operator": {
"code": "39",
"name": "FLUO 54",
"publicCode": ""
}
},
{
"stopID": "71007418",
"x": "5.8933884",
"y": "49.4457261",
"mapName": "WGS84[DD.ddddd]",
"area": "0",
"platform": "1",
"platformName": "",
"stopName": "Bréhain-la-Cour Cabine téléphonique",
"nameWO": "Bréhain-la-Cour Cabine téléphonique",
"countdown": "397",
"dateTime": {
"year": "2024",
"month": "12",
"day": "20",
"weekday": "6",
"hour": "18",
"minute": "37"
},
"servingLine": {
"key": "3",
"code": "3",
"number": "54S117",
"symbol": "54S11",
"motType": "6",
"mtSubcode": "0",
"realtime": "0",
"direction": "Mercy-le-Bas",
"directionFrom": "Longwy Lycée Albert Mézières",
"name": "Bus inter-urbain",
"liErgRiProj": {
"line": "5454S",
"project": "54r",
"direction": "R",
"supplement": "P",
"network": "als"
},
"destID": "71007546",
"stateless": "als:5454S:P:R:54r",
"lineDisplay": "line"
},
"operator": {
"code": "39",
"name": "FLUO 54",
"publicCode": ""
}
}
]
}

338
dataworker/input/xslt_dm_request/80021000.json Normal file
View file

@ -0,0 +1,338 @@
{
"parameters": [
{
"name": "serverID",
"value": "TRIAS-4"
},
{
"name": "requestID",
"value": "0"
},
{
"name": "sessionID",
"value": "0"
},
{
"name": "calcTime",
"value": "323.859"
},
{
"name": "serverTime",
"value": "2024-12-19T22:27:01"
},
{
"name": "logRequestId",
"value": "a1411e78dd98da5c"
}
],
"dm": {
"input": {
"input": "80021000"
},
"points": {
"point": {
"usage": "dm",
"type": "any",
"name": "Dolní Zleb, Dolni Zleb",
"stateless": "80021000",
"anyType": "stop",
"sort": "2",
"quality": "100000",
"best": "1",
"object": "Dolni Zleb",
"ref": {
"id": "80021000",
"gid": "cz:55421:21000:1",
"omc": "55421006",
"placeID": "2",
"place": "Dolní Zleb",
"coords": "14.2151207,50.8464312"
},
"infos": null
}
},
"itdOdvAssignedStops": {
"stopID": "80021000",
"name": "Dolni Zleb",
"x": "14.2151207",
"y": "50.8464312",
"mapName": "WGS84[DD.ddddd]",
"value": "80021000:Dolni Zleb",
"place": "Dolní Zleb",
"nameWithPlace": "Dolni Zleb",
"distanceTime": "0",
"isTransferStop": "0",
"vm": "100",
"gid": "cz:55421:21000:1"
}
},
"arr": {
"input": {
"input": ""
},
"points": null
},
"dateTime": {
"deparr": "dep",
"ttpFrom": "20241101",
"ttpTo": "20251213",
"year": "2024",
"month": "12",
"day": "20",
"hour": "12",
"minute": "0"
},
"dateRange": [
{
"day": "19",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "20",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "21",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "22",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "23",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "24",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "25",
"month": "12",
"year": "2024",
"weekday": "4"
},
{
"day": "26",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "27",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "28",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "29",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "30",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "31",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "01",
"month": "01",
"year": "2025",
"weekday": "4"
},
{
"day": "02",
"month": "01",
"year": "2025",
"weekday": "5"
}
],
"option": {
"ptOption": {
"active": "1",
"maxChanges": "9",
"maxTime": "360",
"maxWait": "120",
"routeType": "LEASTTIME",
"changeSpeed": "normal",
"lineRestriction": "403",
"useProxFootSearch": "0",
"useProxFootSearchOrigin": "0",
"useProxFootSearchDestination": "0",
"bike": "0",
"plane": "0",
"noCrowded": "0",
"noSolidStairs": "0",
"noEscalators": "0",
"noElevators": "0",
"lowPlatformVhcl": "0",
"wheelchair": "0",
"needElevatedPlt": "0",
"assistance": "0",
"SOSAvail": "0",
"noLonelyTransfer": "0",
"illumTransfer": "0",
"overgroundTransfer": "0",
"noInsecurePlaces": "0",
"privateTransport": "0",
"excludedMeans": [
{
"means": "Zug",
"value": "0",
"selected": "0"
},
{
"means": "S-Bahn",
"value": "1",
"selected": "0"
},
{
"means": "U-Bahn",
"value": "2",
"selected": "0"
},
{
"means": "Stadtbahn",
"value": "3",
"selected": "0"
},
{
"means": "Straßen-/Trambahn",
"value": "4",
"selected": "0"
},
{
"means": "Stadtbus",
"value": "5",
"selected": "0"
},
{
"means": "Regionalbus",
"value": "6",
"selected": "0"
},
{
"means": "Schnellbus",
"value": "7",
"selected": "0"
},
{
"means": "Seil-/Zahnradbahn",
"value": "8",
"selected": "0"
},
{
"means": "Schiff",
"value": "9",
"selected": "0"
},
{
"means": "AST (flächengebunde",
"value": "10",
"selected": "0"
},
{
"means": "Sonstige",
"value": "11",
"selected": "0"
},
{
"means": "Flugzeug",
"value": "12",
"selected": "0"
},
{
"means": "Zug (Nahverkehr)",
"value": "13",
"selected": "0"
},
{
"means": "Zug (Fernverkehr)",
"value": "14",
"selected": "0"
},
{
"means": "Zug Fernv m Zuschlag",
"value": "15",
"selected": "0"
},
{
"means": "Zug Fernv m spez Fpr",
"value": "16",
"selected": "0"
},
{
"means": "SEV Schienenersatzv",
"value": "17",
"selected": "0"
},
{
"means": "Zug Shuttle",
"value": "18",
"selected": "0"
},
{
"means": "Bürgerbus",
"value": "19",
"selected": "0"
}
],
"activeImp": "-1",
"activeCom": "-1",
"activeSec": "-1"
}
},
"message": [
{
"name": "code",
"value": "-4030"
},
{
"name": "error",
"value": "no matching departure found"
},
{
"name": "type",
"value": ""
},
{
"name": "module",
"value": "BROKER"
}
],
"servingLines": {
"trainInfo": "show",
"selected": "1",
"lines": [
null
]
},
"departureList": null
}

330
dataworker/input/xslt_dm_request/80040013.json Normal file
View file

@ -0,0 +1,330 @@
{
"parameters": [
{
"name": "serverID",
"value": "TRIAS-4"
},
{
"name": "requestID",
"value": "0"
},
{
"name": "sessionID",
"value": "0"
},
{
"name": "calcTime",
"value": "268.486"
},
{
"name": "serverTime",
"value": "2024-12-19T22:33:12"
},
{
"name": "logRequestId",
"value": "a1411e78dd98dcb4"
}
],
"dm": {
"input": {
"input": "80040013"
},
"points": {
"point": {
"usage": "dm",
"type": "any",
"name": "Paris-Montparnasse",
"stateless": "80040013",
"anyType": "stop",
"sort": "2",
"quality": "100000",
"best": "1",
"object": "Paris-Montparnasse",
"ref": {
"id": "80040013",
"gid": "fr:24075:40013",
"omc": "24075056",
"placeID": "1",
"place": ""
},
"infos": null
}
},
"itdOdvAssignedStops": {
"stopID": "80040013",
"name": "Paris-Montparnasse",
"value": "80040013:Paris-Montparnasse",
"place": "Paris",
"nameWithPlace": "Paris-Montparnasse",
"distanceTime": "0",
"isTransferStop": "0",
"vm": "100",
"gid": "fr:24075:40013"
}
},
"arr": {
"input": {
"input": ""
},
"points": null
},
"dateTime": {
"deparr": "dep",
"ttpFrom": "20241101",
"ttpTo": "20251213",
"year": "2024",
"month": "12",
"day": "20",
"hour": "12",
"minute": "0"
},
"dateRange": [
{
"day": "19",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "20",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "21",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "22",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "23",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "24",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "25",
"month": "12",
"year": "2024",
"weekday": "4"
},
{
"day": "26",
"month": "12",
"year": "2024",
"weekday": "5"
},
{
"day": "27",
"month": "12",
"year": "2024",
"weekday": "6"
},
{
"day": "28",
"month": "12",
"year": "2024",
"weekday": "7"
},
{
"day": "29",
"month": "12",
"year": "2024",
"weekday": "1"
},
{
"day": "30",
"month": "12",
"year": "2024",
"weekday": "2"
},
{
"day": "31",
"month": "12",
"year": "2024",
"weekday": "3"
},
{
"day": "01",
"month": "01",
"year": "2025",
"weekday": "4"
},
{
"day": "02",
"month": "01",
"year": "2025",
"weekday": "5"
}
],
"option": {
"ptOption": {
"active": "1",
"maxChanges": "9",
"maxTime": "360",
"maxWait": "120",
"routeType": "LEASTTIME",
"changeSpeed": "normal",
"lineRestriction": "403",
"useProxFootSearch": "0",
"useProxFootSearchOrigin": "0",
"useProxFootSearchDestination": "0",
"bike": "0",
"plane": "0",
"noCrowded": "0",
"noSolidStairs": "0",
"noEscalators": "0",
"noElevators": "0",
"lowPlatformVhcl": "0",
"wheelchair": "0",
"needElevatedPlt": "0",
"assistance": "0",
"SOSAvail": "0",
"noLonelyTransfer": "0",
"illumTransfer": "0",
"overgroundTransfer": "0",
"noInsecurePlaces": "0",
"privateTransport": "0",
"excludedMeans": [
{
"means": "Zug",
"value": "0",
"selected": "0"
},
{
"means": "S-Bahn",
"value": "1",
"selected": "0"
},
{
"means": "U-Bahn",
"value": "2",
"selected": "0"
},
{
"means": "Stadtbahn",
"value": "3",
"selected": "0"
},
{
"means": "Straßen-/Trambahn",
"value": "4",
"selected": "0"
},
{
"means": "Stadtbus",
"value": "5",
"selected": "0"
},
{
"means": "Regionalbus",
"value": "6",
"selected": "0"
},
{
"means": "Schnellbus",
"value": "7",
"selected": "0"
},
{
"means": "Seil-/Zahnradbahn",
"value": "8",
"selected": "0"
},
{
"means": "Schiff",
"value": "9",
"selected": "0"
},
{
"means": "AST (flächengebunde",
"value": "10",
"selected": "0"
},
{
"means": "Sonstige",
"value": "11",
"selected": "0"
},
{
"means": "Flugzeug",
"value": "12",
"selected": "0"
},
{
"means": "Zug (Nahverkehr)",
"value": "13",
"selected": "0"
},
{
"means": "Zug (Fernverkehr)",
"value": "14",
"selected": "0"
},
{
"means": "Zug Fernv m Zuschlag",
"value": "15",
"selected": "0"
},
{
"means": "Zug Fernv m spez Fpr",
"value": "16",
"selected": "0"
},
{
"means": "SEV Schienenersatzv",
"value": "17",
"selected": "0"
},
{
"means": "Zug Shuttle",
"value": "18",
"selected": "0"
},
{
"means": "Bürgerbus",
"value": "19",
"selected": "0"
}
],
"activeImp": "-1",
"activeCom": "-1",
"activeSec": "-1"
}
},
"message": [
{
"name": "code",
"value": "-4050"
},
{
"name": "error",
"value": "no serving lines found"
},
{
"name": "type",
"value": ""
},
{
"name": "module",
"value": "BROKER"
}
],
"servingLines": {
"lines": null
},
"departureList": null
}

4990
dataworker/input/xslt_dm_request/94300928.json Normal file
View file

File diff suppressed because it is too large Load diff

65
dataworker/src/dtos.rs
View file

@ -1,5 +1,6 @@
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
use std::collections::HashMap;
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
@ -10,7 +11,7 @@ pub struct Root {
pub date_time: DateTime, pub date_time: DateTime,
pub date_range: Vec<DateRange>, pub date_range: Vec<DateRange>,
pub serving_lines: ServingLines, pub serving_lines: ServingLines,
pub departure_list: Option<Vec<DepartureList>>, pub departure_list: Option<List<DepartureList>>,
pub arrival_list: Option<Vec<ArrivalList>>, pub arrival_list: Option<Vec<ArrivalList>>,
} }
@ -68,7 +69,7 @@ pub struct Ref {
#[serde(rename = "placeID")] #[serde(rename = "placeID")]
pub place_id: String, pub place_id: String,
pub place: String, pub place: String,
pub coords: String, pub coords: Option<String>,
} }
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
@ -116,9 +117,9 @@ pub struct ItdOdvAssignedStops {
#[serde(rename = "stopID")] #[serde(rename = "stopID")]
pub stop_id: String, pub stop_id: String,
pub name: String, pub name: String,
pub x: String, pub x: Option<String>,
pub y: String, pub y: Option<String>,
pub map_name: String, pub map_name: Option<String>,
pub value: String, pub value: String,
pub place: String, pub place: String,
pub name_with_place: String, pub name_with_place: String,
@ -207,10 +208,37 @@ pub struct ExcludedMean {
pub selected: String, pub selected: String,
} }
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Single<T> {
#[serde(flatten)]
pub inner: HashMap<String, T>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum List<T> {
None,
Single(Single<T>),
Multiple(Vec<T>),
}
impl<T> IntoIterator for List<T> {
type Item = T;
type IntoIter = std::vec::IntoIter<Self::Item>;
fn into_iter(self) -> Self::IntoIter {
match self {
Self::None => vec![].into_iter(),
Self::Single(line) => line.inner.into_values().collect::<Vec<_>>().into_iter(),
Self::Multiple(lines) => lines.into_iter(),
}
}
}
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct ServingLines { pub struct ServingLines {
pub lines: Vec<Line>, pub lines: Option<List<Option<Line>>>,
} }
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
@ -280,9 +308,9 @@ pub struct Attr {
pub struct DepartureList { pub struct DepartureList {
#[serde(rename = "stopID")] #[serde(rename = "stopID")]
pub stop_id: String, pub stop_id: String,
pub x: String, pub x: Option<String>,
pub y: String, pub y: Option<String>,
pub map_name: String, pub map_name: Option<String>,
pub area: String, pub area: String,
pub platform: String, pub platform: String,
pub platform_name: String, pub platform_name: String,
@ -292,12 +320,13 @@ pub struct DepartureList {
pub point_type: Option<String>, pub point_type: Option<String>,
pub countdown: String, pub countdown: String,
pub date_time: DateTime2, pub date_time: DateTime2,
pub real_date_time: Option<DateTime2>,
pub serving_line: ServingLine, pub serving_line: ServingLine,
pub operator: Option<Operator>, pub operator: Option<Operator>,
pub attrs: Option<Vec<Attr2>>, pub attrs: Option<Vec<Attr>>,
pub stop_infos: Option<StopInfos>, pub stop_infos: Option<StopInfos>,
pub line_infos: Value, pub line_infos: Option<Value>,
pub trip_infos: Value, pub trip_infos: Option<Value>,
} }
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
@ -307,7 +336,7 @@ pub struct ArrivalList {
pub stop_id: String, pub stop_id: String,
pub x: String, pub x: String,
pub y: String, pub y: String,
pub map_name: String, pub map_name: Option<String>,
pub area: String, pub area: String,
pub platform: String, pub platform: String,
pub platform_name: String, pub platform_name: String,
@ -319,7 +348,7 @@ pub struct ArrivalList {
pub real_date_time: Option<DateTime2>, pub real_date_time: Option<DateTime2>,
pub serving_line: ArrServingLine, pub serving_line: ArrServingLine,
pub operator: Option<Operator>, pub operator: Option<Operator>,
pub attrs: Option<Vec<Attr2>>, pub attrs: Option<Vec<Attr>>,
} }
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
@ -391,6 +420,7 @@ pub struct ServingLine {
pub train_num: Option<String>, pub train_num: Option<String>,
#[serde(default)] #[serde(default)]
pub hints: Vec<Hint>, pub hints: Vec<Hint>,
pub delay: Option<String>,
} }
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
@ -418,13 +448,6 @@ pub struct Operator {
pub public_code: String, pub public_code: String,
} }
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Attr2 {
pub name: String,
pub value: String,
}
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct StopInfos { pub struct StopInfos {

832
dataworker/src/lib.rs
View file

@ -1,75 +1,59 @@
#[macro_use] #[macro_use]
pub extern crate diesel; //pub extern crate diesel;
pub mod dtos; pub mod dtos;
pub use diesel::{OptionalExtension, dsl::insert_into, Connection, PgConnection, RunQueryDsl, SelectableHelper};
mod models; mod models;
mod schema; mod schema;
use chrono::NaiveDate; use crate::dtos::DateTime2;
use chrono::{Local, NaiveDateTime}; use crate::models::{DepartureRequest, Movement, ServingLine, StationHint, Stop};
use dtos::Root; use chrono::{Local, NaiveDate, NaiveDateTime};
use models::{ use sqlx::postgres::PgQueryResult;
Arrival, DepatureRequest, ServingLine, ServingLineDepatureRequest, StationHint, StopList, use sqlx::PgPool;
}; use std::option::Option;
use schema::arrivals::dsl::arrivals;
use schema::depature_requests::dsl::depature_requests;
use schema::serving_line_depature_requests::dsl::serving_line_depature_requests;
use schema::serving_lines::dsl::serving_lines;
use schema::station_hints::dsl::station_hints;
use schema::stop_lists::dsl::stop_lists;
use serde_json;
use std::{error::Error, str::FromStr}; use std::{error::Error, str::FromStr};
use uuid::Uuid; use uuid::Uuid;
use thiserror::Error;
#[derive(Debug, Error)] pub async fn insert_document(data: dtos::Root, conn: &mut PgPool) -> Result<(), Box<dyn Error>> {
pub enum WilldatenError { let _empty = vec![];
#[error("Missing arrivals in JSON!")] let id = Uuid::new_v5(
MissingArrivals, &Uuid::NAMESPACE_DNS,
} serde_json::to_string(&data.parameters)?.as_bytes(),
);
let req = if let Ok(req) = insert_req(
pub fn insert_document(data: Root, conn: &mut PgConnection) -> Result<(), Box<dyn Error>> { conn,
let Some(arrival_list) = data.arrival_list else { return Err(Box::new(WilldatenError::MissingArrivals)) }; DepartureRequest {
if let Some(req) = insert_into(depature_requests) id,
.values(DepatureRequest { stop_id: data.dm.itd_odv_assigned_stops.stop_id.parse().ok(),
depatureRequestId: Uuid::new_v5( server_id: data
&Uuid::NAMESPACE_DNS,
serde_json::to_string(&data.parameters)?.as_bytes(),
),
stopid: i32::from_str(&data.dm.itd_odv_assigned_stops.stop_id).ok(),
serverid: data
.parameters .parameters
.iter() .iter()
.filter(|p| p.name == "serverID") .filter(|p| p.name == "serverID")
.map(|p| p.clone().value) .map(|p| p.clone().value)
.next(), .next(),
requestid: data request_id: data
.parameters .parameters
.iter() .iter()
.filter(|p| p.name == "requestID") .filter(|p| p.name == "requestID")
.map(|p| p.clone().value) .map(|p| p.clone().value)
.next(), .next(),
sessionid: data session_id: data
.parameters .parameters
.iter() .iter()
.filter(|p| p.name == "sessionID") .filter(|p| p.name == "sessionID")
.map(|p| p.clone().value) .map(|p| p.clone().value)
.next(), .next(),
calcTime: Some( calc_time: f64::from_str(
f64::from_str( &data
&data .parameters
.parameters .iter()
.iter() .filter(|p| p.name == "calcTime")
.filter(|p| p.name == "calcTime") .map(|p| p.clone().value)
.map(|p| p.clone().value) .next()
.next() .unwrap(),
.unwrap(), )
) .ok(),
.unwrap(), server_time: data
),
serverTime: data
.parameters .parameters
.iter() .iter()
.filter(|p| p.name == "serverTime") .filter(|p| p.name == "serverTime")
@ -78,215 +62,575 @@ pub fn insert_document(data: Root, conn: &mut PgConnection) -> Result<(), Box<dy
.unwrap() .unwrap()
.parse::<NaiveDateTime>() .parse::<NaiveDateTime>()
.ok(), .ok(),
logRequestId: data log_request_id: data
.parameters .parameters
.iter() .iter()
.filter(|p| p.name == "logRequestId") .filter(|p| p.name == "logRequestId")
.map(|p| p.clone().value) .map(|p| p.clone().value)
.next(), .next(),
}) },
.returning(DepatureRequest::as_returning()) )
.on_conflict_do_nothing() .await
.get_result(conn)
.optional()
.unwrap()
{ {
let _ = insert_into(stop_lists) if let Some(req) = req {
.values(StopList { req
input: i32::from_str(&data.dm.input.input).unwrap(), } else {
point_dm: Some(data.dm.points.point.usage), id
point_type: Some(data.dm.points.point.type_field.clone()),
point_name: Some(data.dm.points.point.name),
point_stopId: i32::from_str(&data.dm.points.point.ref_field.id).ok(),
point_stateless: Some(data.dm.points.point.stateless),
point_anytype: Some(data.dm.points.point.any_type),
point_sort: i32::from_str(&data.dm.points.point.sort).ok(),
point_quality: i32::from_str(&data.dm.points.point.quality).ok(),
point_best: i32::from_str(&data.dm.points.point.best).ok(),
point_object: Some(data.dm.points.point.object),
point_ref_id: i32::from_str(&data.dm.points.point.ref_field.id).ok(),
point_ref_gid: Some(data.dm.points.point.ref_field.gid),
point_ref_omc: i32::from_str(&data.dm.points.point.ref_field.omc).ok(),
point_ref_placeID: i32::from_str(&data.dm.points.point.ref_field.place_id).ok(),
point_ref_place: Some(data.dm.points.point.ref_field.place),
point_ref_coords: Some(data.dm.points.point.ref_field.coords),
itdOdvAssignedStops_stopid: i32::from_str(&data.dm.itd_odv_assigned_stops.stop_id)
.ok(),
itdOdvAssignedStops_name: Some(data.dm.itd_odv_assigned_stops.name),
itdOdvAssignedStops_x: f64::from_str(&data.dm.itd_odv_assigned_stops.x).ok(),
itdOdvAssignedStops_y: f64::from_str(&data.dm.itd_odv_assigned_stops.y).ok(),
itdOdvAssignedStops_mapName: Some(data.dm.itd_odv_assigned_stops.map_name),
itdOdvAssignedStops_value: Some(data.dm.itd_odv_assigned_stops.value),
itdOdvAssignedStops_place: Some(data.dm.itd_odv_assigned_stops.place),
itdOdvAssignedStops_nameWithPlace: Some(
data.dm.itd_odv_assigned_stops.name_with_place,
),
itdOdvAssignedStops_distanceTime: i32::from_str(
&data.dm.itd_odv_assigned_stops.distance_time,
)
.ok(),
itdOdvAssignedStops_isTransferStop: i32::from_str(
&data.dm.itd_odv_assigned_stops.is_transfer_stop,
)
.ok(),
itdOdvAssignedStops_vm: i32::from_str(&data.dm.itd_odv_assigned_stops.vm).ok(),
itdOdvAssignedStops_gid: Some(data.dm.itd_odv_assigned_stops.gid),
meta_lastModificationDate: Some(Local::now().naive_local()),
})
.returning(StopList::as_returning())
.on_conflict_do_nothing()
.get_result(conn)
.optional();
for line in data.serving_lines.lines {
let id = Uuid::new_v5(
&Uuid::NAMESPACE_DNS,
serde_json::to_string(&line)?.as_bytes(),
);
let _ = insert_into(serving_lines)
.values(ServingLine {
servingLineId: id.clone(),
mode_name: Some(line.mode.name),
mode_number: Some(line.mode.number),
mode_product: Some(line.mode.product),
mode_productId: i32::from_str(&line.mode.product_id).ok(),
mode_type: i32::from_str(&line.mode.type_field).ok(),
mode_code: i32::from_str(&line.mode.code).ok(),
mode_destination: Some(line.mode.destination),
mode_destID: i32::from_str(&line.mode.dest_id).ok(),
mode_desc: Some(line.mode.desc),
mode_timetablePeriod: Some(line.mode.timetable_period),
diva_branch: Some(line.mode.diva.branch),
diva_line: Some(line.mode.diva.line),
diva_supplement: Some(line.mode.diva.supplement),
diva_dir: Some(line.mode.diva.dir),
diva_project: Some(line.mode.diva.project),
diva_network: Some(line.mode.diva.network),
diva_stateless: Some(line.mode.diva.stateless),
diva_tripCode: i32::from_str(&line.mode.diva.trip_code).ok(),
diva_operator: Some(line.mode.diva.operator),
diva_opPublicCode: line.mode.diva.op_public_code,
diva_opCode: Some(line.mode.diva.op_code),
diva_Vf: NaiveDate::parse_from_str(&line.mode.diva.v_f, "%Y%m%d").ok(),
diva_vTo: NaiveDate::parse_from_str(&line.mode.diva.v_to, "%Y%m%d").ok(),
diva_lineDisplay: Some(line.mode.diva.line_display),
meta_lastModificationDate: req.serverTime,
})
.returning(ServingLine::as_returning())
.on_conflict_do_nothing()
.get_result(conn)
.optional()
.unwrap();
let _ = insert_into(serving_line_depature_requests)
.values(ServingLineDepatureRequest {
depatureRequestId: req.depatureRequestId,
servingLineId: id.clone(),
})
.returning(ServingLineDepatureRequest::as_returning())
.on_conflict_do_nothing()
.get_result(conn)
.optional()
.unwrap();
} }
if let Some(infos) = data.dm.points.point.infos { } else {
let _ = insert_into(station_hints).values(StationHint { id
infoLinkURL: infos.info.info_link_url, };
infoLinkText: Some(infos.info.info_link_text),
infoText: Some(infos.info.info_text.content), insert_stop(
}); conn,
} &Stop {
{ id: data.dm.input.input.parse().ok(),
let _ = insert_into(arrivals) usage: data.dm.points.point.usage,
.values( r#type: data.dm.points.point.type_field.clone(),
&arrival_list name: data.dm.points.point.name,
.into_iter() anytype: data.dm.points.point.any_type,
.map(|arrival| Arrival { sort: data.dm.points.point.sort.parse().ok(),
depatureRequestId: req.depatureRequestId, quality: data.dm.points.point.quality.parse().ok(),
stopID: i32::from_str(&arrival.stop_id).ok(), best: data.dm.points.point.best.parse().ok(),
x: f64::from_str(&arrival.x).ok(), object: data.dm.points.point.object,
y: f64::from_str(&arrival.y).ok(), gid: data.dm.points.point.ref_field.gid,
mapName: Some(arrival.map_name), omc: data.dm.points.point.ref_field.omc.parse().ok(),
area: Some(arrival.area), place_id: data.dm.points.point.ref_field.place_id.parse().ok(),
platform: Some(arrival.platform), x: if let Some(x) = data.dm.itd_odv_assigned_stops.x {
platformName: Some(arrival.platform_name), x.parse().ok()
stopName: Some(arrival.stop_name), } else {
nameWO: Some(arrival.name_wo), None
pointType: Some(data.dm.points.point.type_field.clone()), },
countdown: i32::from_str(&arrival.countdown).ok(), y: if let Some(y) = data.dm.itd_odv_assigned_stops.y {
arrivalTime: NaiveDate::from_ymd_opt( y.parse().ok()
i32::from_str(&arrival.date_time.year).unwrap(), } else {
u32::from_str(&arrival.date_time.month).unwrap(), None
u32::from_str(&arrival.date_time.day).unwrap(), },
) map_name: data.dm.itd_odv_assigned_stops.map_name,
.unwrap() place: data.dm.itd_odv_assigned_stops.place,
.and_hms_opt( name_with_place: data.dm.itd_odv_assigned_stops.name_with_place,
u32::from_str(&arrival.date_time.hour).unwrap(), distance_time: data.dm.itd_odv_assigned_stops.distance_time.parse().ok(),
u32::from_str(&arrival.date_time.minute).unwrap(), is_transfer_stop: data.dm.itd_odv_assigned_stops.is_transfer_stop.parse().ok(),
0, vm: data.dm.itd_odv_assigned_stops.vm.parse().ok(),
last_modified: Local::now().naive_local(),
},
)
.await
.unwrap();
let _ = insert_line(
conn,
if let Some(lines) = &data.serving_lines.lines.map(|line| {
line.into_iter()
.filter_map(|line| {
if let Some(line) = line {
Some(ServingLine {
id: Uuid::new_v5(
&Uuid::NAMESPACE_DNS,
serde_json::to_string(&line).unwrap().as_bytes(),
), ),
realArrivalTime: if let Some(date_time) = arrival.real_date_time { name: line.mode.name,
NaiveDate::from_ymd_opt( number: line.mode.number.parse().ok(),
i32::from_str(&date_time.year).unwrap(), product: line.mode.product,
u32::from_str(&date_time.month).unwrap(), product_id: line.mode.product_id.parse().ok(),
u32::from_str(&date_time.day).unwrap(), r#type: line.mode.type_field.parse().ok(),
) code: line.mode.code.parse().ok(),
destination: line.mode.destination,
dest_id: line.mode.dest_id.parse().ok(),
desc: line.mode.desc,
timetable_period: line.mode.timetable_period,
branch: line.mode.diva.branch,
stateless: line.mode.diva.stateless,
trip_code: line.mode.diva.trip_code.parse().ok(),
operator: line.mode.diva.operator,
op_public_code: line.mode.diva.op_public_code,
op_code: line.mode.diva.op_code,
v_f: NaiveDate::parse_from_str(&line.mode.diva.v_f, "%Y%m%d").ok(),
v_to: NaiveDate::parse_from_str(&line.mode.diva.v_to, "%Y%m%d").ok(),
line_display: line.mode.diva.line_display,
last_modified: if let Ok(date) = data
.parameters
.iter()
.filter(|p| p.name == "serverTime")
.map(|p| p.clone().value)
.next()
.unwrap() .unwrap()
.and_hms_opt( .parse::<NaiveDateTime>()
u32::from_str(&date_time.hour).unwrap(), {
u32::from_str(&date_time.minute).unwrap(), date
0,
)
} else { } else {
None Local::now().naive_local()
}, },
servingLine_key: Some(arrival.serving_line.key),
servingLine_code: Some(arrival.serving_line.code),
servingLine_number: Some(arrival.serving_line.number),
servingLine_symbol: Some(arrival.serving_line.symbol),
servingLine_motType: i32::from_str(&arrival.serving_line.mot_type).ok(),
servingLine_mtSubcode: i32::from_str(&arrival.serving_line.mt_subcode)
.ok(),
servingLine_realtime: bool::from_str(&arrival.serving_line.realtime)
.ok(),
servingLine_direction: Some(arrival.serving_line.direction),
servingLine_directionFrom: Some(arrival.serving_line.direction_from),
servingLine_name: Some(arrival.serving_line.name),
servingLine_delay: if let Some(delay) = arrival.serving_line.delay {
i32::from_str(&delay).ok()
} else {
None
},
servingLine_liErgRiProj_line: Some(
arrival.serving_line.li_erg_ri_proj.line,
),
servingLine_liErgRiProj_project: Some(
arrival.serving_line.li_erg_ri_proj.project,
),
servingLine_liErgRiProj_direction: Some(
arrival.serving_line.li_erg_ri_proj.direction,
),
servingLine_liErgRiProj_supplement: Some(
arrival.serving_line.li_erg_ri_proj.supplement,
),
servingLine_liErgRiProj_network: Some(
arrival.serving_line.li_erg_ri_proj.network,
),
servingLine_destID: Some(arrival.serving_line.dest_id),
servingLine_stateless: Some(arrival.serving_line.stateless),
servingLine_lineDisplay: Some(arrival.serving_line.line_display),
operator_code: arrival.operator.clone().map(|op| op.code),
operator_name: arrival.operator.clone().map(|op| op.name),
operator_publicCode: arrival.operator.map(|op| op.public_code),
attrs_name: None,
attrs_value: None,
}) })
.collect::<Vec<_>>(), } else {
) None
.returning(Arrival::as_returning()) }
.on_conflict_do_nothing() })
.get_results(conn) .collect::<Vec<_>>()
.optional(); }) {
} lines
} else {
&_empty
},
)
.await
.unwrap();
if let Some(infos) = data.dm.points.point.infos {
insert_hints(
conn,
&StationHint {
info_link_url: infos.info.info_link_url,
info_link_text: infos.info.info_link_text,
info_text: infos.info.info_text.content,
},
)
.await
.unwrap();
};
if let Some(arrival_list) = data.arrival_list {
let _ = insert_arrivals(
conn,
&arrival_list
.into_iter()
.map(|arrival| Movement {
departure_request_id: req,
stop_id: arrival.stop_id.parse().ok(),
x: arrival.x.parse().ok(),
y: arrival.y.parse().ok(),
map_name: arrival.map_name,
area: arrival.area,
platform: arrival.platform,
platform_name: arrival.platform_name,
countdown: i32::from_str(&arrival.countdown).ok(),
time: datetime_from_dto(&arrival.date_time),
real_time: if let Some(date_time) = arrival.real_date_time {
datetime_from_dto(&date_time)
} else {
None
},
key: i32::from_str(&arrival.serving_line.key)
.ok()
.unwrap_or_default(),
code: arrival.serving_line.code,
number: arrival.serving_line.number,
symbol: arrival.serving_line.symbol,
mot_type: arrival.serving_line.mot_type.parse().ok(),
mt_subcode: arrival.serving_line.mt_subcode.parse().ok(),
realtime: arrival.serving_line.realtime.parse().ok(),
direction: arrival.serving_line.direction,
direction_from: arrival.serving_line.direction_from,
name: arrival.serving_line.name,
delay: if let Some(delay) = arrival.serving_line.delay {
i32::from_str(&delay).ok()
} else {
None
},
dest_id: arrival.serving_line.dest_id,
stateless: arrival.serving_line.stateless,
line_display: arrival.serving_line.line_display,
operator_code: arrival.operator.clone().map(|op| op.code),
operator_name: arrival.operator.clone().map(|op| op.name),
operator_public_code: arrival.operator.map(|op| op.public_code),
attrs: if let Some(attrs) = arrival.attrs {
attrs
} else {
vec![]
},
r#type: false,
})
.collect::<Vec<_>>(),
)
.await
.unwrap();
} }
if let Some(arrival_list) = data.departure_list {
let _ = insert_arrivals(
conn,
&arrival_list
.into_iter()
.map(|arrival| Movement {
departure_request_id: req,
stop_id: i32::from_str(&arrival.stop_id).ok(),
x: if let Some(x) = arrival.x {
x.parse().ok()
} else {
None
},
y: if let Some(y) = arrival.y {
y.parse().ok()
} else {
None
},
map_name: arrival.map_name,
area: arrival.area,
platform: arrival.platform,
platform_name: arrival.platform_name,
countdown: i32::from_str(&arrival.countdown).ok(),
time: datetime_from_dto(&arrival.date_time),
real_time: if let Some(date_time) = arrival.real_date_time {
datetime_from_dto(&date_time)
} else {
None
},
key: i32::from_str(&arrival.serving_line.key)
.ok()
.unwrap_or_default(),
code: arrival.serving_line.code,
number: arrival.serving_line.number,
symbol: arrival.serving_line.symbol,
mot_type: arrival.serving_line.mot_type.parse().ok(),
mt_subcode: arrival.serving_line.mt_subcode.parse().ok(),
realtime: arrival.serving_line.realtime.parse().ok(),
direction: arrival.serving_line.direction,
direction_from: arrival.serving_line.direction_from,
name: arrival.serving_line.name,
delay: if let Some(delay) = arrival.serving_line.delay {
i32::from_str(&delay).ok()
} else {
None
},
dest_id: arrival.serving_line.dest_id,
stateless: arrival.serving_line.stateless,
line_display: arrival.serving_line.line_display,
operator_code: arrival.operator.clone().map(|op| op.code),
operator_name: arrival.operator.clone().map(|op| op.name),
operator_public_code: arrival.operator.map(|op| op.public_code),
attrs: if let Some(attrs) = arrival.attrs {
attrs
} else {
vec![]
},
r#type: true,
})
.collect::<Vec<_>>(),
)
.await
.unwrap();
}
Ok(()) Ok(())
} }
/*
async fn setup(conn: &PgPool) -> Result<(), sqlx::Error> {
let sql = include_str!("../scheme.sql");
sqlx::query(sql).execute(conn).await?;
Ok(())
}*/
async fn insert_req(conn: &PgPool, req: DepartureRequest) -> Result<Option<Uuid>, sqlx::Error> {
sqlx::query_scalar!(
"INSERT INTO public.departure_requests VALUES ($1, $2, $3, $4, $5, $6, $7, $8) \
ON CONFLICT (id) DO NOTHING \
RETURNING id",
req.id,
req.stop_id,
req.server_id,
req.request_id,
req.session_id,
req.calc_time,
req.server_time,
req.log_request_id
)
.fetch_optional(conn)
.await
}
async fn insert_stop(conn: &PgPool, stop: &Stop) -> Result<Vec<i32>, sqlx::Error> {
sqlx::query_scalar!(
"INSERT INTO stops VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14,
$15, $16, $17, $18, $19, $20, $21)
ON CONFLICT (id) DO NOTHING
RETURNING id",
stop.id,
&stop.usage,
stop.r#type,
stop.name,
stop.anytype,
stop.sort,
stop.quality,
stop.best,
stop.object,
stop.gid,
stop.omc,
stop.place_id,
stop.x,
stop.y,
stop.map_name,
stop.place,
stop.name_with_place,
stop.distance_time,
stop.is_transfer_stop,
stop.vm,
stop.last_modified
)
.fetch_all(conn)
.await
}
async fn insert_line(conn: &PgPool, line: &Vec<ServingLine>) -> Result<Vec<Uuid>, sqlx::Error> {
sqlx::query_scalar!(
"INSERT INTO serving_lines SELECT * FROM
UNNEST($1::uuid[], $2::varchar[], $3::integer[], $4::varchar[], $5::integer[],
$6::integer[], $7::integer[], $8::varchar[], $9::integer[], $10::text[], $11::varchar[],
$12::varchar[], $13::varchar[], $14::integer[], $15::varchar[], $16::varchar[],
$17::varchar[], $18::date[], $19::date[], $20::varchar[], $21::timestamp[])
ON CONFLICT (id) DO NOTHING
RETURNING id",
&line.iter().map(|sv| sv.id).collect::<Vec<_>>(),
&line.iter().map(|sv| sv.name.clone()).collect::<Vec<_>>(),
&line.iter().map(|sv| sv.number).collect::<Vec<_>>() as &[Option<_>],
&line.iter().map(|sv| sv.product.clone()).collect::<Vec<_>>(),
&line.iter().map(|sv| sv.product_id).collect::<Vec<_>>() as &[Option<_>],
&line.iter().map(|sv| sv.r#type).collect::<Vec<_>>() as &[Option<_>],
&line.iter().map(|sv| sv.code).collect::<Vec<_>>() as &[Option<_>],
&line
.iter()
.map(|sv| sv.destination.clone())
.collect::<Vec<_>>(),
&line.iter().map(|sv| sv.dest_id).collect::<Vec<_>>() as &[Option<_>],
&line.iter().map(|sv| sv.desc.clone()).collect::<Vec<_>>(),
&line
.iter()
.map(|sv| sv.timetable_period.clone())
.collect::<Vec<_>>(),
&line.iter().map(|sv| sv.branch.clone()).collect::<Vec<_>>(),
&line
.iter()
.map(|sv| sv.stateless.clone())
.collect::<Vec<_>>(),
&line.iter().map(|sv| sv.trip_code).collect::<Vec<_>>() as &[Option<_>],
&line
.iter()
.map(|sv| sv.operator.clone())
.collect::<Vec<_>>(),
&line
.iter()
.map(|sv| sv.op_public_code.clone())
.collect::<Vec<_>>() as &[Option<_>],
&line.iter().map(|sv| sv.op_code.clone()).collect::<Vec<_>>(),
&line.iter().map(|sv| sv.v_f).collect::<Vec<_>>() as &[Option<_>],
&line.iter().map(|sv| sv.v_to).collect::<Vec<_>>() as &[Option<_>],
&line
.iter()
.map(|sv| sv.line_display.clone())
.collect::<Vec<_>>(),
&line.iter().map(|sv| sv.last_modified).collect::<Vec<_>>(),
)
.fetch_all(conn)
.await
}
/*
async fn insert_req_line(
conn: &PgPool,
req: &Vec<ServingLineDepatureRequest>,
) -> Result<Vec<i32>, sqlx::Error> {
sqlx::query_scalar!(
"INSERT INTO serving_line_departure_requests (departureRequestId, servingLineId) SELECT * FROM \
UNNEST($1::uuid[], $2::uuid[]) ON CONFLICT (id) DO NOTHING returning id",
&req.iter().map(|r| r.departure_request_id).collect::<Vec<_>>(),
&req.iter().map(|r| r.id).collect::<Vec<_>>()).fetch_all(conn).await
}
*/
async fn insert_hints(conn: &PgPool, hints: &StationHint) -> Result<PgQueryResult, sqlx::Error> {
sqlx::query!(
"INSERT INTO station_hints VALUES($1, $2, $3)
ON CONFLICT (infoLinkURL) DO NOTHING",
hints.info_link_url,
hints.info_link_text,
hints.info_text
)
.execute(conn)
.await
}
async fn insert_arrivals(
conn: &PgPool,
arrivals: &Vec<Movement>,
) -> Result<PgQueryResult, sqlx::Error> {
let _ = sqlx::query!("
WITH names_insert AS(
INSERT INTO attr_names (name)
SELECT * FROM UNNEST($1::varchar[])
ON CONFLICT (name) DO NOTHING
RETURNING id
),
values_insert AS(
INSERT INTO attr_vals (value)
SELECT * FROM UNNEST($2::varchar[])
ON CONFLICT ON CONSTRAINT attr_vals_uniq DO NOTHING
RETURNING id
),
names AS(
SELECT id FROM names_insert
UNION
SELECT id FROM attr_vals WHERE (value) IN (SELECT * FROM UNNEST($1::varchar[]))
),
values AS(
SELECT id FROM values_insert
UNION
SELECT id FROM attr_names WHERE (name) IN (SELECT * FROM UNNEST($2::varchar[]))
)
INSERT INTO movement_attrs (departure_request, stateless, key, name_id, value_id)
SELECT * FROM UNNEST($3::uuid[], $4::varchar[], $5::integer[], array(select id from names)::integer[], array(select id from values)::integer[]) as t(departure_request, stateless, key, name_id, value_id)
WHERE name_id IS NOT NULL AND value_id IS NOT NULL
ON CONFLICT ON CONSTRAINT mov_attr_pkey DO NOTHING
",
&arrivals
.iter()
.map(|a| a.attrs.iter().map(|a| a.value.clone()).collect::<Vec<_>>())
.flatten()
.collect::<Vec<_>>(),
&arrivals
.iter()
.map(|a| a.attrs.iter().map(|a| a.name.clone()).collect::<Vec<_>>())
.flatten()
.collect::<Vec<_>>(),
&arrivals
.iter()
.map(|a| a
.attrs
.iter()
.map(|_| a.departure_request_id.clone())
.collect::<Vec<_>>())
.flatten()
.collect::<Vec<_>>(),
&arrivals
.iter()
.map(|a| a
.attrs
.iter()
.map(|_| a.stateless.clone())
.collect::<Vec<_>>())
.flatten()
.collect::<Vec<_>>(),
&arrivals
.iter()
.map(|a| a.attrs.iter().map(|_| a.key).collect::<Vec<_>>())
.flatten()
.collect::<Vec<_>>(),
)
.fetch_all(conn)
.await?;
let operators = sqlx::query_scalar!(
"WITH result AS(
INSERT INTO operators (code, name, pub_code)
SELECT * FROM UNNEST($1::varchar[], $2::varchar[], $3::varchar[])
ON CONFLICT (code) DO NOTHING
RETURNING id
)
SELECT id FROM result
UNION
SELECT id FROM operators WHERE code = ANY($1);",
&arrivals
.iter()
.map(|a| a.operator_code.clone())
.collect::<Vec<_>>() as &[Option<_>],
&arrivals
.iter()
.map(|a| a.operator_name.clone())
.collect::<Vec<_>>() as &[Option<_>],
&arrivals
.iter()
.map(|a| a.operator_public_code.clone())
.collect::<Vec<_>>() as &[Option<_>]
)
.fetch_all(conn)
.await?;
sqlx::query!(
"INSERT INTO movements (
departure_request, stopID, x, y, mapName, area, platform, platformName,
countdown, time, realtime, key, code,
number, symbol, motType, mtSubcode, is_realtime, direction, directionFrom,
name, delay, destID, stateless, lineDisplay, operator_id, type
)
SELECT * FROM UNNEST(
$1::uuid[], $2::integer[], $3::double precision[], $4::double precision[],
$5::varchar[], $6::varchar[], $7::varchar[], $8::varchar[], $9::integer[],
$10::timestamp[], $11::timestamp[], $12::integer[], $13::varchar[], $14::varchar[],
$15::varchar[], $16::integer[], $17::integer[], $18::boolean[],
$19::varchar[], $20::varchar[], $21::varchar[], $22::integer[],
$23::varchar[], $24::varchar[], $25::varchar[], $26::integer[], $27::boolean[]
) ON CONFLICT ON CONSTRAINT movements_uniq DO NOTHING",
&arrivals
.iter()
.map(|r| r.departure_request_id)
.collect::<Vec<_>>(),
&arrivals.iter().map(|r| r.stop_id).collect::<Vec<_>>() as &[Option<_>],
&arrivals.iter().map(|r| r.x).collect::<Vec<_>>() as &[Option<_>],
&arrivals.iter().map(|r| r.y).collect::<Vec<_>>() as &[Option<_>],
&arrivals
.iter()
.map(|r| r.map_name.clone())
.collect::<Vec<_>>() as &[Option<_>],
&arrivals.iter().map(|r| r.area.clone()).collect::<Vec<_>>(),
&arrivals
.iter()
.map(|r| r.platform.clone())
.collect::<Vec<_>>(),
&arrivals
.iter()
.map(|r| r.platform_name.clone())
.collect::<Vec<_>>(),
&arrivals.iter().map(|r| r.countdown).collect::<Vec<_>>() as &[Option<_>],
&arrivals.iter().map(|r| r.time).collect::<Vec<_>>() as &[Option<_>],
&arrivals.iter().map(|r| r.real_time).collect::<Vec<_>>() as &[Option<_>],
&arrivals.iter().map(|r| r.key.clone()).collect::<Vec<_>>(),
&arrivals.iter().map(|r| r.code.clone()).collect::<Vec<_>>(),
&arrivals
.iter()
.map(|r| r.number.clone())
.collect::<Vec<_>>(),
&arrivals
.iter()
.map(|r| r.symbol.clone())
.collect::<Vec<_>>(),
&arrivals.iter().map(|r| r.mot_type).collect::<Vec<_>>() as &[Option<_>],
&arrivals.iter().map(|r| r.mt_subcode).collect::<Vec<_>>() as &[Option<_>],
&arrivals.iter().map(|r| r.realtime).collect::<Vec<_>>() as &[Option<_>],
&arrivals
.iter()
.map(|r| r.direction.clone())
.collect::<Vec<_>>(),
&arrivals
.iter()
.map(|r| r.direction_from.clone())
.collect::<Vec<_>>(),
&arrivals.iter().map(|r| r.name.clone()).collect::<Vec<_>>(),
&arrivals.iter().map(|r| r.delay).collect::<Vec<_>>() as &[Option<_>],
&arrivals
.iter()
.map(|r| r.dest_id.clone())
.collect::<Vec<_>>(),
&arrivals
.iter()
.map(|r| r.stateless.clone())
.collect::<Vec<_>>(),
&arrivals
.iter()
.map(|r| r.line_display.clone())
.collect::<Vec<_>>(),
&operators as &[Option<_>],
&arrivals.iter().map(|r| r.r#type).collect::<Vec<_>>(),
)
.execute(conn)
.await
}
fn datetime_from_dto(dto: &DateTime2) -> Option<NaiveDateTime> {
NaiveDate::from_ymd_opt(
i32::from_str(&dto.year).unwrap(),
u32::from_str(&dto.month).unwrap(),
u32::from_str(&dto.day).unwrap(),
)?
.and_hms_opt(
u32::from_str(&dto.hour).unwrap(),
u32::from_str(&dto.minute).unwrap(),
0,
)
}

37
dataworker/src/main.rs
View file

@ -1,28 +1,32 @@
#[macro_use]
extern crate diesel;
use dataworker::dtos::Root; use dataworker::dtos::Root;
use dataworker::insert_document; use dataworker::insert_document;
use diesel::{Connection, PgConnection};
use dotenvy::dotenv; use dotenvy::dotenv;
use futures_util::StreamExt;
use sqlx::postgres::PgPoolOptions;
use std::env;
use std::error::Error; use std::error::Error;
use std::io::Write; use std::io::Write;
use std::{env, fs}; use tokio::fs;
use tokio::fs::DirEntry;
use tokio_stream::wrappers::ReadDirStream;
#[tokio::main] #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> { async fn main() -> Result<(), Box<dyn Error>> {
dotenv().ok(); dotenv().ok();
let url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); let url = env::var("DATABASE_URL").expect("DATABASE_URL must be set");
let mut conn = let mut conn = PgPoolOptions::new()
PgConnection::establish(&url).unwrap_or_else(|e| panic!("Error connecting to {}: {}", url, e)); .max_connections(5)
.connect(&url)
.await?;
let src = "/home/bread/Downloads/2025-01-22(1)/2025-01-22"; let src = "input/xslt_dm_request";
let total = fs::read_dir(src).unwrap().count(); let total = ReadDirStream::new(fs::read_dir(src).await?).count().await;
let mut stream = ReadDirStream::new(fs::read_dir(src).await?);
let mut count = 0; let mut count = 0;
fs::read_dir(src).unwrap().into_iter().for_each(|path| { while let Some(path) = stream.next().await {
count += 1; count += 1;
let path = path.unwrap(); let path: DirEntry = path.unwrap();
print!( print!(
"\x1B[2K\r {}/{} ({})", "\x1B[2K\r {}/{} ({})",
count, count,
@ -30,10 +34,13 @@ async fn main() -> Result<(), Box<dyn Error>> {
path.file_name().to_str().unwrap() path.file_name().to_str().unwrap()
); );
std::io::stdout().flush().unwrap(); std::io::stdout().flush().unwrap();
let content = fs::read_to_string(path.path()).unwrap(); let content = fs::read_to_string(path.path()).await.unwrap();
let data = serde_json::from_str::<Root>(&content).unwrap(); if content.len() > 0 {
insert_document(data, &mut conn).unwrap(); if let Ok(data) = serde_json::from_str::<Root>(&content) {
}); insert_document(data, &mut conn).await?;
}
}
}
println!(); println!();
Ok(()) Ok(())

333
dataworker/src/models.rs
View file

@ -1,177 +1,222 @@
// Generated by diesel_ext // Generated by diesel_ext
#![allow(unused)] use crate::dtos::Attr;
#![allow(clippy::all)]
#![allow(non_snake_case)]
use crate::schema::*;
use chrono::NaiveDate; use chrono::NaiveDate;
use chrono::NaiveDateTime; use chrono::NaiveDateTime;
use diesel::pg::Pg;
use uuid::Uuid; use uuid::Uuid;
#[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)] //#[derive(Queryable, Selectable, Insertable, Debug, _identifiable, QueryableByName)]
#[diesel(table_name = arrivals)] //#[diesel(table_name = arrivals)]
#[diesel(check_for_backend(Pg))] //#[diesel(check_for_backend(Pg))]
#[diesel(primary_key(stopID, servingLine_key, servingLine_stateless))] //#[diesel(primary_key(stop_id, key, stateless))]
/*
pub struct Arrival { pub struct Arrival {
pub depatureRequestId: Uuid, pub departure_request_id: Uuid,
pub stopID: Option<i32>, pub stop_id: Option<i32>,
pub x: Option<f64>, pub x: Option<f64>,
pub y: Option<f64>, pub y: Option<f64>,
pub mapName: Option<String>, pub map_mame: String,
pub area: Option<String>, pub area: String,
pub platform: Option<String>, pub platform: String,
pub platformName: Option<String>, pub platform_name: String,
pub stopName: Option<String>, pub stop_name: String,
pub nameWO: Option<String>, pub name_wo: String,
pub pointType: Option<String>, pub point_type: String,
pub countdown: Option<i32>, pub countdown: Option<i32>,
pub arrivalTime: Option<NaiveDateTime>, pub arrival_time: NaiveDateTime,
pub realArrivalTime: Option<NaiveDateTime>, pub real_arrival_time: NaiveDateTime,
pub servingLine_key: Option<String>, pub key: String,
pub servingLine_code: Option<String>, pub code: String,
pub servingLine_number: Option<String>, pub number: String,
pub servingLine_symbol: Option<String>, pub symbol: String,
pub servingLine_motType: Option<i32>, pub mot_type: Option<i32>,
pub servingLine_mtSubcode: Option<i32>, pub mt_subcode: Option<i32>,
pub servingLine_realtime: Option<bool>, pub realtime: Option<bool>,
pub servingLine_direction: Option<String>, pub direction: String,
pub servingLine_directionFrom: Option<String>, pub direction_from: String,
pub servingLine_name: Option<String>, pub name: String,
pub servingLine_delay: Option<i32>, pub delay: Option<i32>,
pub servingLine_liErgRiProj_line: Option<String>, /*
pub servingLine_liErgRiProj_project: Option<String>, pub liErgRiProj_line: String,
pub servingLine_liErgRiProj_direction: Option<String>, pub liErgRiProj_project: String,
pub servingLine_liErgRiProj_supplement: Option<String>, pub liErgRiProj_direction: String,
pub servingLine_liErgRiProj_network: Option<String>, pub liErgRiProj_supplement: String,
pub servingLine_destID: Option<String>, pub liErgRiProj_network: String,
pub servingLine_stateless: Option<String>, */
pub servingLine_lineDisplay: Option<String>, pub dest_id: String,
pub stateless: String,
pub line_display: String,
pub operator_code: String,
pub operator_name: String,
pub operator_public_code: String,
pub attrs_name: String,
pub attrs_value: String,
}
*/
//#[derive(Queryable, Selectable, Insertable, Debug, _identifiable, QueryableByName)]
//#[diesel(table_name = departures)]
//#[diesel(check_for_backend(Pg))]
//#[diesel(primary_key(stop_id, key, stateless))]
pub struct Movement {
pub departure_request_id: Uuid,
pub stop_id: Option<i32>,
pub x: Option<f64>,
pub y: Option<f64>,
pub map_name: Option<String>,
pub area: String,
pub platform: String,
pub platform_name: String,
/*
pub stop_name: String,
pub name_wo: String,
pub point_type: String,
*/
pub countdown: Option<i32>,
pub time: Option<NaiveDateTime>,
pub real_time: Option<NaiveDateTime>,
pub key: i32,
pub code: String,
pub number: String,
pub symbol: String,
pub mot_type: Option<i32>,
pub mt_subcode: Option<i32>,
pub realtime: Option<bool>,
pub direction: String,
pub direction_from: String,
pub name: String,
pub delay: Option<i32>,
/*
pub liErgRiProj_line: String,
pub liErgRiProj_project: String,
pub liErgRiProj_direction: String,
pub liErgRiProj_supplement: String,
pub liErgRiProj_network: String,
*/
pub dest_id: String,
pub stateless: String,
pub line_display: String,
pub operator_code: Option<String>, pub operator_code: Option<String>,
pub operator_name: Option<String>, pub operator_name: Option<String>,
pub operator_publicCode: Option<String>, pub operator_public_code: Option<String>,
pub attrs_name: Option<String>, pub r#type: bool,
pub attrs_value: Option<String>, pub attrs: Vec<Attr>,
} }
#[derive(Queryable, Selectable, Insertable, Debug, QueryableByName)] //#[derive(Queryable, Selectable, Insertable, Debug, QueryableByName)]
#[diesel(table_name = depature_requests)] //#[diesel(table_name = departure_requests)]
#[diesel(check_for_backend(Pg))] //#[diesel(check_for_backend(Pg))]
#[diesel(primary_key(depatureRequestId))] //#[diesel(primary_key(departure_request_id))]
pub struct DepatureRequest { #[derive(Debug)]
pub depatureRequestId: Uuid, pub struct DepartureRequest {
pub stopid: Option<i32>, pub id: Uuid,
pub serverid: Option<String>, pub stop_id: Option<i32>,
pub requestid: Option<String>, pub server_id: Option<String>,
pub sessionid: Option<String>, pub request_id: Option<String>,
pub calcTime: Option<f64>, pub session_id: Option<String>,
pub serverTime: Option<NaiveDateTime>, pub calc_time: Option<f64>,
pub logRequestId: Option<String>, pub server_time: Option<NaiveDateTime>,
pub log_request_id: Option<String>,
} }
/*
#[derive(Queryable, Selectable, Insertable, Debug, QueryableByName)] //#[derive(Queryable, Selectable, Insertable, Debug, QueryableByName)]
#[diesel(table_name = depature_requests)] //#[diesel(table_name = departure_requests)]
pub struct NewDepatureRequest { pub struct NewDepatureRequest {
pub stopid: Option<i32>, pub stopid: Option<i32>,
pub serverid: Option<String>, pub serverid: String,
pub requestid: Option<String>, pub requestid: String,
pub sessionid: Option<String>, pub sessionid: String,
pub calcTime: Option<f64>, pub calcTime: f64,
pub serverTime: Option<NaiveDateTime>, pub serverTime: NaiveDateTime,
pub logRequestId: Option<String>, pub logRequest_id: String,
} }
*/
#[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)] //#[derive(Queryable, Selectable, Insertable, Debug, _identifiable, QueryableByName)]
#[diesel(table_name = depature_requeststation_hints)] //#[diesel(table_name = departure_requeststation_hints)]
#[diesel(check_for_backend(Pg))] //#[diesel(check_for_backend(Pg))]
#[diesel(primary_key(infoLinkUrl))] //#[diesel(primary_key(infoLinkUrl))]
/*
pub struct DepatureRequeststationHint { pub struct DepatureRequeststationHint {
pub infoLinkUrl: String, pub infoLinkUrl: String,
pub depatureRequestId: Option<Uuid>, pub departure_request_id: Uuid,
} }
*/
#[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)] //#[derive(Queryable, Selectable, Insertable, Debug, _identifiable, QueryableByName)]
#[diesel(table_name = serving_line_depature_requests)] //#[diesel(table_name = serving_line_departure_requests)]
#[diesel(primary_key(depatureRequestId))] //#[diesel(primary_key(departure_request_id))]
/*
pub struct ServingLineDepatureRequest { pub struct ServingLineDepatureRequest {
pub depatureRequestId: Uuid, pub departure_request_id: Uuid,
pub servingLineId: Uuid, pub id: Uuid,
} }
*/
#[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)] //#[derive(Queryable, Selectable, Insertable, Debug, _identifiable, QueryableByName)]
#[diesel(table_name = serving_lines)] //#[diesel(table_name = serving_lines)]
#[diesel(primary_key(servingLineId))] //#[diesel(primary_key(id))]
pub struct ServingLine { pub struct ServingLine {
pub servingLineId: Uuid, pub id: Uuid,
pub mode_name: Option<String>, pub name: String,
pub mode_number: Option<String>, pub number: Option<i32>,
pub mode_product: Option<String>, pub product: String,
pub mode_productId: Option<i32>, pub product_id: Option<i32>,
pub mode_type: Option<i32>, pub r#type: Option<i32>,
pub mode_code: Option<i32>, pub code: Option<i32>,
pub mode_destination: Option<String>, pub destination: String,
pub mode_destID: Option<i32>, pub dest_id: Option<i32>,
pub mode_desc: Option<String>, pub desc: String,
pub mode_timetablePeriod: Option<String>, pub timetable_period: String,
pub diva_branch: Option<String>, pub branch: String,
pub diva_line: Option<String>, /*pub line: String,
pub diva_supplement: Option<String>, pub supplement: String,
pub diva_dir: Option<String>, pub dir: String,
pub diva_project: Option<String>, pub project: String,
pub diva_network: Option<String>, pub network: String,
pub diva_stateless: Option<String>, */
pub diva_tripCode: Option<i32>, pub stateless: String,
pub diva_operator: Option<String>, pub trip_code: Option<i32>,
pub diva_opPublicCode: Option<String>, pub operator: String,
pub diva_opCode: Option<String>, pub op_public_code: Option<String>,
pub diva_Vf: Option<NaiveDate>, pub op_code: String,
pub diva_vTo: Option<NaiveDate>, pub v_f: Option<NaiveDate>,
pub diva_lineDisplay: Option<String>, pub v_to: Option<NaiveDate>,
pub meta_lastModificationDate: Option<NaiveDateTime>, pub line_display: String,
pub last_modified: NaiveDateTime,
} }
#[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)] //#[derive(Queryable, Selectable, Insertable, Debug, _identifiable, QueryableByName)]
#[diesel(table_name = station_hints)] //#[diesel(table_name = station_hints)]
#[diesel(primary_key(infoLinkURL))] //#[diesel(primary_key(infoLinkURL))]
pub struct StationHint { pub struct StationHint {
pub infoLinkURL: String, pub info_link_url: String,
pub infoLinkText: Option<String>, pub info_link_text: String,
pub infoText: Option<String>, pub info_text: String,
} }
#[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)] //#[derive(Queryable, Selectable, Insertable, Debug, _identifiable, QueryableByName)]
#[diesel(table_name = stop_lists)] //#[diesel(table_name = stop_lists)]
#[diesel(primary_key(input))] //#[diesel(primary_key(input))]
pub struct StopList { pub struct Stop {
pub input: i32, pub id: Option<i32>,
pub point_dm: Option<String>, pub usage: String,
pub point_type: Option<String>, pub r#type: String,
pub point_name: Option<String>, pub name: String,
pub point_stateless: Option<String>, pub anytype: String,
pub point_stopId: Option<i32>, pub sort: Option<i32>,
pub point_anytype: Option<String>, pub quality: Option<i32>,
pub point_sort: Option<i32>, pub best: Option<i32>,
pub point_quality: Option<i32>, pub object: String,
pub point_best: Option<i32>, pub gid: String,
pub point_object: Option<String>, pub omc: Option<i32>,
pub point_ref_id: Option<i32>, pub place_id: Option<i32>,
pub point_ref_gid: Option<String>, pub x: Option<f64>,
pub point_ref_omc: Option<i32>, pub y: Option<f64>,
pub point_ref_placeID: Option<i32>, pub map_name: Option<String>,
pub point_ref_place: Option<String>, pub place: String,
pub point_ref_coords: Option<String>, pub name_with_place: String,
pub itdOdvAssignedStops_stopid: Option<i32>, pub distance_time: Option<i32>,
pub itdOdvAssignedStops_name: Option<String>, pub is_transfer_stop: Option<bool>,
pub itdOdvAssignedStops_x: Option<f64>, pub vm: Option<i32>,
pub itdOdvAssignedStops_y: Option<f64>, pub last_modified: NaiveDateTime,
pub itdOdvAssignedStops_mapName: Option<String>,
pub itdOdvAssignedStops_value: Option<String>,
pub itdOdvAssignedStops_place: Option<String>,
pub itdOdvAssignedStops_nameWithPlace: Option<String>,
pub itdOdvAssignedStops_distanceTime: Option<i32>,
pub itdOdvAssignedStops_isTransferStop: Option<i32>,
pub itdOdvAssignedStops_vm: Option<i32>,
pub itdOdvAssignedStops_gid: Option<String>,
pub meta_lastModificationDate: Option<NaiveDateTime>,
} }

76
dataworker/src/schema.rs
View file

@ -1,5 +1,5 @@
// @generated automatically by Diesel CLI. // @generated automatically by Diesel CLI.
/*
diesel::table! { diesel::table! {
arrivals (stopID, servingLine_key, servingLine_stateless) { arrivals (stopID, servingLine_key, servingLine_stateless) {
depatureRequestId -> Uuid, depatureRequestId -> Uuid,
@ -70,6 +70,76 @@ diesel::table! {
} }
} }
diesel::table! {
departures (stopID, servingLine_key, servingLine_stateless) {
depatureRequestId -> Uuid,
stopID -> Nullable<Int4>,
x -> Nullable<Float8>,
y -> Nullable<Float8>,
#[max_length = 255]
mapName -> Nullable<Varchar>,
#[max_length = 50]
area -> Nullable<Varchar>,
#[max_length = 50]
platform -> Nullable<Varchar>,
#[max_length = 255]
platformName -> Nullable<Varchar>,
#[max_length = 255]
stopName -> Nullable<Varchar>,
#[max_length = 255]
nameWO -> Nullable<Varchar>,
#[max_length = 50]
pointType -> Nullable<Varchar>,
countdown -> Nullable<Int4>,
departureTime -> Nullable<Timestamp>,
realDepartureTime -> Nullable<Timestamp>,
#[max_length = 255]
servingLine_key -> Nullable<Varchar>,
#[max_length = 50]
servingLine_code -> Nullable<Varchar>,
#[max_length = 50]
servingLine_number -> Nullable<Varchar>,
#[max_length = 50]
servingLine_symbol -> Nullable<Varchar>,
servingLine_motType -> Nullable<Int4>,
servingLine_mtSubcode -> Nullable<Int4>,
servingLine_realtime -> Nullable<Bool>,
#[max_length = 255]
servingLine_direction -> Nullable<Varchar>,
#[max_length = 255]
servingLine_directionFrom -> Nullable<Varchar>,
#[max_length = 255]
servingLine_name -> Nullable<Varchar>,
servingLine_delay -> Nullable<Int4>,
#[max_length = 255]
servingLine_liErgRiProj_line -> Nullable<Varchar>,
#[max_length = 255]
servingLine_liErgRiProj_project -> Nullable<Varchar>,
#[max_length = 50]
servingLine_liErgRiProj_direction -> Nullable<Varchar>,
#[max_length = 50]
servingLine_liErgRiProj_supplement -> Nullable<Varchar>,
#[max_length = 255]
servingLine_liErgRiProj_network -> Nullable<Varchar>,
#[max_length = 255]
servingLine_destID -> Nullable<Varchar>,
#[max_length = 255]
servingLine_stateless -> Nullable<Varchar>,
#[max_length = 255]
servingLine_lineDisplay -> Nullable<Varchar>,
#[max_length = 50]
operator_code -> Nullable<Varchar>,
#[max_length = 255]
operator_name -> Nullable<Varchar>,
#[max_length = 50]
operator_publicCode -> Nullable<Varchar>,
#[max_length = 255]
attrs_name -> Nullable<Varchar>,
#[max_length = 255]
attrs_value -> Nullable<Varchar>,
}
}
diesel::table! { diesel::table! {
depature_requests (depatureRequestId) { depature_requests (depatureRequestId) {
depatureRequestId -> Uuid, depatureRequestId -> Uuid,
@ -117,8 +187,7 @@ diesel::table! {
#[max_length = 255] #[max_length = 255]
mode_destination -> Nullable<Varchar>, mode_destination -> Nullable<Varchar>,
mode_destID -> Nullable<Int4>, mode_destID -> Nullable<Int4>,
#[max_length = 255] mode_desc -> Nullable<Text>,
mode_desc -> Nullable<Varchar>,
#[max_length = 255] #[max_length = 255]
mode_timetablePeriod -> Nullable<Varchar>, mode_timetablePeriod -> Nullable<Varchar>,
#[max_length = 50] #[max_length = 50]
@ -227,3 +296,4 @@ diesel::allow_tables_to_appear_in_same_query!(
station_hints, station_hints,
stop_lists, stop_lists,
); );
*/