add Dockerfile; format stuff; rebuild for uuids

.dockerignore

@@ -0,0 +1,6 @@
+target
+.gitignore
+README.md
+Dockerfile
+docker-compose.yml
+migrations

.gitignore

@@ -1 +1,2 @@
 /target
+migrations

Cargo.lock

@@ -134,15 +134,6 @@ version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
 
-[[package]]
-name = "block-buffer"
-version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
-dependencies = [
- "generic-array",
-]
-
 [[package]]
 name = "bumpalo"
 version = "3.16.0"
@@ -223,25 +214,6 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
-[[package]]
-name = "cpufeatures"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "crypto-common"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
 [[package]]
 name = "darling"
 version = "0.20.10"
@@ -299,11 +271,8 @@ dependencies = [
  "diesel",
  "dotenvy",
  "hex",
- "maud",
- "reqwest",
  "serde",
  "serde_json",
- "sha2",
  "tokio",
  "uuid",
 ]
@@ -362,6 +331,7 @@ dependencies = [
  "diesel_derives",
  "itoa",
  "pq-sys",
+ "uuid",
 ]
 
 [[package]]
@@ -386,16 +356,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "digest"
-version = "0.10.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
-dependencies = [
- "block-buffer",
- "crypto-common",
-]
-
 [[package]]
 name = "displaydoc"
 version = "0.2.5"
@@ -586,16 +546,6 @@ dependencies = [
  "windows",
 ]
 
-[[package]]
-name = "generic-array"
-version = "0.14.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
-dependencies = [
- "typenum",
- "version_check",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -1044,9 +994,9 @@ checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
 
 [[package]]
 name = "is-terminal"
-version = "0.4.14"
+version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f187290c0ed3dfe3f7c85bedddd320949b68fc86ca0ceb71adfb05b3dc3af2a"
+checksum = "e19b23d53f35ce9f56aebc7d1bb4e6ac1e9c0db7ac85c8d1760c04379edced37"
 dependencies = [
  "hermit-abi 0.4.0",
  "libc",
@@ -1302,9 +1252,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-probe"
-version = "0.1.5"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
 
 [[package]]
 name = "openssl-sys"
@@ -1883,15 +1833,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "sha2"
+name = "sha1_smol"
-version = "0.10.8"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
+checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
-dependencies = [
- "cfg-if",
- "cpufeatures",
- "digest",
-]
 
 [[package]]
 name = "sharded-slab"
@@ -2302,12 +2247,6 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
-[[package]]
-name = "typenum"
-version = "1.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
-
 [[package]]
 name = "ubyte"
 version = "0.10.4"
@@ -2329,9 +2268,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.14"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
+checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243"
 
 [[package]]
 name = "unicode-xid"
@@ -2373,6 +2312,9 @@ name = "uuid"
 version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b"
+dependencies = [
+ "sha1_smol",
+]
 
 [[package]]
 name = "valuable"

Cargo.toml

@@ -1,2 +1,3 @@
 [workspace]
 members = [ "datafetcher","dataworker"]
+resolver = "2"

Dockerfile

@@ -0,0 +1,8 @@
+FROM rust:latest
+LABEL authors="thebread <contact@thebread.dev>"
+
+WORKDIR /app
+COPY . .
+RUN cargo build
+
+CMD ["cargo", "run", "--bin", "datafetcher"]

datafetcher/.env

@@ -0,0 +1 @@
+.env

datafetcher/src/main.rs

@@ -4,9 +4,7 @@ use maud::{html, Markup, PreEscaped, DOCTYPE};
 use rocket::response::stream::{Event, EventStream};
 use rocket::tokio::time::{interval, Duration};
 
-struct SsePayload {
+struct SsePayload {}
-
-}
 
 #[get("/sse")]
 async fn sse() -> EventStream![] {

dataworker/.env

@@ -0,0 +1 @@
+.env

dataworker/Cargo.toml

@@ -5,13 +5,10 @@ edition = "2021"
 
 [dependencies]
 chrono = "0.4.39"
-diesel = { version = "2.2.6", features = ["64-column-tables", "chrono", "postgres", "postgres_backend"] }
+diesel = { version = "2.2.6", features = ["64-column-tables", "chrono", "postgres", "postgres_backend", "uuid"] }
 dotenvy = "0.15.7"
 hex = "0.4.3"
-maud = "0.26.0"
-reqwest = { version = "0.12.12", features = ["json"] }
 serde = { version = "1.0.217", features = ["derive"] }
 serde_json = "1.0.135"
-sha2 = "0.10.8"
 tokio = { version = "1.43.0", features = ["full"] }
-uuid = "1.12.0"
+uuid = { version = "1.12.1", features = ["v5"] }

dataworker/VRN Data Database.sql

@@ -32,7 +32,7 @@ CREATE TABLE "stop_lists" (
 );
 
 CREATE TABLE "serving_lines" (
-  "servingLineId" varchar(255) PRIMARY KEY,
+  "servingLineId" uuid PRIMARY KEY,
   "mode_name" varchar(255),
   "mode_number" varchar(50),
   "mode_product" varchar(255),
@@ -61,7 +61,7 @@ CREATE TABLE "serving_lines" (
 );
 
 CREATE TABLE "arrivals" (
-  "depatureRequestId" serial,
+  "depatureRequestId" uuid,
   "stopID" int,
   "x" float,
   "y" float,
@@ -103,15 +103,15 @@ CREATE TABLE "arrivals" (
 );
 
 CREATE TABLE "depature_requests" (
-  "depatureRequestId" serial PRIMARY KEY,
+  "depatureRequestId" uuid PRIMARY KEY,
-  "stopid" int unique,
+  "stopid" int,
   "serverid" varchar(255),
   "requestid" varchar(255),
   "sessionid" varchar(255),
   "calcTime" float,
   "serverTime" timestamp,
   "logRequestId" varchar(255),
-  unique ("depatureRequestId", "stopid")
+  unique ("depatureRequestId")
 );
 
 CREATE TABLE "station_hints" (
@@ -122,18 +122,18 @@ CREATE TABLE "station_hints" (
 
 CREATE TABLE "depature_requeststation_hints" (
   "infoLinkUrl" varchar(255) PRIMARY KEY,
-  "depatureRequestId" int
+  "depatureRequestId" uuid
 );
 
 CREATE TABLE "serving_line_depature_requests" (
-  "depatureRequestId" int PRIMARY KEY,
+  "depatureRequestId" uuid PRIMARY KEY,
-  "servingLineId" varchar(255)
+  "servingLineId" uuid
 );
 
 ALTER TABLE "arrivals" ADD FOREIGN KEY ("depatureRequestId") REFERENCES "depature_requests" ("depatureRequestId");
 
 --ALTER TABLE "depature_requests" ADD FOREIGN KEY ("stopid") REFERENCES "stop_lists" ("input");
-ALTER TABLE "stop_lists" ADD FOREIGN KEY ("input") REFERENCES "depature_requests" ("stopid");
+--ALTER TABLE "stop_lists" ADD FOREIGN KEY ("input") REFERENCES "depature_requests" ("stopid");
 
 ALTER TABLE "depature_requeststation_hints" ADD FOREIGN KEY ("infoLinkUrl") REFERENCES "station_hints" ("infoLinkURL");
 

dataworker/diesel.toml

@@ -6,4 +6,4 @@ file = "src/schema.rs"
 custom_type_derives = ["diesel::query_builder::QueryId", "Clone"]
 
 [migrations_directory]
-dir = "/home/bread/Dokumente/willdaten/migrations"
+dir = "migrations"

dataworker/src/dtos.rs

@@ -279,7 +279,7 @@ pub struct Attr {
 #[serde(rename_all = "camelCase")]
 pub struct DepartureList {
     #[serde(rename = "stopID")]
-    pub stop_id: i32,
+    pub stop_id: String,
     pub x: String,
     pub y: String,
     pub map_name: String,

dataworker/src/lib.rs

@@ -12,8 +12,7 @@ use diesel::{dsl::insert_into, Connection, PgConnection, RunQueryDsl, Selectable
 use dtos::Root;
 use hex;
 use models::{
-    Arrival, DepatureRequest, NewDepatureRequest, ServingLine, ServingLineDepatureRequest,
+    Arrival, DepatureRequest, ServingLine, ServingLineDepatureRequest, StationHint, StopList,
-    StationHint, StopList,
 };
 use schema::arrivals::dsl::arrivals;
 use schema::depature_requests::dsl::depature_requests;
@@ -22,14 +21,17 @@ use schema::serving_lines::dsl::serving_lines;
 use schema::station_hints::dsl::station_hints;
 use schema::stop_lists::dsl::stop_lists;
 use serde_json;
-use sha2::Digest;
+use std::{error::Error, fs, io::Write, str::FromStr};
-use sha2::Sha256;
+use uuid::Uuid;
-use std::{env, error::Error, fs, io::Write, str::FromStr};
 
 pub fn insert_document(data: Root, conn: &mut PgConnection) -> Result<(), Box<dyn Error>> {
     let arrival_list = data.arrival_list.unwrap();
     if let Some(req) = insert_into(depature_requests)
-        .values(NewDepatureRequest {
+        .values(DepatureRequest {
+            depatureRequestId: Uuid::new_v5(
+                &Uuid::NAMESPACE_DNS,
+                serde_json::to_string(&data.parameters)?.as_bytes(),
+            ),
             stopid: i32::from_str(&data.dm.itd_odv_assigned_stops.stop_id).ok(),
             serverid: data
                 .parameters
@@ -130,9 +132,10 @@ pub fn insert_document(data: Root, conn: &mut PgConnection) -> Result<(), Box<dy
             .get_result(conn)
             .optional();
         for line in data.serving_lines.lines {
-            let id = hex::encode(Sha256::digest(
+            let id = Uuid::new_v5(
-                serde_json::to_string(&line.mode).unwrap().into_bytes(),
+                &Uuid::NAMESPACE_DNS,
-            ));
+                serde_json::to_string(&line)?.as_bytes(),
+            );
             let _ = insert_into(serving_lines)
                 .values(ServingLine {
                     servingLineId: id.clone(),

dataworker/src/main.rs

@@ -14,7 +14,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
     dotenv().ok();
     let url = env::var("DATABASE_URL").expect("DATABASE_URL must be set");
     let mut conn =
-        PgConnection::establish(&url).unwrap_or_else(|_| panic!("Error connecting to {}", url));
+        PgConnection::establish(&url).unwrap_or_else(|e| panic!("Error connecting to {}: {}", url, e));
 
     let src = "/home/bread/Downloads/2025-01-22(1)/2025-01-22";
     let total = fs::read_dir(src).unwrap().count();

dataworker/src/models.rs

@@ -8,12 +8,14 @@ use crate::schema::*;
 use chrono::NaiveDate;
 use chrono::NaiveDateTime;
 use diesel::pg::Pg;
+use uuid::Uuid;
+
 #[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)]
 #[diesel(table_name = arrivals)]
 #[diesel(check_for_backend(Pg))]
 #[diesel(primary_key(stopID, servingLine_key, servingLine_stateless))]
 pub struct Arrival {
-    pub depatureRequestId: i32,
+    pub depatureRequestId: Uuid,
     pub stopID: Option<i32>,
     pub x: Option<f64>,
     pub y: Option<f64>,
@@ -58,7 +60,7 @@ pub struct Arrival {
 #[diesel(check_for_backend(Pg))]
 #[diesel(primary_key(depatureRequestId))]
 pub struct DepatureRequest {
-    pub depatureRequestId: i32,
+    pub depatureRequestId: Uuid,
     pub stopid: Option<i32>,
     pub serverid: Option<String>,
     pub requestid: Option<String>,
@@ -86,22 +88,22 @@ pub struct NewDepatureRequest {
 #[diesel(primary_key(infoLinkUrl))]
 pub struct DepatureRequeststationHint {
     pub infoLinkUrl: String,
-    pub depatureRequestId: Option<i32>,
+    pub depatureRequestId: Option<Uuid>,
 }
 
 #[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)]
 #[diesel(table_name = serving_line_depature_requests)]
 #[diesel(primary_key(depatureRequestId))]
 pub struct ServingLineDepatureRequest {
-    pub depatureRequestId: i32,
+    pub depatureRequestId: Uuid,
-    pub servingLineId: String,
+    pub servingLineId: Uuid,
 }
 
 #[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)]
 #[diesel(table_name = serving_lines)]
 #[diesel(primary_key(servingLineId))]
 pub struct ServingLine {
-    pub servingLineId: String,
+    pub servingLineId: Uuid,
     pub mode_name: Option<String>,
     pub mode_number: Option<String>,
     pub mode_product: Option<String>,

dataworker/src/schema.rs

@@ -2,7 +2,7 @@
 
 diesel::table! {
     arrivals (stopID, servingLine_key, servingLine_stateless) {
-        depatureRequestId -> Int4,
+        depatureRequestId -> Uuid,
         stopID -> Nullable<Int4>,
         x -> Nullable<Float8>,
         y -> Nullable<Float8>,
@@ -72,7 +72,7 @@ diesel::table! {
 
 diesel::table! {
     depature_requests (depatureRequestId) {
-        depatureRequestId -> Int4,
+        depatureRequestId -> Uuid,
         stopid -> Nullable<Int4>,
         #[max_length = 255]
         serverid -> Nullable<Varchar>,
@@ -91,22 +91,20 @@ diesel::table! {
     depature_requeststation_hints (infoLinkUrl) {
         #[max_length = 255]
         infoLinkUrl -> Varchar,
-        depatureRequestId -> Nullable<Int4>,
+        depatureRequestId -> Nullable<Uuid>,
     }
 }
 
 diesel::table! {
     serving_line_depature_requests (depatureRequestId) {
-        depatureRequestId -> Int4,
+        depatureRequestId -> Uuid,
-        #[max_length = 255]
+        servingLineId -> Uuid,
-        servingLineId -> VarChar,
     }
 }
 
 diesel::table! {
     serving_lines (servingLineId) {
-        #[max_length = 255]
+        servingLineId -> Uuid,
-        servingLineId -> VarChar,
         #[max_length = 255]
         mode_name -> Nullable<Varchar>,
         #[max_length = 50]

docker-compose.yml

@@ -1,10 +1,10 @@
 services:
-
   db:
     image: postgres
     restart: always
     environment:
       POSTGRES_HOST_AUTH_METHOD: trust
-      POSTGRES_USER: postgres
+      POSTGRES_USER: bread
+      POSTGRES_DB: bread
     ports:
-      - 5432:5432
+      - 5433:5432