From 5e681ee1c834ee6361aa76b06cecb3410e193899 Mon Sep 17 00:00:00 2001 From: theBreadCompany Date: Fri, 24 Jan 2025 00:03:22 +0100 Subject: [PATCH] add Dockerfile; format stuff; rebuild for uuids --- .dockerignore | 6 +++ .gitignore | 1 + Cargo.lock | 84 +++++--------------------------- Cargo.toml | 1 + Dockerfile | 8 +++ datafetcher/.env | 1 + datafetcher/src/main.rs | 4 +- dataworker/.env | 1 + dataworker/Cargo.toml | 7 +-- dataworker/VRN Data Database.sql | 18 +++---- dataworker/diesel.toml | 2 +- dataworker/src/dtos.rs | 2 +- dataworker/src/lib.rs | 21 ++++---- dataworker/src/main.rs | 2 +- dataworker/src/models.rs | 14 +++--- dataworker/src/schema.rs | 14 +++--- docker-compose.yml | 6 +-- 17 files changed, 75 insertions(+), 117 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 120000 datafetcher/.env create mode 120000 dataworker/.env diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cd60df2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +target +.gitignore +README.md +Dockerfile +docker-compose.yml +migrations \ No newline at end of file diff --git a/.gitignore b/.gitignore index ea8c4bf..5ad255f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +migrations \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 4d99ee0..0572f21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,15 +134,6 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - [[package]] name = "bumpalo" version = "3.16.0" @@ -223,25 +214,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "cpufeatures" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" -dependencies = [ - "libc", -] - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "darling" version = "0.20.10" @@ -299,11 +271,8 @@ dependencies = [ "diesel", "dotenvy", "hex", - "maud", - "reqwest", "serde", "serde_json", - "sha2", "tokio", "uuid", ] @@ -362,6 +331,7 @@ dependencies = [ "diesel_derives", "itoa", "pq-sys", + "uuid", ] [[package]] @@ -386,16 +356,6 @@ dependencies = [ "syn", ] -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -586,16 +546,6 @@ dependencies = [ "windows", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.15" @@ -1044,9 +994,9 @@ checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "is-terminal" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f187290c0ed3dfe3f7c85bedddd320949b68fc86ca0ceb71adfb05b3dc3af2a" +checksum = "e19b23d53f35ce9f56aebc7d1bb4e6ac1e9c0db7ac85c8d1760c04379edced37" dependencies = [ "hermit-abi 0.4.0", "libc", @@ -1302,9 +1252,9 @@ dependencies = [ [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" @@ -1883,15 +1833,10 @@ dependencies = [ ] [[package]] -name = "sha2" -version = "0.10.8" +name = "sha1_smol" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" [[package]] name = "sharded-slab" @@ -2302,12 +2247,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "typenum" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" - [[package]] name = "ubyte" version = "0.10.4" @@ -2329,9 +2268,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243" [[package]] name = "unicode-xid" @@ -2373,6 +2312,9 @@ name = "uuid" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" +dependencies = [ + "sha1_smol", +] [[package]] name = "valuable" diff --git a/Cargo.toml b/Cargo.toml index 7fcfa9d..ac81d99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,2 +1,3 @@ [workspace] members = [ "datafetcher","dataworker"] +resolver = "2" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..91ab93a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM rust:latest +LABEL authors="thebread " + +WORKDIR /app +COPY . . +RUN cargo build + +CMD ["cargo", "run", "--bin", "datafetcher"] \ No newline at end of file diff --git a/datafetcher/.env b/datafetcher/.env new file mode 120000 index 0000000..2eea525 --- /dev/null +++ b/datafetcher/.env @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/datafetcher/src/main.rs b/datafetcher/src/main.rs index c3bd78c..ba7dcc7 100644 --- a/datafetcher/src/main.rs +++ b/datafetcher/src/main.rs @@ -4,9 +4,7 @@ use maud::{html, Markup, PreEscaped, DOCTYPE}; use rocket::response::stream::{Event, EventStream}; use rocket::tokio::time::{interval, Duration}; -struct SsePayload { - -} +struct SsePayload {} #[get("/sse")] async fn sse() -> EventStream![] { diff --git a/dataworker/.env b/dataworker/.env new file mode 120000 index 0000000..2eea525 --- /dev/null +++ b/dataworker/.env @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/dataworker/Cargo.toml b/dataworker/Cargo.toml index 6b2550f..46fb69f 100644 --- a/dataworker/Cargo.toml +++ b/dataworker/Cargo.toml @@ -5,13 +5,10 @@ edition = "2021" [dependencies] chrono = "0.4.39" -diesel = { version = "2.2.6", features = ["64-column-tables", "chrono", "postgres", "postgres_backend"] } +diesel = { version = "2.2.6", features = ["64-column-tables", "chrono", "postgres", "postgres_backend", "uuid"] } dotenvy = "0.15.7" hex = "0.4.3" -maud = "0.26.0" -reqwest = { version = "0.12.12", features = ["json"] } serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.135" -sha2 = "0.10.8" tokio = { version = "1.43.0", features = ["full"] } -uuid = "1.12.0" +uuid = { version = "1.12.1", features = ["v5"] } diff --git a/dataworker/VRN Data Database.sql b/dataworker/VRN Data Database.sql index 16e9752..ceb1c7c 100644 --- a/dataworker/VRN Data Database.sql +++ b/dataworker/VRN Data Database.sql @@ -32,7 +32,7 @@ CREATE TABLE "stop_lists" ( ); CREATE TABLE "serving_lines" ( - "servingLineId" varchar(255) PRIMARY KEY, + "servingLineId" uuid PRIMARY KEY, "mode_name" varchar(255), "mode_number" varchar(50), "mode_product" varchar(255), @@ -61,7 +61,7 @@ CREATE TABLE "serving_lines" ( ); CREATE TABLE "arrivals" ( - "depatureRequestId" serial, + "depatureRequestId" uuid, "stopID" int, "x" float, "y" float, @@ -103,15 +103,15 @@ CREATE TABLE "arrivals" ( ); CREATE TABLE "depature_requests" ( - "depatureRequestId" serial PRIMARY KEY, - "stopid" int unique, + "depatureRequestId" uuid PRIMARY KEY, + "stopid" int, "serverid" varchar(255), "requestid" varchar(255), "sessionid" varchar(255), "calcTime" float, "serverTime" timestamp, "logRequestId" varchar(255), - unique ("depatureRequestId", "stopid") + unique ("depatureRequestId") ); CREATE TABLE "station_hints" ( @@ -122,18 +122,18 @@ CREATE TABLE "station_hints" ( CREATE TABLE "depature_requeststation_hints" ( "infoLinkUrl" varchar(255) PRIMARY KEY, - "depatureRequestId" int + "depatureRequestId" uuid ); CREATE TABLE "serving_line_depature_requests" ( - "depatureRequestId" int PRIMARY KEY, - "servingLineId" varchar(255) + "depatureRequestId" uuid PRIMARY KEY, + "servingLineId" uuid ); ALTER TABLE "arrivals" ADD FOREIGN KEY ("depatureRequestId") REFERENCES "depature_requests" ("depatureRequestId"); --ALTER TABLE "depature_requests" ADD FOREIGN KEY ("stopid") REFERENCES "stop_lists" ("input"); -ALTER TABLE "stop_lists" ADD FOREIGN KEY ("input") REFERENCES "depature_requests" ("stopid"); +--ALTER TABLE "stop_lists" ADD FOREIGN KEY ("input") REFERENCES "depature_requests" ("stopid"); ALTER TABLE "depature_requeststation_hints" ADD FOREIGN KEY ("infoLinkUrl") REFERENCES "station_hints" ("infoLinkURL"); diff --git a/dataworker/diesel.toml b/dataworker/diesel.toml index 6698255..a0d61bf 100644 --- a/dataworker/diesel.toml +++ b/dataworker/diesel.toml @@ -6,4 +6,4 @@ file = "src/schema.rs" custom_type_derives = ["diesel::query_builder::QueryId", "Clone"] [migrations_directory] -dir = "/home/bread/Dokumente/willdaten/migrations" +dir = "migrations" diff --git a/dataworker/src/dtos.rs b/dataworker/src/dtos.rs index 47287d7..6367f47 100644 --- a/dataworker/src/dtos.rs +++ b/dataworker/src/dtos.rs @@ -279,7 +279,7 @@ pub struct Attr { #[serde(rename_all = "camelCase")] pub struct DepartureList { #[serde(rename = "stopID")] - pub stop_id: i32, + pub stop_id: String, pub x: String, pub y: String, pub map_name: String, diff --git a/dataworker/src/lib.rs b/dataworker/src/lib.rs index dbe2660..a68a8e5 100644 --- a/dataworker/src/lib.rs +++ b/dataworker/src/lib.rs @@ -12,8 +12,7 @@ use diesel::{dsl::insert_into, Connection, PgConnection, RunQueryDsl, Selectable use dtos::Root; use hex; use models::{ - Arrival, DepatureRequest, NewDepatureRequest, ServingLine, ServingLineDepatureRequest, - StationHint, StopList, + Arrival, DepatureRequest, ServingLine, ServingLineDepatureRequest, StationHint, StopList, }; use schema::arrivals::dsl::arrivals; use schema::depature_requests::dsl::depature_requests; @@ -22,14 +21,17 @@ use schema::serving_lines::dsl::serving_lines; use schema::station_hints::dsl::station_hints; use schema::stop_lists::dsl::stop_lists; use serde_json; -use sha2::Digest; -use sha2::Sha256; -use std::{env, error::Error, fs, io::Write, str::FromStr}; +use std::{error::Error, fs, io::Write, str::FromStr}; +use uuid::Uuid; pub fn insert_document(data: Root, conn: &mut PgConnection) -> Result<(), Box> { let arrival_list = data.arrival_list.unwrap(); if let Some(req) = insert_into(depature_requests) - .values(NewDepatureRequest { + .values(DepatureRequest { + depatureRequestId: Uuid::new_v5( + &Uuid::NAMESPACE_DNS, + serde_json::to_string(&data.parameters)?.as_bytes(), + ), stopid: i32::from_str(&data.dm.itd_odv_assigned_stops.stop_id).ok(), serverid: data .parameters @@ -130,9 +132,10 @@ pub fn insert_document(data: Root, conn: &mut PgConnection) -> Result<(), Box Result<(), Box> { dotenv().ok(); let url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); let mut conn = - PgConnection::establish(&url).unwrap_or_else(|_| panic!("Error connecting to {}", url)); + PgConnection::establish(&url).unwrap_or_else(|e| panic!("Error connecting to {}: {}", url, e)); let src = "/home/bread/Downloads/2025-01-22(1)/2025-01-22"; let total = fs::read_dir(src).unwrap().count(); diff --git a/dataworker/src/models.rs b/dataworker/src/models.rs index 60b9159..f95d1b5 100644 --- a/dataworker/src/models.rs +++ b/dataworker/src/models.rs @@ -8,12 +8,14 @@ use crate::schema::*; use chrono::NaiveDate; use chrono::NaiveDateTime; use diesel::pg::Pg; +use uuid::Uuid; + #[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)] #[diesel(table_name = arrivals)] #[diesel(check_for_backend(Pg))] #[diesel(primary_key(stopID, servingLine_key, servingLine_stateless))] pub struct Arrival { - pub depatureRequestId: i32, + pub depatureRequestId: Uuid, pub stopID: Option, pub x: Option, pub y: Option, @@ -58,7 +60,7 @@ pub struct Arrival { #[diesel(check_for_backend(Pg))] #[diesel(primary_key(depatureRequestId))] pub struct DepatureRequest { - pub depatureRequestId: i32, + pub depatureRequestId: Uuid, pub stopid: Option, pub serverid: Option, pub requestid: Option, @@ -86,22 +88,22 @@ pub struct NewDepatureRequest { #[diesel(primary_key(infoLinkUrl))] pub struct DepatureRequeststationHint { pub infoLinkUrl: String, - pub depatureRequestId: Option, + pub depatureRequestId: Option, } #[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)] #[diesel(table_name = serving_line_depature_requests)] #[diesel(primary_key(depatureRequestId))] pub struct ServingLineDepatureRequest { - pub depatureRequestId: i32, - pub servingLineId: String, + pub depatureRequestId: Uuid, + pub servingLineId: Uuid, } #[derive(Queryable, Selectable, Insertable, Debug, Identifiable, QueryableByName)] #[diesel(table_name = serving_lines)] #[diesel(primary_key(servingLineId))] pub struct ServingLine { - pub servingLineId: String, + pub servingLineId: Uuid, pub mode_name: Option, pub mode_number: Option, pub mode_product: Option, diff --git a/dataworker/src/schema.rs b/dataworker/src/schema.rs index c418af0..749f35c 100644 --- a/dataworker/src/schema.rs +++ b/dataworker/src/schema.rs @@ -2,7 +2,7 @@ diesel::table! { arrivals (stopID, servingLine_key, servingLine_stateless) { - depatureRequestId -> Int4, + depatureRequestId -> Uuid, stopID -> Nullable, x -> Nullable, y -> Nullable, @@ -72,7 +72,7 @@ diesel::table! { diesel::table! { depature_requests (depatureRequestId) { - depatureRequestId -> Int4, + depatureRequestId -> Uuid, stopid -> Nullable, #[max_length = 255] serverid -> Nullable, @@ -91,22 +91,20 @@ diesel::table! { depature_requeststation_hints (infoLinkUrl) { #[max_length = 255] infoLinkUrl -> Varchar, - depatureRequestId -> Nullable, + depatureRequestId -> Nullable, } } diesel::table! { serving_line_depature_requests (depatureRequestId) { - depatureRequestId -> Int4, - #[max_length = 255] - servingLineId -> VarChar, + depatureRequestId -> Uuid, + servingLineId -> Uuid, } } diesel::table! { serving_lines (servingLineId) { - #[max_length = 255] - servingLineId -> VarChar, + servingLineId -> Uuid, #[max_length = 255] mode_name -> Nullable, #[max_length = 50] diff --git a/docker-compose.yml b/docker-compose.yml index 6b5db7a..4fa9055 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,10 @@ services: - db: image: postgres restart: always environment: POSTGRES_HOST_AUTH_METHOD: trust - POSTGRES_USER: postgres + POSTGRES_USER: bread + POSTGRES_DB: bread ports: - - 5432:5432 + - 5433:5432