async hell

datafetcher/src/main.rs

@@ -6,7 +6,8 @@ use sqlx::postgres::PgPoolOptions;
 use dotenvy::dotenv;
 use std::env;
 use std::io::Write;
-use tokio::time::{self, Duration};
+use tokio::task;
+use tokio::time::{self, sleep, Duration};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -27,15 +28,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut count = 0;
 
     let client = reqwest::Client::new();
-    let mut interval = time::interval(Duration::from_secs(1800));
+    //let mut interval = time::interval(Duration::from_secs(1800));
 
     loop {
-        print!("\nWaiting for next interval tick... ");
+        let mut handles = vec![];
         std::io::stdout().flush().unwrap();
-        interval.tick().await;
+        //interval.tick().await;
         let date = Local::now();
-        println!("Ticked on {}", &date.format("%Y.%m.%d %H:%M"));
+        //println!("Ticked on {}", &date.format("%Y.%m.%d %H:%M"));
             for station in &stations {
+                let station = station.clone();
+                let req = req.clone();
+                let client = client.clone();
+                let mut conn = conn.clone();
+                handles.push(task::spawn(async move {
                     count += 1;
                 let req = req
                     .replace("TEMPLATE_ID", &station)
@@ -45,9 +51,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 std::io::stdout().flush().unwrap();
                 if let Ok(doc) = client
                     .get(req.replace("TEMPLATE_DEPARR", "arr"))
-                .header("Authorization", &token)
+                    //.header("Authorization", &token)
                     .send()
-                .await?
+                    .await
+                    .unwrap()
                     .json::<Root>()
                     .await
                 {
@@ -60,9 +67,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 }
                 if let Ok(doc) = client
                     .get(req.replace("TEMPLATE_DEPARR", "dep"))
-                .header("Authorization", &token)
+                    //.header("Authorization", &token)
                     .send()
-                .await?
+                    .await
+                    .unwrap()
                     .json::<Root>()
                     .await
                 {
@@ -73,6 +81,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 } else {
                     println!("\x1B[2K\rFailed to fetch departures for station {}", station);
                 }
+                }));
+            }
+        for handle in handles {
+            handle.await?;
+            sleep(Duration::from_millis(100)).await;
         }
         count = 0;
     }

dataworker/src/main.rs

@@ -19,7 +19,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
         .connect(&url)
         .await?;
 
-    let src = "input";
+    let src = "input/xslt_dm_request";
     let total = ReadDirStream::new(fs::read_dir(src).await?).count().await;
     let mut stream = ReadDirStream::new(fs::read_dir(src).await?);
     let mut count = 0;