Add command to update data and report blockages

src/bin/server.rs

@@ -67,16 +67,21 @@ async fn update_daily_info(db: &Db, distributors: &BTreeMap<BridgeDistributor, S
 
 async fn create_context_manager(
     db_config: DbConfig,
+    distributors: BTreeMap<BridgeDistributor, String>,
     context_rx: mpsc::Receiver<Command>,
     mut kill: broadcast::Receiver<()>,
 ) {
     tokio::select! {
-        create_context = context_manager(db_config, context_rx) => create_context,
+        create_context = context_manager(db_config, distributors, context_rx) => create_context,
         _ = kill.recv() => {println!("Shut down manager");},
     }
 }
 
-async fn context_manager(db_config: DbConfig, mut context_rx: mpsc::Receiver<Command>) {
+async fn context_manager(
+    db_config: DbConfig,
+    distributors: BTreeMap<BridgeDistributor, String>,
+    mut context_rx: mpsc::Receiver<Command>,
+) {
     let db: Db = sled::open(&db_config.db_path).unwrap();
 
     while let Some(cmd) = context_rx.recv().await {
@@ -94,6 +99,9 @@ async fn context_manager(db_config: DbConfig, mut context_rx: mpsc::Receiver<Com
                 drop(shutdown_sig);
                 println!("Shutdown Sent.");
             }
+            Update {} => {
+                update_daily_info(&db, &distributors).await;
+            }
         }
     }
 }
@@ -108,6 +116,7 @@ enum Command {
     Shutdown {
         shutdown_sig: broadcast::Sender<()>,
     },
+    Update {},
 }
 
 #[tokio::main]
@@ -149,8 +158,9 @@ async fn main() {
         }
     });
 
-    let context_manager =
-        spawn(async move { create_context_manager(config.db, request_rx, kill).await });
+    let context_manager = spawn(async move {
+        create_context_manager(config.db, config.distributors, request_rx, kill).await
+    });
 
     let make_service = make_service_fn(move |_conn: &AddrStream| {
         let request_tx = request_tx.clone();

src/extra_info.rs

@@ -174,7 +174,21 @@ pub async fn download_extra_infos(
         }
     }
 
-    let doc = Document::from(body_str.as_str());
+    // Removed because it caused some problem...
+    //let doc = Document::from(body_str.clone().as_str());
+    // Instead, do this
+    let mut links = HashSet::<String>::new();
+    for line in body_str.lines() {
+        let begin_match = "<a href=\"";
+        let end_match = "\">";
+        if line.contains(begin_match) {
+            let link = &line[line.find(begin_match).unwrap() + begin_match.len()..];
+            if link.contains(end_match) {
+                let link = &link[0..link.find(end_match).unwrap()];
+                links.insert(link.to_string());
+            }
+        }
+    }
 
     // Create extra-infos directory if it doesn't exist
     std::fs::create_dir_all(&DIRECTORY)?;
@@ -182,7 +196,6 @@ pub async fn download_extra_infos(
     let mut new_files = HashSet::<String>::new();
 
     // Go through all the links in the page and download new files
-    let links = doc.find(Name("a")).filter_map(|n| n.attr("href"));
     for link in links {
         if link.ends_with("-extra-infos") {
             let filename = format!("{}/{}", DIRECTORY, link);
@@ -191,7 +204,7 @@ pub async fn download_extra_infos(
             if !Path::new(&filename).exists() {
                 let extra_infos_url = format!("{}{}", base_url, link);
                 println!("Downloading {}", extra_infos_url);
-                let mut res = client.get(extra_infos_url.parse().unwrap()).await?;
+                let mut res = client.get(extra_infos_url.parse().unwrap()).await.unwrap();
                 assert_eq!(res.status(), StatusCode::OK);
                 let mut file = std::fs::File::create(filename).unwrap();
                 while let Some(next) = res.frame().await {