Search in parallel

Cargo.lock

@@ -88,6 +88,7 @@ name = "boilr"
 version = "0.2.7"
 dependencies = [
  "config",
+ "dashmap",
  "failure",
  "fl2rust",
  "flate2",
@@ -184,6 +185,17 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "dashmap"
+version = "4.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
+dependencies = [
+ "cfg-if",
+ "num_cpus",
+ "serde 1.0.130",
+]
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.28"

Cargo.toml

@@ -20,10 +20,11 @@ nom_locate = "^3.0.*"
 flate2 = "^1.0.22"
 toml = { version = "^0.5.8", optional = true }
 futures = { version = "^0.3.17" }
+dashmap = { version = "^4.0.2", features = ["serde"] }
 
 [build-dependencies]
 fl2rust = { version = "0.4", optional = true }
 
 [features]
 # default = ["ui"]
-ui = ["fltk", "fl2rust", "toml"]
+ui = ["fltk", "fl2rust", "toml"]

src/steamgriddb/cached_search.rs

@@ -1,6 +1,7 @@
-use std::{collections::HashMap, fs::File, io::Write, path::Path};
+use std::{ fs::File, io::Write, path::Path};
+use dashmap::DashMap;
 
-type SearchMap = HashMap<u32, (String, usize)>;
+type SearchMap = DashMap<u32, (String, usize)>;
 
 pub struct CachedSearch<'a> {
     search_map: SearchMap,
@@ -20,7 +21,7 @@ impl<'a> CachedSearch<'a> {
     }
 
     pub async fn search<S>(
-        &mut self,
+        &self,
         app_id: u32,
         query: S,
     ) -> Result<Option<usize>, Box<dyn std::error::Error>>

src/steamgriddb/downloader.rs

@@ -20,59 +20,49 @@ const CONCURRENT_REQUESTS: usize = 10;
 pub async fn download_images_for_users<'b>(settings: &Settings, users: &Vec<SteamUsersInfo>) {
     let start_time = std::time::Instant::now();
 
-    let to_downloads = stream::iter(users)
-        .map(|user| {
-            let shortcut_info = get_shortcuts_for_user(user);
-            async move {
-                start_search_for_to_download(settings, user, &shortcut_info.shortcuts)
-                    .await
-                    .unwrap_or(vec![])
-            }
-        })
-        .buffer_unordered(CONCURRENT_REQUESTS)
-        .collect::<Vec<Vec<ToDownload>>>()
-        .await;
-    let to_downloads = to_downloads.iter().flatten().collect::<Vec<&ToDownload>>();
-
-    stream::iter(to_downloads)
-        .map(|to_download| async move {
-            if let Err(e) = download_to_download(&to_download).await {
-                println!("Error downloading {:?}: {}", &to_download.path, e);
-            }
-        })
-        .buffer_unordered(CONCURRENT_REQUESTS)
-        .collect::<Vec<()>>()
-        .await;
-    let duration = start_time.elapsed();
-
-    println!("Finished getting images in: {:?}", duration);
-}
-
-async fn start_search_for_to_download(
-    settings: &Settings,
-    user: &crate::steam::SteamUsersInfo,
-    shortcut_info: &Vec<ShortcutOwned>,
-) -> Result<Vec<ToDownload>, Box<dyn Error>> {
     let auth_key = &settings.steamgrid_db.auth_key;
-
     if let Some(auth_key) = auth_key {
         println!("Checking for game images");
         let client = steamgriddb_api::Client::new(auth_key);
-        let mut search = CachedSearch::new(&client);
+        let search = CachedSearch::new(&client);
-        let known_images = get_users_images(user).unwrap();
+        let search = &search;
-        let res = search_fo_to_download(
+        let client = &client;
-            known_images,
+        let to_downloads = stream::iter(users)
-            user.steam_user_data_folder.as_str(),
+            .map(|user| {
-            shortcut_info,
+                let shortcut_info = get_shortcuts_for_user(user);
-            &mut search,
+                async move {
-            &client,
+                    let known_images = get_users_images(user).unwrap_or_default();
-        )
+                    let res = search_fo_to_download(
-        .await?;
+                        known_images,
+                        user.steam_user_data_folder.as_str(),
+                        &shortcut_info.shortcuts,
+                        search,
+                        client,
+                    )
+                    .await;
+                    res.unwrap_or_default()
+                }
+            })
+            .buffer_unordered(CONCURRENT_REQUESTS)
+            .collect::<Vec<Vec<ToDownload>>>()
+            .await;
+        let to_downloads = to_downloads.iter().flatten().collect::<Vec<&ToDownload>>();
         search.save();
-        Ok(res)
+
+        stream::iter(to_downloads)
+            .map(|to_download| async move {
+                if let Err(e) = download_to_download(&to_download).await {
+                    println!("Error downloading {:?}: {}", &to_download.path, e);
+                }
+            })
+            .buffer_unordered(CONCURRENT_REQUESTS)
+            .collect::<Vec<()>>()
+            .await;
+        let duration = start_time.elapsed();
+
+        println!("Finished getting images in: {:?}", duration);
     } else {
         println!("Steamgrid DB Auth Key not found, please add one as described here:  https://github.com/PhilipK/steam_shortcuts_sync#configuration");
-        Ok(Vec::new())
     }
 }
 
@@ -80,7 +70,7 @@ async fn search_fo_to_download<'b>(
     known_images: Vec<String>,
     user_data_folder: &str,
     shortcuts: &Vec<ShortcutOwned>,
-    search: &mut CachedSearch<'b>,
+    search: &CachedSearch<'b>,
     client: &Client,
 ) -> Result<Vec<ToDownload>, Box<dyn Error>> {
     let shortcuts_to_search_for = shortcuts.iter().filter(|s| {
@@ -96,10 +86,25 @@ async fn search_fo_to_download<'b>(
         return Ok(vec![]);
     }
     let mut search_results = HashMap::new();
-    for s in shortcuts_to_search_for {
+    let search_results_a = stream::iter(shortcuts_to_search_for)
-        let search = search.search(s.app_id, &s.app_name).await?;
+        .map(|s| async move {
-        if let Some(search) = search {
+            let search_result = search.search(s.app_id, &s.app_name).await;
-            search_results.insert(s.app_id, search);
+            if search_result.is_err() {
+                return None;
+            }
+            let search_result = search_result.unwrap();
+            if search_result.is_none() {
+                return None;
+            }
+            let search_result = search_result.unwrap();
+            Some((s.app_id, search_result))
+        })
+        .buffer_unordered(CONCURRENT_REQUESTS)
+        .collect::<Vec<Option<(u32, usize)>>>()
+        .await;
+    for r in search_results_a {
+        if let Some((app_id, search)) = r {
+            search_results.insert(app_id, search);
         }
     }
     let types = vec![ImageType::Logo, ImageType::Hero, ImageType::Grid];