diff --git a/Cargo.lock b/Cargo.lock index fd74d4d..7e5fc20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,7 @@ name = "boilr" version = "0.2.7" dependencies = [ "config", + "dashmap", "failure", "fl2rust", "flate2", @@ -184,6 +185,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if", + "num_cpus", + "serde 1.0.130", +] + [[package]] name = "encoding_rs" version = "0.8.28" diff --git a/Cargo.toml b/Cargo.toml index 3d53122..d02ba2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,10 +20,11 @@ nom_locate = "^3.0.*" flate2 = "^1.0.22" toml = { version = "^0.5.8", optional = true } futures = { version = "^0.3.17" } +dashmap = { version = "^4.0.2", features = ["serde"] } [build-dependencies] fl2rust = { version = "0.4", optional = true } [features] # default = ["ui"] -ui = ["fltk", "fl2rust", "toml"] \ No newline at end of file +ui = ["fltk", "fl2rust", "toml"] diff --git a/src/steamgriddb/cached_search.rs b/src/steamgriddb/cached_search.rs index 0e86dd4..61afef8 100644 --- a/src/steamgriddb/cached_search.rs +++ b/src/steamgriddb/cached_search.rs @@ -1,6 +1,7 @@ -use std::{collections::HashMap, fs::File, io::Write, path::Path}; +use std::{ fs::File, io::Write, path::Path}; +use dashmap::DashMap; -type SearchMap = HashMap; +type SearchMap = DashMap; pub struct CachedSearch<'a> { search_map: SearchMap, @@ -20,7 +21,7 @@ impl<'a> CachedSearch<'a> { } pub async fn search( - &mut self, + &self, app_id: u32, query: S, ) -> Result, Box> diff --git a/src/steamgriddb/downloader.rs b/src/steamgriddb/downloader.rs index 9d466dd..f5000e7 100644 --- a/src/steamgriddb/downloader.rs +++ b/src/steamgriddb/downloader.rs @@ -20,59 +20,49 @@ const CONCURRENT_REQUESTS: usize = 10; pub async fn download_images_for_users<'b>(settings: &Settings, users: &Vec) { let start_time = std::time::Instant::now(); - let to_downloads = stream::iter(users) - .map(|user| { - let shortcut_info = get_shortcuts_for_user(user); - async move { - start_search_for_to_download(settings, user, &shortcut_info.shortcuts) - .await - .unwrap_or(vec![]) - } - }) - .buffer_unordered(CONCURRENT_REQUESTS) - .collect::>>() - .await; - let to_downloads = to_downloads.iter().flatten().collect::>(); - - stream::iter(to_downloads) - .map(|to_download| async move { - if let Err(e) = download_to_download(&to_download).await { - println!("Error downloading {:?}: {}", &to_download.path, e); - } - }) - .buffer_unordered(CONCURRENT_REQUESTS) - .collect::>() - .await; - let duration = start_time.elapsed(); - - println!("Finished getting images in: {:?}", duration); -} - -async fn start_search_for_to_download( - settings: &Settings, - user: &crate::steam::SteamUsersInfo, - shortcut_info: &Vec, -) -> Result, Box> { let auth_key = &settings.steamgrid_db.auth_key; - if let Some(auth_key) = auth_key { println!("Checking for game images"); let client = steamgriddb_api::Client::new(auth_key); - let mut search = CachedSearch::new(&client); - let known_images = get_users_images(user).unwrap(); - let res = search_fo_to_download( - known_images, - user.steam_user_data_folder.as_str(), - shortcut_info, - &mut search, - &client, - ) - .await?; + let search = CachedSearch::new(&client); + let search = &search; + let client = &client; + let to_downloads = stream::iter(users) + .map(|user| { + let shortcut_info = get_shortcuts_for_user(user); + async move { + let known_images = get_users_images(user).unwrap_or_default(); + let res = search_fo_to_download( + known_images, + user.steam_user_data_folder.as_str(), + &shortcut_info.shortcuts, + search, + client, + ) + .await; + res.unwrap_or_default() + } + }) + .buffer_unordered(CONCURRENT_REQUESTS) + .collect::>>() + .await; + let to_downloads = to_downloads.iter().flatten().collect::>(); search.save(); - Ok(res) + + stream::iter(to_downloads) + .map(|to_download| async move { + if let Err(e) = download_to_download(&to_download).await { + println!("Error downloading {:?}: {}", &to_download.path, e); + } + }) + .buffer_unordered(CONCURRENT_REQUESTS) + .collect::>() + .await; + let duration = start_time.elapsed(); + + println!("Finished getting images in: {:?}", duration); } else { println!("Steamgrid DB Auth Key not found, please add one as described here: https://github.com/PhilipK/steam_shortcuts_sync#configuration"); - Ok(Vec::new()) } } @@ -80,7 +70,7 @@ async fn search_fo_to_download<'b>( known_images: Vec, user_data_folder: &str, shortcuts: &Vec, - search: &mut CachedSearch<'b>, + search: &CachedSearch<'b>, client: &Client, ) -> Result, Box> { let shortcuts_to_search_for = shortcuts.iter().filter(|s| { @@ -96,10 +86,25 @@ async fn search_fo_to_download<'b>( return Ok(vec![]); } let mut search_results = HashMap::new(); - for s in shortcuts_to_search_for { - let search = search.search(s.app_id, &s.app_name).await?; - if let Some(search) = search { - search_results.insert(s.app_id, search); + let search_results_a = stream::iter(shortcuts_to_search_for) + .map(|s| async move { + let search_result = search.search(s.app_id, &s.app_name).await; + if search_result.is_err() { + return None; + } + let search_result = search_result.unwrap(); + if search_result.is_none() { + return None; + } + let search_result = search_result.unwrap(); + Some((s.app_id, search_result)) + }) + .buffer_unordered(CONCURRENT_REQUESTS) + .collect::>>() + .await; + for r in search_results_a { + if let Some((app_id, search)) = r { + search_results.insert(app_id, search); } } let types = vec![ImageType::Logo, ImageType::Hero, ImageType::Grid];