From 79ceb56c60a83a8aaff661e7f1fdd5fa6e763636 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 4 Apr 2023 20:25:17 -0700 Subject: [PATCH] Fix issues --- Dockerfile | 2 +- daedalus/Cargo.toml | 2 +- daedalus/src/lib.rs | 52 ++- daedalus/src/modded.rs | 19 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 456 +++++++++++---------- daedalus_client/src/forge.rs | 675 +++++++++++++++---------------- daedalus_client/src/main.rs | 85 +++- daedalus_client/src/minecraft.rs | 73 ++-- 9 files changed, 741 insertions(+), 625 deletions(-) diff --git a/Dockerfile b/Dockerfile index a63ddae..eae1ea3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.65 +FROM rust:1.68.2 COPY ./ ./ diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 3909d84..ada94ed 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.17" +version = "0.1.18" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 678bf1c..51221ee 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -46,22 +46,34 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { let name_items = artifact.split(':').collect::>(); let package = name_items.first().ok_or_else(|| { - Error::ParseError(format!("Unable to find package for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find package for library {}", + &artifact + )) })?; let name = name_items.get(1).ok_or_else(|| { - Error::ParseError(format!("Unable to find name for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find name for library {}", + &artifact + )) })?; if name_items.len() == 3 { let version_ext = name_items .get(2) .ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find version for library {}", + &artifact + )) })? .split('@') .collect::>(); let version = version_ext.first().ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find version for library {}", + &artifact + )) })?; let ext = version_ext.get(1); @@ -76,18 +88,27 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { )) } else { let version = name_items.get(2).ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find version for library {}", + &artifact + )) })?; let data_ext = name_items .get(3) .ok_or_else(|| { - Error::ParseError(format!("Unable to find data for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find data for library {}", + &artifact + )) })? .split('@') .collect::>(); let data = data_ext.first().ok_or_else(|| { - Error::ParseError(format!("Unable to find data for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find data for library {}", + &artifact + )) })?; let ext = data_ext.get(1); @@ -126,10 +147,21 @@ pub async fn download_file_mirrors( } /// Downloads a file with retry and checksum functionality -pub async fn download_file(url: &str, sha1: Option<&str>) -> Result { +pub async fn download_file( + url: &str, + sha1: Option<&str>, +) -> Result { + let mut headers = reqwest::header::HeaderMap::new(); + if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!( + "modrinth/daedalus/{} (support@modrinth.com)", + env!("CARGO_PKG_VERSION") + )) { + headers.insert(reqwest::header::USER_AGENT, header); + } let client = reqwest::Client::builder() .tcp_keepalive(Some(std::time::Duration::from_secs(10))) .timeout(std::time::Duration::from_secs(15)) + .default_headers(headers) .build() .map_err(|err| Error::FetchError { inner: err, @@ -183,7 +215,9 @@ pub async fn download_file(url: &str, sha1: Option<&str>) -> Result Result { - let hash = tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()).await?; + let hash = + tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()) + .await?; Ok(hash) } diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 2e62c98..2494ded 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -15,6 +15,9 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; +/// The dummy replace string library names, inheritsFrom, and version names should be replaced with +pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; + /// A data variable entry that depends on the side of the installation #[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] @@ -93,6 +96,8 @@ pub fn merge_partial_version( partial: PartialVersionInfo, merge: VersionInfo, ) -> VersionInfo { + let merge_id = merge.id.clone(); + VersionInfo { arguments: if let Some(partial_args) = partial.arguments { if let Some(merge_args) = merge.arguments { @@ -126,12 +131,22 @@ pub fn merge_partial_version( asset_index: merge.asset_index, assets: merge.assets, downloads: merge.downloads, - id: partial.id, + id: merge.id, java_version: merge.java_version, libraries: partial .libraries .into_iter() .chain(merge.libraries) + .map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name.replace(DUMMY_REPLACE_STRING, &merge_id), + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: x.include_in_classpath, + }) .collect::>(), main_class: if let Some(main_class) = partial.main_class { main_class @@ -163,6 +178,8 @@ pub struct Manifest { pub struct Version { /// The minecraft version ID pub id: String, + /// Whether the release is stable or not + pub stable: bool, /// A map that contains loader versions for the game version pub loaders: Vec, } diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 0355887..389c45a 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.17" +version = "0.1.18" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 3aef4fd..b40c31e 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -1,266 +1,284 @@ -use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::download_file; +use crate::{download_file, format_url, upload_file_to_bucket, Error}; use daedalus::minecraft::{Library, VersionManifest}; -use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Version}; -use log::info; +use daedalus::modded::{ + LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING, +}; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::{Mutex, RwLock, Semaphore}; pub async fn retrieve_data( minecraft_versions: &VersionManifest, uploaded_files: &mut Vec, + semaphore: Arc, ) -> Result<(), Error> { - let mut list = fetch_fabric_versions(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&format!( + let mut list = fetch_fabric_versions(None, semaphore.clone()).await?; + let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "fabric/v{}/manifest.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - )) + ))) .await .ok(); - let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + let mut versions = if let Some(old_manifest) = old_manifest { old_manifest.game_versions } else { Vec::new() - })); + }; - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + let loaders_mutex = RwLock::new(Vec::new()); - if let Some(latest) = list.loader.get(0) { - let loaders_mutex = Arc::new(RwLock::new(Vec::new())); - let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + { + let mut loaders = loaders_mutex.write().await; - { - let mut loaders = loaders_mutex.write().await; + for loader in &list.loader { + loaders.push((Box::new(loader.stable), loader.version.clone())) + } - // for loader in &list.loader { - // loaders.push((Box::new(loader.stable), loader.version.clone())) - // } + list.loader + .retain(|x| loaders.iter().any(|val| val.1 == x.version)) + } - loaders.push((Box::new(latest.stable), latest.version.clone())); + const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; - if !latest.stable { - if let Some(stable) = list.loader.iter().find(|x| x.stable) { - loaders.push((Box::new(stable.stable), stable.version.clone())); + let loader_version_mutex = Mutex::new(Vec::new()); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + + let loader_versions = futures::future::try_join_all( + loaders_mutex.read().await.clone().into_iter().map( + |(stable, loader)| async { + { + if versions.iter().any(|x| { + x.id == DUMMY_REPLACE_STRING + && x.loaders.iter().any(|x| x.id == loader) + }) { + return Ok(None); + } } - } - list.loader.retain(|x| loaders.iter().any(|val| val.1 == x.version)) - } + let version = fetch_fabric_version( + DUMMY_GAME_VERSION, + &loader, + semaphore.clone(), + ) + .await?; - let mut version_futures = Vec::new(); - - for game_version in list.game.iter_mut() { - let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); - let loaders_mutex = Arc::clone(&loaders_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - - let versions_mutex = Arc::clone(&versions); - version_futures.push(async move { - let loader_version_mutex = Mutex::new(Vec::new()); - - let versions = - futures::future::try_join_all( - loaders_mutex.read().await.clone().into_iter().map( - |(stable, loader)| async { - { - if versions_mutex.lock().await.iter().any(|x| { - x.id == game_version.version - && x.loaders.iter().any(|x| x.id == loader) - }) { - return Ok(None); - } - } - - let version = - fetch_fabric_version(&game_version.version, &loader).await?; - - Ok::, String, PartialVersionInfo)>, Error>(Some( - (stable, loader, version), - )) - }, - ), - ) - .await? - .into_iter() - .flatten(); - - futures::future::try_join_all(versions.map(|(stable, loader, version)| async { - let libs = futures::future::try_join_all(version.libraries.into_iter().map( - |mut lib| async { - { - let mut visited_assets = visited_artifacts_mutex.lock().await; - - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - let artifact_path = daedalus::get_path_from_artifact(&lib.name)?; - - let artifact = daedalus::download_file( - &format!( - "{}{}", - lib.url.unwrap_or_else(|| { - "https://maven.fabricmc.net/".to_string() - }), - artifact_path - ), - None, - ) - .await?; + //println!("{}", loader); + Ok::, String, PartialVersionInfo)>, Error>( + Some((stable, loader, version)), + ) + }, + ), + ) + .await?; + + let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + futures::future::try_join_all(loader_versions.into_iter() + .flatten().map( + |(stable, loader, version)| async { + let libs = futures::future::try_join_all( + version.libraries.into_iter().map(|mut lib| async { + { + let mut visited_assets = + visited_artifacts_mutex.lock().await; + if visited_assets.contains(&lib.name) { lib.url = Some(format_url("maven/")); - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - ) - .await?; - - Ok::(lib) - }, - )) - .await?; - - let version_path = format!( - "fabric/v{}/versions/{}-{}.json", - daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - version.inherits_from, - &loader - ); - - let inherits_from = version.inherits_from.clone(); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&PartialVersionInfo { - arguments: version.arguments, - id: version.id, - main_class: version.main_class, - release_time: version.release_time, - time: version.time, - type_: version.type_, - inherits_from: version.inherits_from, - libraries: libs, - minecraft_arguments: version.minecraft_arguments, - processors: None, - data: None, - })?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - ) - .await?; - - { - let mut loader_version_map = loader_version_mutex.lock().await; - async move { - loader_version_map.push(LoaderVersion { - id: format!("{}-{}", inherits_from, loader), - url: format_url(&version_path), - stable: *stable, - }); + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) } - .await; } - Ok::<(), Error>(()) - })) - .await?; + if lib.name.contains(DUMMY_GAME_VERSION) { + lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + lib.url = Some(format_url("maven/")); + futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { + let semaphore = semaphore.clone(); + let uploaded_files_mutex = uploaded_files_mutex.clone(); + let lib_name = lib.name.clone(); + let lib_url = lib.url.clone(); + + async move { + let artifact_path = + daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?; + + let artifact = download_file( + &format!( + "{}{}", + lib_url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + Ok::<(), Error>(()) + }.await?; + + Ok::<(), Error>(()) + })).await?; + + return Ok(lib); + } - let mut versions = versions_mutex.lock().await; - versions.push(Version { - id: game_version.version.clone(), - loaders: loader_version_mutex.into_inner(), - }); + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; - Ok::<(), Error>(()) - }); - } + let artifact = download_file( + &format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); + lib.url = Some(format_url("maven/")); - let chunk: Vec<_> = versions.by_ref().take(10).collect(); - futures::future::try_join_all(chunk).await?; + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; - tokio::time::sleep(Duration::from_secs(1)).await; + Ok::(lib) + }), + ) + .await?; - chunk_index += 1; + let version_path = format!( + "fabric/v{}/versions/{}.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + &loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version + .id + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version + .inherits_from + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + libraries: libs, + minecraft_arguments: version.minecraft_arguments, + processors: None, + data: None, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + { + let mut loader_version_map = loader_version_mutex.lock().await; + async move { + loader_version_map.push(LoaderVersion { + id: loader.to_string(), + url: format_url(&version_path), + stable: *stable, + }); + } + .await; + } - let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); - } + Ok::<(), Error>(()) + }, + )) + .await?; + + versions.push(Version { + id: DUMMY_REPLACE_STRING.to_string(), + stable: true, + loaders: loader_version_mutex.into_inner(), + }); + + for version in &list.game { + versions.push(Version { + id: version.version.clone(), + stable: version.stable, + loaders: vec![], + }); } - if let Ok(versions) = Arc::try_unwrap(versions) { - let mut versions = versions.into_inner(); - - versions.sort_by(|x, y| { - minecraft_versions - .versions + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); + + for version in &mut versions { + version.loaders.sort_by(|x, y| { + list.loader .iter() - .position(|z| x.id == z.id) + .position(|z| { + x.id.split('-').next().unwrap_or_default() == &*z.version + }) .unwrap_or_default() .cmp( - &minecraft_versions - .versions + &list + .loader .iter() - .position(|z| y.id == z.id) + .position(|z| { + y.id.split('-').next().unwrap_or_default() + == z.version + }) .unwrap_or_default(), ) - }); - - for version in &mut versions { - version.loaders.sort_by(|x, y| { - list.loader - .iter() - .position(|z| { - x.id.split('-') - .next() - .unwrap_or_default() - == &*z.version - }) - .unwrap_or_default() - .cmp( - &list - .loader - .iter() - .position(|z| { - y.id.split('-') - .next() - .unwrap_or_default() - == z.version - }) - .unwrap_or_default(), - ) - }) - } - - upload_file_to_bucket( - format!( - "fabric/v{}/manifest.json", - daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - ), - serde_json::to_vec(&Manifest { - game_versions: versions, - })?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - ) - .await?; + }) } + upload_file_to_bucket( + format!( + "fabric/v{}/manifest.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore, + ) + .await?; + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { uploaded_files.extend(uploaded_files_mutex.into_inner()); } @@ -273,6 +291,7 @@ const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; async fn fetch_fabric_version( version_number: &str, loader_version: &str, + semaphore: Arc, ) -> Result { Ok(serde_json::from_slice( &download_file( @@ -281,6 +300,7 @@ async fn fetch_fabric_version( FABRIC_META_URL, version_number, loader_version ), None, + semaphore, ) .await?, )?) @@ -319,11 +339,15 @@ struct FabricLoaderVersion { pub stable: bool, } /// Fetches the list of fabric versions -async fn fetch_fabric_versions(url: Option<&str>) -> Result { +async fn fetch_fabric_versions( + url: Option<&str>, + semaphore: Arc, +) -> Result { Ok(serde_json::from_slice( &download_file( url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), None, + semaphore, ) .await?, )?) diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 25da023..cb903f1 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,8 +1,14 @@ -use crate::{format_url, upload_file_to_bucket, Error}; +use crate::{ + download_file, download_file_mirrors, format_url, upload_file_to_bucket, + Error, +}; use chrono::{DateTime, Utc}; -use daedalus::download_file; -use daedalus::minecraft::{Argument, ArgumentType, Library, VersionManifest, VersionType}; -use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry}; +use daedalus::minecraft::{ + Argument, ArgumentType, Library, VersionManifest, VersionType, +}; +use daedalus::modded::{ + LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, +}; use lazy_static::lazy_static; use log::info; use semver::{Version, VersionReq}; @@ -10,8 +16,8 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io::Read; use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::Mutex; +use std::time::Instant; +use tokio::sync::{Mutex, Semaphore}; lazy_static! { static ref FORGE_MANIFEST_V1_QUERY: VersionReq = @@ -20,14 +26,16 @@ lazy_static! { VersionReq::parse(">=23.5.2851, <31.2.52").unwrap(); static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq = VersionReq::parse(">=32.0.1, <37.0.0").unwrap(); - static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); + static ref FORGE_MANIFEST_V3_QUERY: VersionReq = + VersionReq::parse(">=37.0.0").unwrap(); } pub async fn retrieve_data( minecraft_versions: &VersionManifest, uploaded_files: &mut Vec, + semaphore: Arc, ) -> Result<(), Error> { - let maven_metadata = fetch_maven_metadata(None).await?; + let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?; let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "forge/v{}/manifest.json", daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, @@ -35,11 +43,12 @@ pub async fn retrieve_data( .await .ok(); - let old_versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { - old_manifest.game_versions - } else { - Vec::new() - })); + let old_versions = + Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + })); let versions = Arc::new(Mutex::new(Vec::new())); @@ -52,13 +61,14 @@ pub async fn retrieve_data( let mut loaders = Vec::new(); for loader_version_full in loader_versions { - let loader_version = loader_version_full.split('-').into_iter().nth(1); + let loader_version = loader_version_full.split('-').nth(1); if let Some(loader_version_raw) = loader_version { // This is a dirty hack to get around Forge not complying with SemVer, but whatever // Most of this is a hack anyways :( // Works for all forge versions! - let split = loader_version_raw.split('.').collect::>(); + let split = + loader_version_raw.split('.').collect::>(); let loader_version = if split.len() >= 4 { if split[0].parse::().unwrap_or(0) < 6 { format!("{}.{}.{}", split[0], split[1], split[3]) @@ -80,203 +90,209 @@ pub async fn retrieve_data( } } } - version_futures.push(async { - let mut loaders_versions = Vec::new(); - - { - let mut loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { - let versions_mutex = Arc::clone(&old_versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - let minecraft_version = minecraft_version.clone(); - - async move { - /// These forge versions are not worth supporting! - const WHITELIST : [&str; 1] = [ - // Not supported due to `data` field being `[]` even though the type is a map - "1.12.2-14.23.5.2851" - ]; - - if WHITELIST.contains(&&*loader_version_full) { - return Ok(None); - } - - { - let versions = versions_mutex.lock().await; - let version = versions.iter().find(|x| - x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); - - if let Some(version) = version { - return Ok::, Error>(Some(version.clone())); + + if !loaders.is_empty() { + version_futures.push(async { + let loaders_versions = Vec::new(); + + { + let loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { + let versions_mutex = Arc::clone(&old_versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); + let semaphore = Arc::clone(&semaphore); + let minecraft_version = minecraft_version.clone(); + + async move { + /// These forge versions are not worth supporting! + const WHITELIST : [&str; 1] = [ + // Not supported due to `data` field being `[]` even though the type is a map + "1.12.2-14.23.5.2851", + ]; + + if WHITELIST.contains(&&*loader_version_full) { + return Ok(None); } - } - info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + { + let versions = versions_mutex.lock().await; + let version = versions.iter().find(|x| + x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); - let reader = std::io::Cursor::new(bytes); + if let Some(version) = version { + return Ok::, Error>(Some(version.clone())); + } + } - if let Ok(archive) = zip::ZipArchive::new(reader) { - if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; + info!("Forge - Installer Start {}", loader_version_full.clone()); + let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?; - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; + let reader = std::io::Cursor::new(bytes); - Ok::(serde_json::from_str::(&contents)?) - }).await??; + if let Ok(archive) = zip::ZipArchive::new(reader) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) { + let mut archive_clone = archive.clone(); + let profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; - let mut archive_clone = archive.clone(); - let file_path = profile.install.file_path.clone(); - let forge_universal_bytes = tokio::task::spawn_blocking(move || { - let mut forge_universal_file = archive_clone.by_name(&file_path)?; - let mut forge_universal = Vec::new(); - forge_universal_file.read_to_end(&mut forge_universal)?; + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + Ok::(serde_json::from_str::(&contents)?) + }).await??; - Ok::(bytes::Bytes::from(forge_universal)) - }).await??; - let forge_universal_path = profile.install.path.clone(); + let mut archive_clone = archive.clone(); + let file_path = profile.install.file_path.clone(); + let forge_universal_bytes = tokio::task::spawn_blocking(move || { + let mut forge_universal_file = archive_clone.by_name(&file_path)?; + let mut forge_universal = Vec::new(); + forge_universal_file.read_to_end(&mut forge_universal)?; - let now = Instant::now(); - let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { - if let Some(url) = lib.url { - { - let mut visited_assets = visited_assets.lock().await; - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); + Ok::(bytes::Bytes::from(forge_universal)) + }).await??; + let forge_universal_path = profile.install.path.clone(); - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { + if let Some(url) = lib.url { + { + let mut visited_assets = visited_assets.lock().await; + + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } } + + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; + + let artifact = if lib.name == forge_universal_path { + forge_universal_bytes.clone() + } else { + let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; + + download_file_mirrors( + &artifact_path, + &mirrors, + None, + semaphore.clone(), + ) + .await? + }; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; } - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: profile.version_info.id, + inherits_from: profile.install.minecraft, + release_time: profile.version_info.release_time, + time: profile.version_info.time, + main_class: profile.version_info.main_class, + minecraft_arguments: profile.version_info.minecraft_arguments.clone(), + arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), + libraries: libs, + type_: profile.version_info.type_, + data: None, + processors: None + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; + + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&version_path), + stable: false + })); + } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + let mut archive_clone = archive.clone(); + let mut profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&contents)?) + }).await??; + + let mut archive_clone = archive.clone(); + let version_info = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("version.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&contents)?) + }).await??; + + + let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name, + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: false + })).collect(); + + let mut local_libs : HashMap = HashMap::new(); + + for lib in &libs { + if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { + let mut archive_clone = archive.clone(); + let lib_name_clone = lib.name.clone(); - let artifact = if lib.name == forge_universal_path { - forge_universal_bytes.clone() - } else { - let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; - - daedalus::download_file_mirrors( - &artifact_path, - &mirrors, - None, - ) - .await? - }; - - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - ).await?; - } + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: profile.version_info.id, - inherits_from: profile.install.minecraft, - release_time: profile.version_info.release_time, - time: profile.version_info.time, - main_class: profile.version_info.main_class, - minecraft_arguments: profile.version_info.minecraft_arguments.clone(), - arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), - libraries: libs, - type_: profile.version_info.type_, - data: None, - processors: None - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); - } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let mut profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - let mut archive_clone = archive.clone(); - let version_info = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("version.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - - let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { - downloads: x.downloads, - extract: x.extract, - name: x.name, - url: x.url, - natives: x.natives, - rules: x.rules, - checksums: x.checksums, - include_in_classpath: false - })).collect(); - - let mut local_libs : HashMap = HashMap::new(); - - for lib in &libs { - if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { - let mut archive_clone = archive.clone(); - let lib_name_clone = lib.name.clone(); - - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - local_libs.insert(lib.name.clone(), lib_bytes); + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + local_libs.insert(lib.name.clone(), lib_bytes); + } } - } - let path = profile.path.clone(); - let version = profile.version.clone(); + let path = profile.path.clone(); + let version = profile.version.clone(); - for entry in profile.data.values_mut() { - if entry.client.starts_with('/') || entry.server.starts_with('/') { - macro_rules! read_data { + for entry in profile.data.values_mut() { + if entry.client.starts_with('/') || entry.server.starts_with('/') { + macro_rules! read_data { ($value:expr) => { let mut archive_clone = archive.clone(); let value_clone = $value.clone(); @@ -315,179 +331,149 @@ pub async fn retrieve_data( } } - if entry.client.starts_with('/') { - read_data!(entry.client); - } + if entry.client.starts_with('/') { + read_data!(entry.client); + } - // Do we really need to support server installs? Keeping this here - // just in case - // - // if entry.server.starts_with('/') { - // read_data!(entry.server); - // } + if entry.server.starts_with('/') { + read_data!(entry.server); + } + } } - } - let now = Instant::now(); - let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; + let now = Instant::now(); + let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; - { - let mut visited_assets = visited_assets.lock().await; + { + let mut visited_assets = visited_assets.lock().await; - if visited_assets.contains(&lib.name) { - if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&format!("maven/{}", artifact_path)); + if visited_assets.contains(&lib.name) { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + artifact.url = format_url(&format!("maven/{}", artifact_path)); + } + } else if lib.url.is_some() { + lib.url = Some(format_url("maven/")); } - } else if lib.url.is_some() { - lib.url = Some(format_url("maven/")); - } - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } } - } - let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - let res = if artifact.url.is_empty() { + let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + let res = if artifact.url.is_empty() { + local_libs.get(&lib.name).cloned() + } else { + Some(download_file( + &artifact.url, + Some(&*artifact.sha1), + semaphore.clone(), + ) + .await?) + }; + + if res.is_some() { + artifact.url = format_url(&format!("maven/{}", artifact_path)); + } + + res + } else { None } + } else if let Some(ref mut url) = lib.url { + let res = if url.is_empty() { local_libs.get(&lib.name).cloned() } else { - Some(daedalus::download_file( - &artifact.url, - Some(&*artifact.sha1), + Some(download_file( + url, + None, + semaphore.clone(), ) .await?) }; if res.is_some() { - artifact.url = format_url(&format!("maven/{}", artifact_path)); + lib.url = Some(format_url("maven/")); } res - } else { None } - } else if let Some(ref mut url) = lib.url { - let res = if url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(daedalus::download_file( - url, - None, - ) - .await?) - }; - - if res.is_some() { - lib.url = Some(format_url("maven/")); + } else { None }; + + if let Some(bytes) = artifact_bytes { + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + bytes.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; } - res - } else { None }; - - if let Some(bytes) = artifact_bytes { - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - bytes.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - } - - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: version_info.id, - inherits_from: version_info.inherits_from, - release_time: version_info.release_time, - time: version_info.time, - main_class: version_info.main_class, - minecraft_arguments: version_info.minecraft_arguments, - arguments: version_info.arguments, - libraries: libs, - type_: version_info.type_, - data: Some(profile.data), - processors: Some(profile.processors), - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: version_info.id, + inherits_from: version_info.inherits_from, + release_time: version_info.release_time, + time: version_info.time, + main_class: version_info.main_class, + minecraft_arguments: version_info.minecraft_arguments, + arguments: version_info.arguments, + libraries: libs, + type_: version_info.type_, + data: Some(profile.data), + processors: Some(profile.processors), + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; + + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&version_path), + stable: false + })); + } } - } - - Ok(None) - }.await - }).into_iter().peekable()/*.into_iter().flatten().collect()*/; - - let mut chunk_index = 0; - while loaders_futures.peek().is_some() { - info!("Loader Chunk {} Start", chunk_index); - let now = Instant::now(); - - let chunk: Vec<_> = loaders_futures.by_ref().take(10).collect(); - - let res = futures::future::try_join_all(chunk).await?; - loaders_versions.extend(res.into_iter().flatten()); - tokio::time::sleep(Duration::from_secs(1)).await; + Ok(None) + }.await + }); - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Loader Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + futures::future::try_join_all(loaders_futures).await?; } - } - - versions.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - loaders: loaders_versions - }); - - Ok::<(), Error>(()) - }); - } - - { - let mut versions_peek = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions_peek.peek().is_some() { - info!("Chunk {} Start", chunk_index); - let now = Instant::now(); - - let chunk: Vec<_> = versions_peek.by_ref().take(1).collect(); - futures::future::try_join_all(chunk).await?; - - tokio::time::sleep(Duration::from_secs(1)).await; - chunk_index += 1; + versions.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + stable: true, + loaders: loaders_versions + }); - let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + Ok::<(), Error>(()) + }); } } + futures::future::try_join_all(version_futures).await?; + if let Ok(versions) = Arc::try_unwrap(versions) { let mut versions = versions.into_inner(); @@ -495,13 +481,17 @@ pub async fn retrieve_data( minecraft_versions .versions .iter() - .position(|z| x.id == z.id) + .position(|z| { + x.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id + }) .unwrap_or_default() .cmp( &minecraft_versions .versions .iter() - .position(|z| y.id == z.id) + .position(|z| { + y.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id + }) .unwrap_or_default(), ) }); @@ -524,6 +514,8 @@ pub async fn retrieve_data( } } + println!("{:?}", versions); + upload_file_to_bucket( format!( "forge/v{}/manifest.json", @@ -534,6 +526,7 @@ pub async fn retrieve_data( })?, Some("application/json".to_string()), uploaded_files_mutex.as_ref(), + semaphore, ) .await?; } @@ -554,9 +547,15 @@ const DEFAULT_MAVEN_METADATA_URL: &str = /// the specified Minecraft version pub async fn fetch_maven_metadata( url: Option<&str>, + semaphore: Arc, ) -> Result>, Error> { Ok(serde_json::from_slice( - &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?, + &download_file( + url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), + None, + semaphore, + ) + .await?, )?) } diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 70a4fa6..cefa4e1 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,8 +1,10 @@ use log::{error, warn}; -use std::time::Duration; -use s3::{Bucket, Region}; use s3::creds::Credentials; use s3::error::S3Error; +use s3::{Bucket, Region}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Semaphore; mod fabric; mod forge; @@ -19,10 +21,7 @@ pub enum Error { #[error("Error while managing asynchronous tasks")] TaskError(#[from] tokio::task::JoinError), #[error("Error while uploading file to S3")] - S3Error { - inner: S3Error, - file: String, - }, + S3Error { inner: S3Error, file: String }, #[error("Error while parsing version as semver: {0}")] SemVerError(#[from] semver::Error), #[error("Error while reading zip file: {0}")] @@ -31,6 +30,8 @@ pub enum Error { IoError(#[from] std::io::Error), #[error("Error while obtaining strong reference to Arc")] ArcError, + #[error("Error acquiring semaphore: {0}")] + AcquireError(#[from] tokio::sync::AcquireError), } #[tokio::main] @@ -43,14 +44,20 @@ async fn main() { return; } - let mut timer = tokio::time::interval(Duration::from_secs(10 * 60)); + let mut timer = tokio::time::interval(Duration::from_secs(30 * 60)); + let semaphore = Arc::new(Semaphore::new(50)); loop { timer.tick().await; let mut uploaded_files = Vec::new(); - let versions = match minecraft::retrieve_data(&mut uploaded_files).await { + let versions = match minecraft::retrieve_data( + &mut uploaded_files, + semaphore.clone(), + ) + .await + { Ok(res) => Some(res), Err(err) => { error!("{:?}", err); @@ -60,15 +67,22 @@ async fn main() { }; if let Some(manifest) = versions { - match fabric::retrieve_data(&manifest, &mut uploaded_files).await { + match fabric::retrieve_data( + &manifest, + &mut uploaded_files, + semaphore.clone(), + ) + .await + { Ok(..) => {} Err(err) => error!("{:?}", err), }; - match forge::retrieve_data(&manifest, &mut uploaded_files).await { + match forge::retrieve_data(&manifest, &mut uploaded_files, semaphore.clone()).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; } + break; } } @@ -104,7 +118,6 @@ fn check_env_vars() -> bool { failed } - lazy_static::lazy_static! { static ref CLIENT : Bucket = Bucket::new( &dotenvy::var("S3_BUCKET_NAME").unwrap(), @@ -133,24 +146,26 @@ pub async fn upload_file_to_bucket( bytes: Vec, content_type: Option, uploaded_files: &tokio::sync::Mutex>, + semaphore: Arc, ) -> Result<(), Error> { + let _permit = semaphore.acquire().await?; let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path); for attempt in 1..=4 { let result = if let Some(ref content_type) = content_type { - CLIENT.put_object_with_content_type( - key.clone(), - &bytes, - content_type, - ).await + CLIENT + .put_object_with_content_type(key.clone(), &bytes, content_type) + .await } else { - CLIENT.put_object( - key.clone(), - &bytes, - ).await - }.map_err(|err| Error::S3Error { + CLIENT.put_object(key.clone(), &bytes).await + } + .map_err(|err| Error::S3Error { inner: err, - file: format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path), + file: format!( + "{}/{}", + &*dotenvy::var("BASE_FOLDER").unwrap(), + path + ), }); match result { @@ -180,3 +195,29 @@ pub fn format_url(path: &str) -> String { path ) } + +pub async fn download_file( + url: &str, + sha1: Option<&str>, + semaphore: Arc, +) -> Result { + let _permit = semaphore.acquire().await?; + println!("{} url start", url); + + let val = daedalus::download_file(url, sha1).await?; + println!("{} url end", url); + Ok(val) +} + +pub async fn download_file_mirrors( + base: &str, + mirrors: &[&str], + sha1: Option<&str>, + semaphore: Arc, +) -> Result { + let _permit = semaphore.acquire().await?; + + let val = daedalus::download_file_mirrors(base, mirrors, sha1).await?; + + Ok(val) +} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 56007b9..256eef7 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,21 +1,26 @@ +use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::download_file; use daedalus::minecraft::VersionManifest; use log::info; use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::Mutex; - -pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result { - let old_manifest = - daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&format!( +use std::time::Instant; +use tokio::sync::{Mutex, Semaphore}; + +pub async fn retrieve_data( + uploaded_files: &mut Vec, + semaphore: Arc, +) -> Result { + let old_manifest = daedalus::minecraft::fetch_version_manifest(Some( + &*format_url(&format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION - )))) - .await - .ok(); + )), + )) + .await + .ok(); - let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?; + let mut manifest = + daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); @@ -42,13 +47,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result = versions.by_ref().take(100).collect(); - futures::future::try_join_all(chunk).await?; - - tokio::time::sleep(Duration::from_secs(1)).await; - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); - } - } + futures::future::try_join_all(version_futures).await?; upload_file_to_bucket( format!( @@ -157,6 +157,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result