diff --git a/Cargo.lock b/Cargo.lock index abfedf1..6fa0131 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "1.0.5" @@ -74,12 +85,86 @@ dependencies = [ "webpki-roots 0.22.6", ] +[[package]] +name = "attohttpc" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" +dependencies = [ + "http", + "log", + "rustls 0.20.9", + "serde", + "serde_json", + "url", + "webpki", + "webpki-roots 0.22.6", +] + +[[package]] +name = "attohttpc" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f77d243921b0979fbbd728dd2d5162e68ac8252976797c24eb5b3a6af9090dc" +dependencies = [ + "http", + "log", + "rustls 0.21.7", + "serde", + "serde_json", + "url", + "webpki-roots 0.25.2", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-creds" +version = "0.34.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5" +dependencies = [ + "attohttpc 0.22.0", + "dirs", + "log", + "quick-xml 0.26.0", + "rust-ini 0.18.0", + "serde", + "thiserror", + "time", + "url", +] + +[[package]] +name = "aws-creds" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390ad3b77f3e21e01a4a0355865853b681daf1988510b0b15e31c0c4ae7eb0f6" +dependencies = [ + "attohttpc 0.26.1", + "home", + "log", + "quick-xml 0.30.0", + "rust-ini 0.19.0", + "serde", + "thiserror", + "time", + "url", +] + +[[package]] +name = "aws-region" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42fed2b9fca70f2908268d057a607f2a906f47edbf856ea8587de9038d264e22" +dependencies = [ + "thiserror", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -187,6 +272,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "const-random" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11df32a13d7892ec42d51d3d175faba5211ffe13ed25d4fb348ac9e9ce835593" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -230,6 +335,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -271,6 +382,42 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", ] [[package]] @@ -357,6 +504,7 @@ checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -379,6 +527,17 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.28" @@ -458,9 +617,11 @@ name = "gold-pass-bot" version = "0.1.0" dependencies = [ "arc-swap", + "aws-creds 0.36.0", "chrono", "prometheus", "reqwest", + "rust-s3", "rust_xlsxwriter", "serde", "serde_json", @@ -494,6 +655,15 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" [[package]] name = "hashbrown" @@ -501,6 +671,30 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + [[package]] name = "http" version = "0.2.9" @@ -690,6 +884,23 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "maybe-async" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.6.3" @@ -837,6 +1048,26 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list 0.3.0", + "hashbrown 0.12.3", +] + +[[package]] +name = "ordered-multimap" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" +dependencies = [ + "dlv-list 0.5.2", + "hashbrown 0.13.2", +] + [[package]] name = "overload" version = "0.1.1" @@ -861,7 +1092,7 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.3.5", "smallvec", "windows-targets", ] @@ -926,6 +1157,26 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.33" @@ -965,6 +1216,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -974,6 +1234,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall 0.2.16", + "thiserror", +] + [[package]] name = "regex" version = "1.9.5" @@ -1063,6 +1334,58 @@ dependencies = [ "winapi", ] +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if", + "ordered-multimap 0.4.3", +] + +[[package]] +name = "rust-ini" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091" +dependencies = [ + "cfg-if", + "ordered-multimap 0.6.0", +] + +[[package]] +name = "rust-s3" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027" +dependencies = [ + "async-trait", + "aws-creds 0.34.1", + "aws-region", + "base64 0.13.1", + "bytes", + "cfg-if", + "futures", + "hex", + "hmac", + "http", + "log", + "maybe-async", + "md5", + "percent-encoding", + "quick-xml 0.26.0", + "reqwest", + "serde", + "serde_derive", + "sha2", + "thiserror", + "time", + "tokio", + "tokio-stream", + "url", +] + [[package]] name = "rust_xlsxwriter" version = "0.44.0" @@ -1289,6 +1612,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -1345,6 +1679,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "1.0.109" @@ -1375,7 +1715,7 @@ checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", + "redox_syscall 0.3.5", "rustix", "windows-sys", ] @@ -1438,6 +1778,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1511,6 +1860,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.8" diff --git a/Cargo.toml b/Cargo.toml index 07c58a6..3f8e443 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,3 +17,5 @@ tracing = { version = "0.1.37", features = ["async-await"] } tracing-subscriber = "0.3.17" rust_xlsxwriter = { version = "0.44" } prometheus = "0.13.3" +aws-creds = {version = "0.36.0", default_features = false, features = ["attohttpc", "http-credentials", "rustls-tls"]} +rust-s3 = { version = "0.33.0", default_features = false, features = ["reqwest", "tokio", "tokio-rustls-tls", "no-verify-ssl"] } diff --git a/README-md b/README-md new file mode 100644 index 0000000..183550f --- /dev/null +++ b/README-md @@ -0,0 +1 @@ +# Gold-Pass diff --git a/README.md b/README.md new file mode 100644 index 0000000..fe97a08 --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# Gold-Pass-Bot + +## Environment-Variables +* `S3_BUCKET` +* `S3_ACCESS_KEY` +* `S3_SECRET_KEY` diff --git a/src/main.rs b/src/main.rs index 30bbb2c..86524a4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,7 +7,8 @@ use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use gold_pass_bot::{ - ClanTag, ExcelStats, FileStorage, RaidMember, RaidWeekendStats, Season, Storage, StorageBackend, + ClanTag, ExcelStats, FileStorage, RaidMember, RaidWeekendStats, Replicated, S3Storage, Season, + Storage, StorageBackend, }; use serenity::async_trait; use serenity::framework::standard::macros::{command, group}; @@ -43,6 +44,10 @@ async fn main() { let store_path = std::env::var("STORE_PATH").unwrap_or_else(|_| "data.json".to_string()); let api_path = std::env::var("API_PATH").unwrap_or_else(|_| "api.key".to_string()); + let s3_bucket = std::env::var("S3_BUCKET").unwrap(); + let s3_access_key = std::env::var("S3_ACCESS_KEY").unwrap(); + let s3_secret_key = std::env::var("S3_SECRET_KEY").unwrap(); + #[cfg(not(debug_assertions))] let prefix = "!"; #[cfg(debug_assertions)] @@ -61,8 +66,27 @@ async fn main() { .await .expect("Error creating client"); - let mut storage_backend: Box = - Box::new(FileStorage::new(store_path.clone())); + let mut storage_backend: Box = Box::new(Replicated::new( + FileStorage::new(store_path.clone()), + S3Storage::new( + s3::Bucket::new( + &s3_bucket, + s3::Region::Custom { + region: "default".to_string(), + endpoint: "https://ceph-s3.service.aachen.dc.consul:7480".to_string(), + }, + s3::creds::Credentials::new( + Some(&s3_access_key), + Some(&s3_secret_key), + None, + None, + None, + ) + .unwrap(), + ) + .unwrap(), + ), + )); let elapsed = SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/src/storage.rs b/src/storage.rs index f56d742..0ccef1c 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -12,14 +12,18 @@ use crate::{ClanTag, ClanWarLeagueSeason, PlayerTag, Time}; mod files; pub use files::FileStorage; +mod s3; +pub use s3::S3Storage; + +mod replicated; +pub use replicated::Replicated; + pub trait StorageBackend: Send { fn write( &mut self, content: Vec, - ) -> Pin> + Send + Sync + 'static>>; - fn load( - &mut self, - ) -> Pin, ()>> + Send + Sync + 'static>>; + ) -> Pin> + Send + 'static>>; + fn load(&mut self) -> Pin, ()>> + Send + 'static>>; } #[derive(Debug, Clone, Deserialize, Serialize)] diff --git a/src/storage/files.rs b/src/storage/files.rs index 782319c..d119402 100644 --- a/src/storage/files.rs +++ b/src/storage/files.rs @@ -31,7 +31,7 @@ impl StorageBackend for FileStorage { fn write( &mut self, content: Vec, - ) -> Pin> + Send + Sync + 'static>> { + ) -> Pin> + Send + 'static>> { let path = self.path.clone(); Box::pin(async move { @@ -45,8 +45,7 @@ impl StorageBackend for FileStorage { fn load( &mut self, - ) -> Pin, ()>> + Send + Sync + 'static>> - { + ) -> Pin, ()>> + Send + 'static>> { let path = self.path.clone(); Box::pin(async move { tokio::fs::read(&path).await.map_err(|e| ()) }) diff --git a/src/storage/replicated.rs b/src/storage/replicated.rs new file mode 100644 index 0000000..6c93b3d --- /dev/null +++ b/src/storage/replicated.rs @@ -0,0 +1,49 @@ +use crate::StorageBackend; + +pub struct Replicated { + primary: P, + secondary: S, +} + +impl Replicated { + pub fn new(primary: P, secondary: S) -> Self { + Self { primary, secondary } + } +} + +impl StorageBackend for Replicated +where + P: StorageBackend, + S: StorageBackend, +{ + fn write( + &mut self, + content: Vec, + ) -> std::pin::Pin> + Send + 'static>> { + let pfut = self.primary.write(content.clone()); + let sfut = self.secondary.write(content); + + Box::pin(async move { + let pres = pfut.await; + let sres = sfut.await; + + pres.or(sres) + }) + } + + fn load( + &mut self, + ) -> std::pin::Pin, ()>> + Send + 'static>> + { + let pfut = self.primary.load(); + let sfut = self.secondary.load(); + + Box::pin(async move { + if let Ok(r) = pfut.await { + return Ok(r); + } + + sfut.await + }) + } +} diff --git a/src/storage/s3.rs b/src/storage/s3.rs new file mode 100644 index 0000000..02bffc9 --- /dev/null +++ b/src/storage/s3.rs @@ -0,0 +1,52 @@ +use crate::StorageBackend; + +pub struct S3Storage { + bucket: s3::Bucket, + filename: String, +} + +impl S3Storage { + pub fn new(bucket: s3::Bucket) -> Self { + Self { + bucket, + filename: "/storage.json".to_string(), + } + } +} + +impl StorageBackend for S3Storage { + fn write( + &mut self, + content: Vec, + ) -> std::pin::Pin> + Send + 'static>> { + let bucket = self.bucket.clone(); + let filename = self.filename.clone(); + + Box::pin(async move { + let filename = filename; + let content = content; + + let res = bucket.put_object(&filename, &content); + + match res.await { + Ok(_) => Ok(()), + Err(e) => Err(()), + } + }) + } + + fn load( + &mut self, + ) -> std::pin::Pin, ()>> + Send + 'static>> + { + let bucket = self.bucket.clone(); + let filename = self.filename.clone(); + + Box::pin(async move { + match bucket.get_object(filename).await { + Ok(c) => Ok(c.to_vec()), + Err(e) => Err(()), + } + }) + } +}