From 701d17658cd2a21d1bf89ec43fb1c22f1f26c9a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Mon, 10 Jun 2024 16:05:42 -0300 Subject: [PATCH 01/26] Implement simple stress binary --- src/bin/cairo-native-stress.rs | 85 ++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 src/bin/cairo-native-stress.rs diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs new file mode 100644 index 000000000..a0a7d216b --- /dev/null +++ b/src/bin/cairo-native-stress.rs @@ -0,0 +1,85 @@ +use std::fs; + +use cairo_lang_sierra::program_registry::ProgramRegistry; +use cairo_lang_starknet::compile::compile_path; +use cairo_native::{ + context::NativeContext, + executor::AotNativeExecutor, + metadata::gas::GasMetadata, + module_to_object, object_to_shared_lib, + starknet::DummySyscallHandler, + utils::{find_entry_point_by_idx, SHARED_LIBRARY_EXT}, +}; +use libloading::Library; + +fn main() { + let program = generate_program("Name", 252); + + let native_context = NativeContext::new(); + let native_module = native_context + .compile(&program, None) + .expect("should compile"); + + let object_data = + module_to_object(native_module.module(), cairo_native::OptLevel::None).unwrap(); + let shared_library_path = tempfile::Builder::new() + .prefix("lib") + .suffix(SHARED_LIBRARY_EXT) + .tempfile() + .unwrap() + .into_temp_path(); + object_to_shared_lib(&object_data, &shared_library_path).unwrap(); + let shared_library = unsafe { Library::new(shared_library_path).unwrap() }; + + let registry = ProgramRegistry::new(&program).unwrap(); + + let executor = AotNativeExecutor::new( + shared_library, + registry, + native_module + .metadata() + .get::() + .cloned() + .unwrap(), + ); + + let entry_point_id = &find_entry_point_by_idx(&program, 0).unwrap().id; + let execution_result = executor + .invoke_contract_dynamic(entry_point_id, &[], Some(u128::MAX), DummySyscallHandler) + .unwrap(); + + assert!( + execution_result.failure_flag == false, + "contract execution failed" + ) +} + +fn generate_program(name: &str, output: u32) -> cairo_lang_sierra::program::Program { + let program_str = format!( + "\ +#[starknet::contract] +mod {name} {{ + #[storage] + struct Storage {{}} + + #[external(v0)] + fn main(self: @ContractState) -> felt252 {{ + return {output}; + }} +}} +" + ); + + let mut program_file = tempfile::Builder::new() + .prefix("test_") + .suffix(".cairo") + .tempfile() + .unwrap(); + fs::write(&mut program_file, program_str).unwrap(); + + let contract_class = compile_path(program_file.path(), None, Default::default()).unwrap(); + + let program = contract_class.extract_sierra_program().unwrap(); + + program +} From ab72b7c53fe8bb902e2a57d932705e93aa46d27d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Mon, 10 Jun 2024 16:33:54 -0300 Subject: [PATCH 02/26] Add loop with cache and clap args --- src/bin/cairo-native-stress.rs | 83 ++++++++++++++++------------------ 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index a0a7d216b..40bf31a3c 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -1,60 +1,51 @@ use std::fs; -use cairo_lang_sierra::program_registry::ProgramRegistry; +use cairo_lang_sierra::ids::FunctionId; use cairo_lang_starknet::compile::compile_path; use cairo_native::{ - context::NativeContext, - executor::AotNativeExecutor, - metadata::gas::GasMetadata, - module_to_object, object_to_shared_lib, - starknet::DummySyscallHandler, - utils::{find_entry_point_by_idx, SHARED_LIBRARY_EXT}, + cache::AotProgramCache, context::NativeContext, starknet::DummySyscallHandler, + utils::find_entry_point_by_idx, }; -use libloading::Library; +use clap::Parser; + +#[derive(Parser, Debug)] +struct CliArgs { + /// Amount of iterations to perform + iterations: u32, +} fn main() { - let program = generate_program("Name", 252); + let cli_args = CliArgs::parse(); let native_context = NativeContext::new(); - let native_module = native_context - .compile(&program, None) - .expect("should compile"); - - let object_data = - module_to_object(native_module.module(), cairo_native::OptLevel::None).unwrap(); - let shared_library_path = tempfile::Builder::new() - .prefix("lib") - .suffix(SHARED_LIBRARY_EXT) - .tempfile() - .unwrap() - .into_temp_path(); - object_to_shared_lib(&object_data, &shared_library_path).unwrap(); - let shared_library = unsafe { Library::new(shared_library_path).unwrap() }; - - let registry = ProgramRegistry::new(&program).unwrap(); - - let executor = AotNativeExecutor::new( - shared_library, - registry, - native_module - .metadata() - .get::() - .cloned() - .unwrap(), - ); + let mut cache = AotProgramCache::new(&native_context); - let entry_point_id = &find_entry_point_by_idx(&program, 0).unwrap().id; - let execution_result = executor - .invoke_contract_dynamic(entry_point_id, &[], Some(u128::MAX), DummySyscallHandler) - .unwrap(); + for round in 0..cli_args.iterations { + let (entry_point_id, program) = generate_program("Name", round); - assert!( - execution_result.failure_flag == false, - "contract execution failed" - ) + if cache.get(&round).is_some() { + panic!("encountered cache hit, all contracts must be different") + } + + let executor = cache.compile_and_insert(round, &program, cairo_native::OptLevel::None); + + let execution_result = executor + .invoke_contract_dynamic(&entry_point_id, &[], Some(u128::MAX), DummySyscallHandler) + .unwrap(); + + assert!( + execution_result.failure_flag == false, + "contract execution failed" + ); + + println!( + "Finished round {round} with result {}", + execution_result.return_values[0] + ); + } } -fn generate_program(name: &str, output: u32) -> cairo_lang_sierra::program::Program { +fn generate_program(name: &str, output: u32) -> (FunctionId, cairo_lang_sierra::program::Program) { let program_str = format!( "\ #[starknet::contract] @@ -81,5 +72,7 @@ mod {name} {{ let program = contract_class.extract_sierra_program().unwrap(); - program + let entry_point_id = find_entry_point_by_idx(&program, 0).unwrap().id.clone(); + + (entry_point_id, program) } From 1a98545fbb9b68f0ff2a2e278bd61ed3e6755814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Mon, 10 Jun 2024 16:37:01 -0300 Subject: [PATCH 03/26] Add expect instead of unwrap --- src/bin/cairo-native-stress.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 40bf31a3c..286ab8456 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -31,7 +31,7 @@ fn main() { let execution_result = executor .invoke_contract_dynamic(&entry_point_id, &[], Some(u128::MAX), DummySyscallHandler) - .unwrap(); + .expect("contract execution failed"); assert!( execution_result.failure_flag == false, @@ -65,14 +65,20 @@ mod {name} {{ .prefix("test_") .suffix(".cairo") .tempfile() - .unwrap(); - fs::write(&mut program_file, program_str).unwrap(); + .expect("temporary file creation failed"); + fs::write(&mut program_file, program_str).expect("writing to temporary file failed"); - let contract_class = compile_path(program_file.path(), None, Default::default()).unwrap(); + let contract_class = compile_path(program_file.path(), None, Default::default()) + .expect("compiling contract failed"); - let program = contract_class.extract_sierra_program().unwrap(); + let program = contract_class + .extract_sierra_program() + .expect("extracting sierra failed"); - let entry_point_id = find_entry_point_by_idx(&program, 0).unwrap().id.clone(); + let entry_point_id = find_entry_point_by_idx(&program, 0) + .expect("cairo file should have an entry point") + .id + .clone(); (entry_point_id, program) } From 5d270049c728a1b6c852291a8b1b5da3d935ecae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Mon, 10 Jun 2024 16:42:20 -0300 Subject: [PATCH 04/26] Improve error handling --- src/bin/cairo-native-stress.rs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 286ab8456..e98dab492 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -21,21 +21,21 @@ fn main() { let mut cache = AotProgramCache::new(&native_context); for round in 0..cli_args.iterations { - let (entry_point_id, program) = generate_program("Name", round); + let (entry_point, program) = generate_program("Name", round); if cache.get(&round).is_some() { - panic!("encountered cache hit, all contracts must be different") + panic!("all contracts should be different") } let executor = cache.compile_and_insert(round, &program, cairo_native::OptLevel::None); let execution_result = executor - .invoke_contract_dynamic(&entry_point_id, &[], Some(u128::MAX), DummySyscallHandler) - .expect("contract execution failed"); + .invoke_contract_dynamic(&entry_point, &[], Some(u128::MAX), DummySyscallHandler) + .expect("failed to execute contract"); assert!( execution_result.failure_flag == false, - "contract execution failed" + "contract execution had failure flag set" ); println!( @@ -65,20 +65,27 @@ mod {name} {{ .prefix("test_") .suffix(".cairo") .tempfile() - .expect("temporary file creation failed"); - fs::write(&mut program_file, program_str).expect("writing to temporary file failed"); + .expect("failed to create temporary file for cairo test program"); + fs::write(&mut program_file, program_str).expect("failed to write cairo test file"); let contract_class = compile_path(program_file.path(), None, Default::default()) - .expect("compiling contract failed"); + .expect("failed to compile cairo contract"); let program = contract_class .extract_sierra_program() - .expect("extracting sierra failed"); + .expect("failed to extract sierra program"); - let entry_point_id = find_entry_point_by_idx(&program, 0) - .expect("cairo file should have an entry point") + let entry_point_idx = contract_class + .entry_points_by_type + .external + .first() + .expect("contrat should have at least one entrypoint") + .function_idx; + + let entry_point = find_entry_point_by_idx(&program, entry_point_idx) + .expect("failed to find entrypoint") .id .clone(); - (entry_point_id, program) + (entry_point, program) } From 0e152cdcf38c3587bbd46b265be80afdbbfffb2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Mon, 10 Jun 2024 18:30:16 -0300 Subject: [PATCH 05/26] Add tracing --- src/bin/cairo-native-stress.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index e98dab492..1c4a75991 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -1,4 +1,4 @@ -use std::fs; +use std::{fs, time::Instant}; use cairo_lang_sierra::ids::FunctionId; use cairo_lang_starknet::compile::compile_path; @@ -7,6 +7,8 @@ use cairo_native::{ utils::find_entry_point_by_idx, }; use clap::Parser; +use tracing::{info, info_span, trace}; +use tracing_subscriber::{EnvFilter, FmtSubscriber}; #[derive(Parser, Debug)] struct CliArgs { @@ -15,30 +17,48 @@ struct CliArgs { } fn main() { + tracing::subscriber::set_global_default( + FmtSubscriber::builder() + .with_env_filter(EnvFilter::from_default_env()) + .finish(), + ) + .expect("failed to set global tracing subscriber"); + let cli_args = CliArgs::parse(); let native_context = NativeContext::new(); let mut cache = AotProgramCache::new(&native_context); for round in 0..cli_args.iterations { + let _enter_span = info_span!("round"); + + let now = Instant::now(); let (entry_point, program) = generate_program("Name", round); + let elapsed = now.elapsed().as_millis(); + trace!("generated test program, took {elapsed}ms"); if cache.get(&round).is_some() { panic!("all contracts should be different") } + let now = Instant::now(); let executor = cache.compile_and_insert(round, &program, cairo_native::OptLevel::None); + let elapsed = now.elapsed().as_millis(); + trace!("compiled test program, took {elapsed}ms"); + let now = Instant::now(); let execution_result = executor .invoke_contract_dynamic(&entry_point, &[], Some(u128::MAX), DummySyscallHandler) .expect("failed to execute contract"); + let elapsed = now.elapsed().as_millis(); + trace!("executed test program, took {elapsed}ms"); assert!( execution_result.failure_flag == false, "contract execution had failure flag set" ); - println!( + info!( "Finished round {round} with result {}", execution_result.return_values[0] ); From 64d823aad0988fa1f8dbfa4136e9b52509322927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Mon, 10 Jun 2024 18:31:03 -0300 Subject: [PATCH 06/26] Rename function --- src/bin/cairo-native-stress.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 1c4a75991..b1728b57a 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -33,7 +33,7 @@ fn main() { let _enter_span = info_span!("round"); let now = Instant::now(); - let (entry_point, program) = generate_program("Name", round); + let (entry_point, program) = generate_program_from_scratch("Name", round); let elapsed = now.elapsed().as_millis(); trace!("generated test program, took {elapsed}ms"); @@ -65,7 +65,10 @@ fn main() { } } -fn generate_program(name: &str, output: u32) -> (FunctionId, cairo_lang_sierra::program::Program) { +fn generate_program_from_scratch( + name: &str, + output: u32, +) -> (FunctionId, cairo_lang_sierra::program::Program) { let program_str = format!( "\ #[starknet::contract] From bd839187258d3c82e3363a2bb2b110d36ce411df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 11 Jun 2024 10:54:20 -0300 Subject: [PATCH 07/26] Use always the same program --- src/bin/cairo-native-stress.rs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index b1728b57a..a603bd76e 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -29,16 +29,16 @@ fn main() { let native_context = NativeContext::new(); let mut cache = AotProgramCache::new(&native_context); + let now = Instant::now(); + let (entry_point, program) = generate_initial_program(); + let elapsed = now.elapsed().as_millis(); + trace!("generated test program, took {elapsed}ms"); + for round in 0..cli_args.iterations { let _enter_span = info_span!("round"); - let now = Instant::now(); - let (entry_point, program) = generate_program_from_scratch("Name", round); - let elapsed = now.elapsed().as_millis(); - trace!("generated test program, took {elapsed}ms"); - if cache.get(&round).is_some() { - panic!("all contracts should be different") + panic!("all keys should be different") } let now = Instant::now(); @@ -65,20 +65,17 @@ fn main() { } } -fn generate_program_from_scratch( - name: &str, - output: u32, -) -> (FunctionId, cairo_lang_sierra::program::Program) { +fn generate_initial_program() -> (FunctionId, cairo_lang_sierra::program::Program) { let program_str = format!( "\ #[starknet::contract] -mod {name} {{ +mod Contract {{ #[storage] struct Storage {{}} #[external(v0)] fn main(self: @ContractState) -> felt252 {{ - return {output}; + return 252; }} }} " From bea1f670630cd7ad06df897d1afcb40dfc9e71a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 11 Jun 2024 11:11:59 -0300 Subject: [PATCH 08/26] Move cache implementation to File --- src/bin/cairo-native-stress.rs | 72 +++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index a603bd76e..a093a99f4 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -1,12 +1,19 @@ -use std::{fs, time::Instant}; +use std::hash::Hash; +use std::{collections::HashMap, fs, rc::Rc, time::Instant}; use cairo_lang_sierra::ids::FunctionId; +use cairo_lang_sierra::program::Program; +use cairo_lang_sierra::program_registry::ProgramRegistry; use cairo_lang_starknet::compile::compile_path; +use cairo_native::metadata::gas::GasMetadata; +use cairo_native::utils::SHARED_LIBRARY_EXT; use cairo_native::{ - cache::AotProgramCache, context::NativeContext, starknet::DummySyscallHandler, + context::NativeContext, executor::AotNativeExecutor, starknet::DummySyscallHandler, utils::find_entry_point_by_idx, }; +use cairo_native::{module_to_object, object_to_shared_lib, OptLevel}; use clap::Parser; +use libloading::Library; use tracing::{info, info_span, trace}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; @@ -109,3 +116,64 @@ mod Contract {{ (entry_point, program) } + +struct AotProgramCache<'a, K> +where + K: PartialEq + Eq + Hash, +{ + context: &'a NativeContext, + cache: HashMap>, +} + +impl<'a, K> AotProgramCache<'a, K> +where + K: PartialEq + Eq + Hash, +{ + pub fn new(context: &'a NativeContext) -> Self { + Self { + context, + cache: Default::default(), + } + } + pub fn get(&self, key: &K) -> Option> { + self.cache.get(key).cloned() + } + + pub fn compile_and_insert( + &mut self, + key: K, + program: &Program, + opt_level: OptLevel, + ) -> Rc { + let native_module = self.context.compile(program, None).expect("should compile"); + + let object_data = module_to_object(&native_module.module(), opt_level).unwrap(); + + let shared_library_path = tempfile::Builder::new() + .prefix("lib") + .suffix(SHARED_LIBRARY_EXT) + .tempfile() + .unwrap() + .into_temp_path(); + object_to_shared_lib(&object_data, &shared_library_path).unwrap(); + + let registry = ProgramRegistry::new(program).unwrap(); + + let shared_library = unsafe { Library::new(shared_library_path).unwrap() }; + let executor = AotNativeExecutor::new( + shared_library, + registry, + native_module + .metadata() + .get::() + .cloned() + .unwrap(), + ); + + let executor = Rc::new(executor); + + self.cache.insert(key, executor.clone()); + + executor + } +} From ac8b3c072d05c088c4ceab21f29ece3e8bd94986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 11 Jun 2024 11:49:54 -0300 Subject: [PATCH 09/26] Add duration to stress test --- src/bin/cairo-native-stress.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index a093a99f4..8351933d4 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -70,6 +70,9 @@ fn main() { execution_result.return_values[0] ); } + + let elapsed = now.elapsed().as_millis(); + trace!("finished stress test, took {elapsed}ms"); } fn generate_initial_program() -> (FunctionId, cairo_lang_sierra::program::Program) { From de46960dd4e73e0ffef3ea15c6e231f4dc001fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 11 Jun 2024 12:05:10 -0300 Subject: [PATCH 10/26] Ignore .aot-cache --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a7f891151..4659b7c5d 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ cairo-*.tar *.a *.mlir /*.info +.aot-cache From cee2e1126e2c36d77ffa1e96fd57ff69d0926708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 11 Jun 2024 12:20:19 -0300 Subject: [PATCH 11/26] Refactor --- src/bin/cairo-native-stress.rs | 109 +++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 45 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 8351933d4..f233f1b55 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -1,4 +1,7 @@ +use std::fmt::Display; +use std::fs::create_dir_all; use std::hash::Hash; +use std::path::Path; use std::{collections::HashMap, fs, rc::Rc, time::Instant}; use cairo_lang_sierra::ids::FunctionId; @@ -24,6 +27,8 @@ struct CliArgs { } fn main() { + let cli_args = CliArgs::parse(); + tracing::subscriber::set_global_default( FmtSubscriber::builder() .with_env_filter(EnvFilter::from_default_env()) @@ -31,15 +36,18 @@ fn main() { ) .expect("failed to set global tracing subscriber"); - let cli_args = CliArgs::parse(); + let before_stress_test = Instant::now(); let native_context = NativeContext::new(); let mut cache = AotProgramCache::new(&native_context); - let now = Instant::now(); - let (entry_point, program) = generate_initial_program(); - let elapsed = now.elapsed().as_millis(); - trace!("generated test program, took {elapsed}ms"); + let (entry_point, program) = { + let before_generate = Instant::now(); + let initial_program = generate_initial_program(); + let elapsed = before_generate.elapsed().as_millis(); + trace!("generated test program, took {elapsed}ms"); + initial_program + }; for round in 0..cli_args.iterations { let _enter_span = info_span!("round"); @@ -48,17 +56,23 @@ fn main() { panic!("all keys should be different") } - let now = Instant::now(); - let executor = cache.compile_and_insert(round, &program, cairo_native::OptLevel::None); - let elapsed = now.elapsed().as_millis(); - trace!("compiled test program, took {elapsed}ms"); - - let now = Instant::now(); - let execution_result = executor - .invoke_contract_dynamic(&entry_point, &[], Some(u128::MAX), DummySyscallHandler) - .expect("failed to execute contract"); - let elapsed = now.elapsed().as_millis(); - trace!("executed test program, took {elapsed}ms"); + let executor = { + let before_compile = Instant::now(); + let executor = cache.compile_and_insert(round, &program, cairo_native::OptLevel::None); + let elapsed = before_compile.elapsed().as_millis(); + trace!("compiled test program, took {elapsed}ms"); + executor + }; + + let execution_result = { + let now = Instant::now(); + let execution_result = executor + .invoke_contract_dynamic(&entry_point, &[], Some(u128::MAX), DummySyscallHandler) + .expect("failed to execute contract"); + let elapsed = now.elapsed().as_millis(); + trace!("executed test program, took {elapsed}ms"); + execution_result + }; assert!( execution_result.failure_flag == false, @@ -71,8 +85,8 @@ fn main() { ); } - let elapsed = now.elapsed().as_millis(); - trace!("finished stress test, took {elapsed}ms"); + let elapsed = before_stress_test.elapsed().as_millis(); + info!("finished stress test, took {elapsed}ms"); } fn generate_initial_program() -> (FunctionId, cairo_lang_sierra::program::Program) { @@ -122,7 +136,7 @@ mod Contract {{ struct AotProgramCache<'a, K> where - K: PartialEq + Eq + Hash, + K: PartialEq + Eq + Hash + Display, { context: &'a NativeContext, cache: HashMap>, @@ -130,7 +144,7 @@ where impl<'a, K> AotProgramCache<'a, K> where - K: PartialEq + Eq + Hash, + K: PartialEq + Eq + Hash + Display, { pub fn new(context: &'a NativeContext) -> Self { Self { @@ -148,31 +162,36 @@ where program: &Program, opt_level: OptLevel, ) -> Rc { - let native_module = self.context.compile(program, None).expect("should compile"); - - let object_data = module_to_object(&native_module.module(), opt_level).unwrap(); - - let shared_library_path = tempfile::Builder::new() - .prefix("lib") - .suffix(SHARED_LIBRARY_EXT) - .tempfile() - .unwrap() - .into_temp_path(); - object_to_shared_lib(&object_data, &shared_library_path).unwrap(); - - let registry = ProgramRegistry::new(program).unwrap(); - - let shared_library = unsafe { Library::new(shared_library_path).unwrap() }; - let executor = AotNativeExecutor::new( - shared_library, - registry, - native_module - .metadata() - .get::() - .cloned() - .unwrap(), - ); - + let native_module = self + .context + .compile(program, None) + .expect("failed to compile program"); + + let registry = ProgramRegistry::new(program).expect("failed to get program registry"); + let metadata = native_module + .metadata() + .get::() + .cloned() + .expect("module should have gas metadata"); + + let shared_library = { + let object_data = module_to_object(&native_module.module(), opt_level) + .expect("failed to convert MLIR to object"); + + let shared_library_dir = Path::new(".aot-cache"); + create_dir_all(shared_library_dir).expect("failed to create shared library directory"); + let shared_library_name = format!("lib{key}{SHARED_LIBRARY_EXT}"); + let shared_library_path = shared_library_dir.join(&shared_library_name); + + object_to_shared_lib(&object_data, &shared_library_path) + .expect("failed to link object into shared library"); + + unsafe { + Library::new(shared_library_path).expect("failed to load dynamic shared library") + } + }; + + let executor = AotNativeExecutor::new(shared_library, registry, metadata); let executor = Rc::new(executor); self.cache.insert(key, executor.clone()); From 0591435ac7e9441cb3228292f4a02e816836f720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 11 Jun 2024 13:20:22 -0300 Subject: [PATCH 12/26] Add documentation --- src/bin/cairo-native-stress.rs | 54 +++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index f233f1b55..2a8394ed9 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -1,3 +1,7 @@ +//! A stress tester for Cairo Native +//! +//! See `StressTestCommand` + use std::fmt::Display; use std::fs::create_dir_all; use std::hash::Hash; @@ -20,14 +24,21 @@ use libloading::Library; use tracing::{info, info_span, trace}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; +/// The directory used to store compiled native programs +const AOT_CACHE_DIR: &str = ".aot-cache"; + +/// A stress tester for Cairo Native +/// +/// It Sierra programs compiles with Cairo Native, caches, and executes them with AOT runner. +/// The compiled dynamic libraries are stores in `AOT_CACHE_DIR` relative to the current working directory. #[derive(Parser, Debug)] -struct CliArgs { - /// Amount of iterations to perform +struct StressTestCommand { + /// Amount of programs to generate and test iterations: u32, } fn main() { - let cli_args = CliArgs::parse(); + let cli_args = StressTestCommand::parse(); tracing::subscriber::set_global_default( FmtSubscriber::builder() @@ -39,23 +50,27 @@ fn main() { let before_stress_test = Instant::now(); let native_context = NativeContext::new(); - let mut cache = AotProgramCache::new(&native_context); + let mut cache = NaiveAotCache::new(&native_context); + // Generate initial program. let (entry_point, program) = { let before_generate = Instant::now(); - let initial_program = generate_initial_program(); + let initial_program = generate_starknet_contract(); let elapsed = before_generate.elapsed().as_millis(); trace!("generated test program, took {elapsed}ms"); initial_program }; - for round in 0..cli_args.iterations { + for round in 1..=cli_args.iterations { let _enter_span = info_span!("round"); + // The round count is used as a key. After making sure each iteration uses + // a different unique program, the program hash should be used. if cache.get(&round).is_some() { - panic!("all keys should be different") + panic!("all program keys should be different") } + // Compile and caches the program let executor = { let before_compile = Instant::now(); let executor = cache.compile_and_insert(round, &program, cairo_native::OptLevel::None); @@ -64,6 +79,7 @@ fn main() { executor }; + // Executes the program let execution_result = { let now = Instant::now(); let execution_result = executor @@ -89,7 +105,12 @@ fn main() { info!("finished stress test, took {elapsed}ms"); } -fn generate_initial_program() -> (FunctionId, cairo_lang_sierra::program::Program) { +/// Generate a dummy starknet contract +/// +/// This is should only be done once as it takes a long time. +/// We should modify the program returned from this to obtain +/// different unique programs without recompiling each time +fn generate_starknet_contract() -> (FunctionId, cairo_lang_sierra::program::Program) { let program_str = format!( "\ #[starknet::contract] @@ -134,7 +155,15 @@ mod Contract {{ (entry_point, program) } -struct AotProgramCache<'a, K> +/// A naive implementation of an AOT Program Cache. +/// +/// Stores `AotNativeExecutor`s by a given key. Each executors has it's corresponding +/// dynamic shared library loaded. +/// +/// Possible improvements include: +/// - Keeping only some executores on memory, while storing the remianing compiled shared libraries on disk. +/// - When restarting the program, reutilize already compiled programs from `AOT_CACHE_DIR` +struct NaiveAotCache<'a, K> where K: PartialEq + Eq + Hash + Display, { @@ -142,7 +171,7 @@ where cache: HashMap>, } -impl<'a, K> AotProgramCache<'a, K> +impl<'a, K> NaiveAotCache<'a, K> where K: PartialEq + Eq + Hash + Display, { @@ -156,6 +185,9 @@ where self.cache.get(key).cloned() } + /// Compiles and inserts a given program into the cache + /// + /// The dynamic library is stored in `AOT_CACHE_DIR` directory pub fn compile_and_insert( &mut self, key: K, @@ -178,7 +210,7 @@ where let object_data = module_to_object(&native_module.module(), opt_level) .expect("failed to convert MLIR to object"); - let shared_library_dir = Path::new(".aot-cache"); + let shared_library_dir = Path::new(AOT_CACHE_DIR); create_dir_all(shared_library_dir).expect("failed to create shared library directory"); let shared_library_name = format!("lib{key}{SHARED_LIBRARY_EXT}"); let shared_library_path = shared_library_dir.join(&shared_library_name); From 53242453fe6a63d68657b146fd70d24c7ddde03f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 11 Jun 2024 13:58:08 -0300 Subject: [PATCH 13/26] Use outer and inner loop --- src/bin/cairo-native-stress.rs | 96 +++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 37 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 2a8394ed9..c51e8aee3 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -21,7 +21,7 @@ use cairo_native::{ use cairo_native::{module_to_object, object_to_shared_lib, OptLevel}; use clap::Parser; use libloading::Library; -use tracing::{info, info_span, trace}; +use tracing::{debug, debug_span, info, info_span}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; /// The directory used to store compiled native programs @@ -33,8 +33,10 @@ const AOT_CACHE_DIR: &str = ".aot-cache"; /// The compiled dynamic libraries are stores in `AOT_CACHE_DIR` relative to the current working directory. #[derive(Parser, Debug)] struct StressTestCommand { - /// Amount of programs to generate and test - iterations: u32, + /// Amount of rounds to execute + rounds: u32, + /// Amount of programs to generate and test per round + programs: u32, } fn main() { @@ -57,52 +59,67 @@ fn main() { let before_generate = Instant::now(); let initial_program = generate_starknet_contract(); let elapsed = before_generate.elapsed().as_millis(); - trace!("generated test program, took {elapsed}ms"); + info!(time = elapsed, "generated test program"); initial_program }; - for round in 1..=cli_args.iterations { - let _enter_span = info_span!("round"); + for round in 0..cli_args.rounds { + let _enter_round_span = info_span!("round"); + let before_round = Instant::now(); - // The round count is used as a key. After making sure each iteration uses - // a different unique program, the program hash should be used. - if cache.get(&round).is_some() { - panic!("all program keys should be different") - } - - // Compile and caches the program - let executor = { - let before_compile = Instant::now(); - let executor = cache.compile_and_insert(round, &program, cairo_native::OptLevel::None); - let elapsed = before_compile.elapsed().as_millis(); - trace!("compiled test program, took {elapsed}ms"); - executor - }; + for program_number in 0..cli_args.programs { + let _enter_program_span = debug_span!("program"); - // Executes the program - let execution_result = { - let now = Instant::now(); - let execution_result = executor - .invoke_contract_dynamic(&entry_point, &[], Some(u128::MAX), DummySyscallHandler) - .expect("failed to execute contract"); - let elapsed = now.elapsed().as_millis(); - trace!("executed test program, took {elapsed}ms"); - execution_result - }; + // The round and program count is used as a key. After making sure each iteration uses + // a different unique program, the program hash should be used. + let key = round * cli_args.programs + program_number; + if cache.get(&key).is_some() { + panic!("all program keys should be different") + } - assert!( - execution_result.failure_flag == false, - "contract execution had failure flag set" - ); + // Compile and caches the program + let executor = { + let before_compile = Instant::now(); + let executor = + cache.compile_and_insert(key, &program, cairo_native::OptLevel::None); + let elapsed = before_compile.elapsed().as_millis(); + debug!(time = elapsed, "compiled test program"); + executor + }; + + // Executes the program + let execution_result = { + let now = Instant::now(); + let execution_result = executor + .invoke_contract_dynamic( + &entry_point, + &[], + Some(u128::MAX), + DummySyscallHandler, + ) + .expect("failed to execute contract"); + let elapsed = now.elapsed().as_millis(); + debug!(time = elapsed, "executed test program"); + execution_result + }; + + assert!( + execution_result.failure_flag == false, + "contract execution had failure flag set" + ); + } + let elapsed = before_round.elapsed().as_millis(); info!( - "Finished round {round} with result {}", - execution_result.return_values[0] + round, + time = elapsed, + cache_len = cache.len(), + "finished round" ); } let elapsed = before_stress_test.elapsed().as_millis(); - info!("finished stress test, took {elapsed}ms"); + info!(time = elapsed, "finished stress test"); } /// Generate a dummy starknet contract @@ -181,10 +198,15 @@ where cache: Default::default(), } } + pub fn get(&self, key: &K) -> Option> { self.cache.get(key).cloned() } + pub fn len(&self) -> usize { + self.cache.len() + } + /// Compiles and inserts a given program into the cache /// /// The dynamic library is stored in `AOT_CACHE_DIR` directory From 29da5bddaa213db9b8ad330cd01d903500eeaf01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 11 Jun 2024 14:32:14 -0300 Subject: [PATCH 14/26] Show directory size --- src/bin/cairo-native-stress.rs | 56 ++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index c51e8aee3..5f9ebf499 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -3,8 +3,9 @@ //! See `StressTestCommand` use std::fmt::Display; -use std::fs::create_dir_all; +use std::fs::{create_dir_all, read_dir}; use std::hash::Hash; +use std::io; use std::path::Path; use std::{collections::HashMap, fs, rc::Rc, time::Instant}; @@ -21,7 +22,7 @@ use cairo_native::{ use cairo_native::{module_to_object, object_to_shared_lib, OptLevel}; use clap::Parser; use libloading::Library; -use tracing::{debug, debug_span, info, info_span}; +use tracing::{debug, debug_span, info, info_span, warn}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; /// The directory used to store compiled native programs @@ -49,6 +50,10 @@ fn main() { ) .expect("failed to set global tracing subscriber"); + if !directory_is_empty(AOT_CACHE_DIR).expect("failed to open aot cache dir") { + warn!("{AOT_CACHE_DIR} directory is not empty") + } + let before_stress_test = Instant::now(); let native_context = NativeContext::new(); @@ -109,13 +114,18 @@ fn main() { ); } - let elapsed = before_round.elapsed().as_millis(); - info!( - round, - time = elapsed, - cache_len = cache.len(), - "finished round" - ); + { + let cache_disk_size = + directory_get_size(AOT_CACHE_DIR).expect("failed to calculate cache disk size"); + let elapsed = before_round.elapsed().as_millis(); + info!( + round = round, + time = elapsed, + cache_mem_len = cache.len(), + cache_disk_size = cache_disk_size, + "finished round" + ); + } } let elapsed = before_stress_test.elapsed().as_millis(); @@ -253,3 +263,31 @@ where executor } } + +/// Returns the size of a directory in bytes +fn directory_get_size(path: impl AsRef) -> io::Result { + let mut dir = read_dir(path)?; + + dir.try_fold(0, |total_size, entry| { + let entry = entry?; + + let size = match entry.metadata()? { + data if data.is_dir() => directory_get_size(entry.path())?, + data => data.len(), + }; + + return Ok(total_size + size); + }) +} + +fn directory_is_empty(path: impl AsRef) -> io::Result { + let is_empty = match read_dir(path) { + Ok(mut directory) => directory.next().is_none(), + Err(error) => match error.kind() { + io::ErrorKind::NotFound => true, + _ => return Err(error), + }, + }; + + Ok(is_empty) +} From 4a1e464defb2441a4b4753e6e14ff5a1f7e20ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 11 Jun 2024 14:50:11 -0300 Subject: [PATCH 15/26] Fix clippy --- src/bin/cairo-native-stress.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 5f9ebf499..846123b24 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -109,7 +109,7 @@ fn main() { }; assert!( - execution_result.failure_flag == false, + !execution_result.failure_flag, "contract execution had failure flag set" ); } @@ -138,8 +138,7 @@ fn main() { /// We should modify the program returned from this to obtain /// different unique programs without recompiling each time fn generate_starknet_contract() -> (FunctionId, cairo_lang_sierra::program::Program) { - let program_str = format!( - "\ + let program_str = "\ #[starknet::contract] mod Contract {{ #[storage] @@ -150,8 +149,7 @@ mod Contract {{ return 252; }} }} -" - ); +"; let mut program_file = tempfile::Builder::new() .prefix("test_") @@ -239,13 +237,13 @@ where .expect("module should have gas metadata"); let shared_library = { - let object_data = module_to_object(&native_module.module(), opt_level) + let object_data = module_to_object(native_module.module(), opt_level) .expect("failed to convert MLIR to object"); let shared_library_dir = Path::new(AOT_CACHE_DIR); create_dir_all(shared_library_dir).expect("failed to create shared library directory"); let shared_library_name = format!("lib{key}{SHARED_LIBRARY_EXT}"); - let shared_library_path = shared_library_dir.join(&shared_library_name); + let shared_library_path = shared_library_dir.join(shared_library_name); object_to_shared_lib(&object_data, &shared_library_path) .expect("failed to link object into shared library"); @@ -276,7 +274,7 @@ fn directory_get_size(path: impl AsRef) -> io::Result { data => data.len(), }; - return Ok(total_size + size); + Ok(total_size + size) }) } From 52683b42224154dc9ce1bb16c1d71154e6e959ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 11 Jun 2024 16:48:07 -0300 Subject: [PATCH 16/26] Fix bug --- src/bin/cairo-native-stress.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 846123b24..4278ee85b 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -140,15 +140,15 @@ fn main() { fn generate_starknet_contract() -> (FunctionId, cairo_lang_sierra::program::Program) { let program_str = "\ #[starknet::contract] -mod Contract {{ +mod Contract { #[storage] - struct Storage {{}} + struct Storage {} #[external(v0)] - fn main(self: @ContractState) -> felt252 {{ + fn main(self: @ContractState) -> felt252 { return 252; - }} -}} + } +} "; let mut program_file = tempfile::Builder::new() From cd6372e31c8ebf8f8bd4d59dd516db1b86f1fcf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 11 Jun 2024 19:13:42 -0300 Subject: [PATCH 17/26] Modify program --- src/bin/cairo-native-stress.rs | 66 +++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 4278ee85b..54539575c 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -10,7 +10,7 @@ use std::path::Path; use std::{collections::HashMap, fs, rc::Rc, time::Instant}; use cairo_lang_sierra::ids::FunctionId; -use cairo_lang_sierra::program::Program; +use cairo_lang_sierra::program::{GenericArg, Program}; use cairo_lang_sierra::program_registry::ProgramRegistry; use cairo_lang_starknet::compile::compile_path; use cairo_native::metadata::gas::GasMetadata; @@ -22,12 +22,19 @@ use cairo_native::{ use cairo_native::{module_to_object, object_to_shared_lib, OptLevel}; use clap::Parser; use libloading::Library; +use num_bigint::BigInt; use tracing::{debug, debug_span, info, info_span, warn}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; /// The directory used to store compiled native programs const AOT_CACHE_DIR: &str = ".aot-cache"; +/// An unique value hardcoded into the first contract that it's +/// used as an anchor point to safely modify the return value on +/// the following contracts. +/// It can be any value as long as it's unique in the initial contract. +const RETURN_ANCHOR: u32 = 835; + /// A stress tester for Cairo Native /// /// It Sierra programs compiles with Cairo Native, caches, and executes them with AOT runner. @@ -75,9 +82,11 @@ fn main() { for program_number in 0..cli_args.programs { let _enter_program_span = debug_span!("program"); - // The round and program count is used as a key. After making sure each iteration uses - // a different unique program, the program hash should be used. - let key = round * cli_args.programs + program_number; + let global_program_number = round * cli_args.programs + program_number; + let program = modify_starknet_contract(program.clone(), global_program_number); + + // The round and program count is used as a key. Later on the program hash should be used. + let key = global_program_number; if cache.get(&key).is_some() { panic!("all program keys should be different") } @@ -104,7 +113,8 @@ fn main() { ) .expect("failed to execute contract"); let elapsed = now.elapsed().as_millis(); - debug!(time = elapsed, "executed test program"); + let result = execution_result.return_values[0]; + debug!(time = elapsed, result = %result, "executed test program"); execution_result }; @@ -138,18 +148,20 @@ fn main() { /// We should modify the program returned from this to obtain /// different unique programs without recompiling each time fn generate_starknet_contract() -> (FunctionId, cairo_lang_sierra::program::Program) { - let program_str = "\ + let program_str = format!( + "\ #[starknet::contract] -mod Contract { +mod Contract {{ #[storage] - struct Storage {} + struct Storage {{}} #[external(v0)] - fn main(self: @ContractState) -> felt252 { - return 252; - } -} -"; + fn main(self: @ContractState) -> felt252 {{ + return {RETURN_ANCHOR}; + }} +}} +" + ); let mut program_file = tempfile::Builder::new() .prefix("test_") @@ -180,6 +192,34 @@ mod Contract { (entry_point, program) } +/// Modifies the given contract by replacing the `RETURN_ANCHOR` wtith `new_return_value` +/// +/// The contract must only contain the value `RETURN_ANCHOR` exactly once +fn modify_starknet_contract(mut program: Program, new_return_value: u32) -> Program { + let mut anchor_counter = 0; + + for type_declaration in &mut program.type_declarations { + for generic_arg in &mut type_declaration.long_id.generic_args { + let anchor = BigInt::from(RETURN_ANCHOR); + + match generic_arg { + GenericArg::Value(return_value) if *return_value == anchor => { + *return_value = BigInt::from(new_return_value); + anchor_counter += 1; + } + _ => {} + }; + } + } + + assert!( + anchor_counter == 1, + "RETURN_ANCHOR was not found exactly once" + ); + + program +} + /// A naive implementation of an AOT Program Cache. /// /// Stores `AotNativeExecutor`s by a given key. Each executors has it's corresponding From 9e6af4afbf95a91678f60694f2a371ca116cb128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 12 Jun 2024 11:33:52 -0300 Subject: [PATCH 18/26] Add alloc stats better logging --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + src/bin/cairo-native-stress.rs | 33 +++++++++++++++++++++++++-------- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7cbc7276e..048935338 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -952,6 +952,7 @@ dependencies = [ "serde", "serde_json", "starknet-types-core", + "stats_alloc", "tempfile", "test-case", "thiserror", @@ -3481,6 +3482,12 @@ dependencies = [ "serde", ] +[[package]] +name = "stats_alloc" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e04424e733e69714ca1bbb9204c1a57f09f5493439520f9f68c132ad25eec" + [[package]] name = "string_cache" version = "0.8.7" diff --git a/Cargo.toml b/Cargo.toml index d423771dd..5d1b01628 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,7 @@ scarb-metadata = { git = "https://github.com/software-mansion/scarb.git", rev = scarb-ui = { git = "https://github.com/software-mansion/scarb.git", rev = "v2.6.3", optional = true } sec1 = { version = "0.7.3", optional = true } serde_json = { version = "1.0.117", optional = true } +stats_alloc = "0.1.10" [dev-dependencies] cairo-vm = { version = "1.0.0-rc3", features = ["cairo-1-hints"] } diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 4278ee85b..259899d31 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -2,6 +2,7 @@ //! //! See `StressTestCommand` +use std::alloc::System; use std::fmt::Display; use std::fs::{create_dir_all, read_dir}; use std::hash::Hash; @@ -22,9 +23,13 @@ use cairo_native::{ use cairo_native::{module_to_object, object_to_shared_lib, OptLevel}; use clap::Parser; use libloading::Library; +use stats_alloc::{Region, StatsAlloc, INSTRUMENTED_SYSTEM}; use tracing::{debug, debug_span, info, info_span, warn}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; +#[global_allocator] +static GLOBAL_ALLOC: &StatsAlloc = &INSTRUMENTED_SYSTEM; + /// The directory used to store compiled native programs const AOT_CACHE_DIR: &str = ".aot-cache"; @@ -54,20 +59,23 @@ fn main() { warn!("{AOT_CACHE_DIR} directory is not empty") } - let before_stress_test = Instant::now(); - - let native_context = NativeContext::new(); - let mut cache = NaiveAotCache::new(&native_context); - // Generate initial program. let (entry_point, program) = { let before_generate = Instant::now(); let initial_program = generate_starknet_contract(); let elapsed = before_generate.elapsed().as_millis(); - info!(time = elapsed, "generated test program"); + debug!(time = elapsed, "generated test program"); initial_program }; + let global_region = Region::new(GLOBAL_ALLOC); + let before_stress_test = Instant::now(); + + // Initialize context and cache + let native_context = NativeContext::new(); + let mut cache = NaiveAotCache::new(&native_context); + info!("initialized context and cache"); + for round in 0..cli_args.rounds { let _enter_round_span = info_span!("round"); let before_round = Instant::now(); @@ -75,9 +83,15 @@ fn main() { for program_number in 0..cli_args.programs { let _enter_program_span = debug_span!("program"); + // The program is cloned to simulate that it received a new program + let program = program.clone(); + // The round and program count is used as a key. After making sure each iteration uses // a different unique program, the program hash should be used. let key = round * cli_args.programs + program_number; + + debug!(hash = key, "obtained test program"); + if cache.get(&key).is_some() { panic!("all program keys should be different") } @@ -115,13 +129,16 @@ fn main() { } { + let elapsed = before_round.elapsed().as_millis(); let cache_disk_size = directory_get_size(AOT_CACHE_DIR).expect("failed to calculate cache disk size"); - let elapsed = before_round.elapsed().as_millis(); + let global_stats = global_region.change(); + let cache_mem_size = global_stats.bytes_allocated - global_stats.bytes_deallocated; info!( - round = round, + number = round, time = elapsed, cache_mem_len = cache.len(), + cache_mem_size = cache_mem_size, cache_disk_size = cache_disk_size, "finished round" ); From 1830c47d5fa9cb89cb60e2b9565b4c7ff7e788a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 12 Jun 2024 12:50:52 -0300 Subject: [PATCH 19/26] Use only rounds --- src/bin/cairo-native-stress.rs | 97 ++++++++++++++-------------------- 1 file changed, 40 insertions(+), 57 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 259899d31..410ef689f 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -24,7 +24,7 @@ use cairo_native::{module_to_object, object_to_shared_lib, OptLevel}; use clap::Parser; use libloading::Library; use stats_alloc::{Region, StatsAlloc, INSTRUMENTED_SYSTEM}; -use tracing::{debug, debug_span, info, info_span, warn}; +use tracing::{debug, info, info_span, warn}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; #[global_allocator] @@ -36,13 +36,11 @@ const AOT_CACHE_DIR: &str = ".aot-cache"; /// A stress tester for Cairo Native /// /// It Sierra programs compiles with Cairo Native, caches, and executes them with AOT runner. -/// The compiled dynamic libraries are stores in `AOT_CACHE_DIR` relative to the current working directory. +/// The compiled dynamic libraries are stored in `AOT_CACHE_DIR` relative to the current working directory. #[derive(Parser, Debug)] struct StressTestCommand { /// Amount of rounds to execute rounds: u32, - /// Amount of programs to generate and test per round - programs: u32, } fn main() { @@ -59,7 +57,7 @@ fn main() { warn!("{AOT_CACHE_DIR} directory is not empty") } - // Generate initial program. + // Generate initial program let (entry_point, program) = { let before_generate = Instant::now(); let initial_program = generate_starknet_contract(); @@ -74,71 +72,60 @@ fn main() { // Initialize context and cache let native_context = NativeContext::new(); let mut cache = NaiveAotCache::new(&native_context); - info!("initialized context and cache"); + + info!("starting stress test"); for round in 0..cli_args.rounds { - let _enter_round_span = info_span!("round"); + let _enter_round_span = info_span!("round", number = round).entered(); + let before_round = Instant::now(); - for program_number in 0..cli_args.programs { - let _enter_program_span = debug_span!("program"); + // The program is cloned to simulate that it received a new program + let program = program.clone(); + // The round count is used as a key. After making sure each iteration uses + // a different unique program, the program hash should be used. + let key = round; - // The program is cloned to simulate that it received a new program - let program = program.clone(); + debug!(hash = key, "obtained test program"); - // The round and program count is used as a key. After making sure each iteration uses - // a different unique program, the program hash should be used. - let key = round * cli_args.programs + program_number; + if cache.get(&key).is_some() { + panic!("all program keys should be different") + } - debug!(hash = key, "obtained test program"); + // Compile and caches the program + let executor = { + let before_compile = Instant::now(); + let executor = cache.compile_and_insert(key, &program, cairo_native::OptLevel::None); + let elapsed = before_compile.elapsed().as_millis(); + debug!(time = elapsed, "compiled test program"); + executor + }; - if cache.get(&key).is_some() { - panic!("all program keys should be different") - } + // Executes the program + let execution_result = { + let now = Instant::now(); + let execution_result = executor + .invoke_contract_dynamic(&entry_point, &[], Some(u128::MAX), DummySyscallHandler) + .expect("failed to execute contract"); + let elapsed = now.elapsed().as_millis(); + debug!(time = elapsed, "executed test program"); + execution_result + }; - // Compile and caches the program - let executor = { - let before_compile = Instant::now(); - let executor = - cache.compile_and_insert(key, &program, cairo_native::OptLevel::None); - let elapsed = before_compile.elapsed().as_millis(); - debug!(time = elapsed, "compiled test program"); - executor - }; - - // Executes the program - let execution_result = { - let now = Instant::now(); - let execution_result = executor - .invoke_contract_dynamic( - &entry_point, - &[], - Some(u128::MAX), - DummySyscallHandler, - ) - .expect("failed to execute contract"); - let elapsed = now.elapsed().as_millis(); - debug!(time = elapsed, "executed test program"); - execution_result - }; - - assert!( - !execution_result.failure_flag, - "contract execution had failure flag set" - ); - } + assert!( + !execution_result.failure_flag, + "contract execution had failure flag set" + ); { let elapsed = before_round.elapsed().as_millis(); let cache_disk_size = directory_get_size(AOT_CACHE_DIR).expect("failed to calculate cache disk size"); let global_stats = global_region.change(); - let cache_mem_size = global_stats.bytes_allocated - global_stats.bytes_deallocated; + let memory_used = global_stats.bytes_allocated - global_stats.bytes_deallocated; info!( - number = round, time = elapsed, - cache_mem_len = cache.len(), - cache_mem_size = cache_mem_size, + memory_used = memory_used, cache_disk_size = cache_disk_size, "finished round" ); @@ -228,10 +215,6 @@ where self.cache.get(key).cloned() } - pub fn len(&self) -> usize { - self.cache.len() - } - /// Compiles and inserts a given program into the cache /// /// The dynamic library is stored in `AOT_CACHE_DIR` directory From f3fe41d4485cd8d8d9bd4ed663f7ea542ef64585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 12 Jun 2024 13:18:42 -0300 Subject: [PATCH 20/26] Refactor --- src/bin/cairo-native-stress.rs | 42 ++++++++++++++++------------------ 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 025b8a1ea..f36df22e6 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -34,11 +34,10 @@ static GLOBAL_ALLOC: &StatsAlloc = &INSTRUMENTED_SYSTEM; /// The directory used to store compiled native programs const AOT_CACHE_DIR: &str = ".aot-cache"; -/// An unique value hardcoded into the first contract that it's -/// used as an anchor point to safely modify the return value on -/// the following contracts. -/// It can be any value as long as it's unique in the initial contract. -const RETURN_ANCHOR: u32 = 835; +/// An unique value hardcoded into the initial contract that it's +/// used as an anchor point to safely modify it. +/// It can be any value as long as it's unique. +const CONTRACT_MODIFICATION_ANCHOR: u32 = 835; /// A stress tester for Cairo Native /// @@ -122,19 +121,18 @@ fn main() { "contract execution had failure flag set" ); - { - let elapsed = before_round.elapsed().as_millis(); - let cache_disk_size = - directory_get_size(AOT_CACHE_DIR).expect("failed to calculate cache disk size"); - let global_stats = global_region.change(); - let memory_used = global_stats.bytes_allocated - global_stats.bytes_deallocated; - info!( - time = elapsed, - memory_used = memory_used, - cache_disk_size = cache_disk_size, - "finished round" - ); - } + // Logs end of round + let elapsed = before_round.elapsed().as_millis(); + let cache_disk_size = + directory_get_size(AOT_CACHE_DIR).expect("failed to calculate cache disk size"); + let global_stats = global_region.change(); + let memory_used = global_stats.bytes_allocated - global_stats.bytes_deallocated; + info!( + time = elapsed, + memory_used = memory_used, + cache_disk_size = cache_disk_size, + "finished round" + ); } let elapsed = before_stress_test.elapsed().as_millis(); @@ -156,7 +154,7 @@ mod Contract {{ #[external(v0)] fn main(self: @ContractState) -> felt252 {{ - return {RETURN_ANCHOR}; + return {CONTRACT_MODIFICATION_ANCHOR}; }} }} " @@ -180,7 +178,7 @@ mod Contract {{ .entry_points_by_type .external .first() - .expect("contrat should have at least one entrypoint") + .expect("contract should have at least one entrypoint") .function_idx; let entry_point = find_entry_point_by_idx(&program, entry_point_idx) @@ -199,7 +197,7 @@ fn modify_starknet_contract(mut program: Program, new_return_value: u32) -> Prog for type_declaration in &mut program.type_declarations { for generic_arg in &mut type_declaration.long_id.generic_args { - let anchor = BigInt::from(RETURN_ANCHOR); + let anchor = BigInt::from(CONTRACT_MODIFICATION_ANCHOR); match generic_arg { GenericArg::Value(return_value) if *return_value == anchor => { @@ -213,7 +211,7 @@ fn modify_starknet_contract(mut program: Program, new_return_value: u32) -> Prog assert!( anchor_counter == 1, - "RETURN_ANCHOR was not found exactly once" + "CONTRACT_MODIFICATION_ANCHOR was not found exactly once" ); program From ce650edd1dfbf6e2ce92b4b3c782c69e0f359b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 12 Jun 2024 13:34:30 -0300 Subject: [PATCH 21/26] Refactor --- src/bin/cairo-native-stress.rs | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index f36df22e6..b64317af7 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -36,7 +36,7 @@ const AOT_CACHE_DIR: &str = ".aot-cache"; /// An unique value hardcoded into the initial contract that it's /// used as an anchor point to safely modify it. -/// It can be any value as long as it's unique. +/// It can be any value as long as it's unique in the contract. const CONTRACT_MODIFICATION_ANCHOR: u32 = 835; /// A stress tester for Cairo Native @@ -66,7 +66,7 @@ fn main() { // Generate initial program let (entry_point, program) = { let before_generate = Instant::now(); - let initial_program = generate_starknet_contract(); + let initial_program = generate_starknet_contract(CONTRACT_MODIFICATION_ANCHOR); let elapsed = before_generate.elapsed().as_millis(); debug!(time = elapsed, "generated test program"); initial_program @@ -86,7 +86,8 @@ fn main() { let before_round = Instant::now(); - let program = modify_starknet_contract(program.clone(), round); + let program = + modify_starknet_contract(program.clone(), CONTRACT_MODIFICATION_ANCHOR, round); // TODO: use the program hash instead of round number. let hash = round; @@ -96,7 +97,7 @@ fn main() { panic!("all program keys should be different") } - // Compile and caches the program + // Compiles and caches the program let executor = { let before_compile = Instant::now(); let executor = cache.compile_and_insert(hash, &program, cairo_native::OptLevel::None); @@ -141,10 +142,10 @@ fn main() { /// Generate a dummy starknet contract /// -/// This is should only be done once as it takes a long time. -/// We should modify the program returned from this to obtain -/// different unique programs without recompiling each time -fn generate_starknet_contract() -> (FunctionId, cairo_lang_sierra::program::Program) { +/// The contract contains an external main function that returns `return_value` +fn generate_starknet_contract( + return_value: u32, +) -> (FunctionId, cairo_lang_sierra::program::Program) { let program_str = format!( "\ #[starknet::contract] @@ -154,7 +155,7 @@ mod Contract {{ #[external(v0)] fn main(self: @ContractState) -> felt252 {{ - return {CONTRACT_MODIFICATION_ANCHOR}; + return {return_value}; }} }} " @@ -189,19 +190,19 @@ mod Contract {{ (entry_point, program) } -/// Modifies the given contract by replacing the `RETURN_ANCHOR` wtith `new_return_value` +/// Modifies the given contract by replacing the `anchor_value` with `new_value` /// -/// The contract must only contain the value `RETURN_ANCHOR` exactly once -fn modify_starknet_contract(mut program: Program, new_return_value: u32) -> Program { +/// The contract must only contain the value `anchor_value` once +fn modify_starknet_contract(mut program: Program, anchor_value: u32, new_value: u32) -> Program { let mut anchor_counter = 0; for type_declaration in &mut program.type_declarations { for generic_arg in &mut type_declaration.long_id.generic_args { - let anchor = BigInt::from(CONTRACT_MODIFICATION_ANCHOR); + let anchor = BigInt::from(anchor_value); match generic_arg { GenericArg::Value(return_value) if *return_value == anchor => { - *return_value = BigInt::from(new_return_value); + *return_value = BigInt::from(new_value); anchor_counter += 1; } _ => {} @@ -223,7 +224,7 @@ fn modify_starknet_contract(mut program: Program, new_return_value: u32) -> Prog /// dynamic shared library loaded. /// /// Possible improvements include: -/// - Keeping only some executores on memory, while storing the remianing compiled shared libraries on disk. +/// - Keeping only some executores on memory, while storing the remaianing compiled shared libraries on disk. /// - When restarting the program, reutilize already compiled programs from `AOT_CACHE_DIR` struct NaiveAotCache<'a, K> where From 1aa62ab78852c137ddf4d5a28bdb6f892440cb6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 12 Jun 2024 13:45:00 -0300 Subject: [PATCH 22/26] Fix Typo --- src/bin/cairo-native-stress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index b64317af7..6672ed08d 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -224,7 +224,7 @@ fn modify_starknet_contract(mut program: Program, anchor_value: u32, new_value: /// dynamic shared library loaded. /// /// Possible improvements include: -/// - Keeping only some executores on memory, while storing the remaianing compiled shared libraries on disk. +/// - Keeping only some executors on memory, while storing the remaining compiled shared libraries on disk. /// - When restarting the program, reutilize already compiled programs from `AOT_CACHE_DIR` struct NaiveAotCache<'a, K> where From 034805e7f0a4d18d6221fdb8b9042b41ad153aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 12 Jun 2024 15:05:01 -0300 Subject: [PATCH 23/26] Improve documentation --- src/bin/cairo-native-stress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 6672ed08d..1b0e85797 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -190,7 +190,7 @@ mod Contract {{ (entry_point, program) } -/// Modifies the given contract by replacing the `anchor_value` with `new_value` +/// Modifies the given contract by replacing the `anchor_value` with `new_value` in any type declaration /// /// The contract must only contain the value `anchor_value` once fn modify_starknet_contract(mut program: Program, anchor_value: u32, new_value: u32) -> Program { From f0c468e5fec8ef371286a3c3cb959373a05cce87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 12 Jun 2024 16:08:53 -0300 Subject: [PATCH 24/26] Log execution result --- src/bin/cairo-native-stress.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 1b0e85797..6b76d68e0 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -113,7 +113,8 @@ fn main() { .invoke_contract_dynamic(&entry_point, &[], Some(u128::MAX), DummySyscallHandler) .expect("failed to execute contract"); let elapsed = now.elapsed().as_millis(); - debug!(time = elapsed, "executed test program"); + let result = execution_result.return_values[0]; + debug!(time = elapsed, result = %result, "executed test program"); execution_result }; From 653c99b05c2233c068786aa14ced1d4d3c8b4f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Thu, 13 Jun 2024 11:27:06 -0300 Subject: [PATCH 25/26] Rename variables --- src/bin/cairo-native-stress.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index 6b76d68e0..b3cb2171d 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -37,7 +37,7 @@ const AOT_CACHE_DIR: &str = ".aot-cache"; /// An unique value hardcoded into the initial contract that it's /// used as an anchor point to safely modify it. /// It can be any value as long as it's unique in the contract. -const CONTRACT_MODIFICATION_ANCHOR: u32 = 835; +const UNIQUE_CONTRACT_VALUE: u32 = 835; /// A stress tester for Cairo Native /// @@ -66,7 +66,7 @@ fn main() { // Generate initial program let (entry_point, program) = { let before_generate = Instant::now(); - let initial_program = generate_starknet_contract(CONTRACT_MODIFICATION_ANCHOR); + let initial_program = generate_starknet_contract(UNIQUE_CONTRACT_VALUE); let elapsed = before_generate.elapsed().as_millis(); debug!(time = elapsed, "generated test program"); initial_program @@ -86,8 +86,7 @@ fn main() { let before_round = Instant::now(); - let program = - modify_starknet_contract(program.clone(), CONTRACT_MODIFICATION_ANCHOR, round); + let program = modify_starknet_contract(program.clone(), UNIQUE_CONTRACT_VALUE, round); // TODO: use the program hash instead of round number. let hash = round; @@ -191,20 +190,20 @@ mod Contract {{ (entry_point, program) } -/// Modifies the given contract by replacing the `anchor_value` with `new_value` in any type declaration +/// Modifies the given contract by replacing the `old_value` with `new_value` in any type declaration /// -/// The contract must only contain the value `anchor_value` once -fn modify_starknet_contract(mut program: Program, anchor_value: u32, new_value: u32) -> Program { - let mut anchor_counter = 0; +/// The contract must only contain the value `old_value` once +fn modify_starknet_contract(mut program: Program, old_value: u32, new_value: u32) -> Program { + let mut old_value_counter = 0; for type_declaration in &mut program.type_declarations { for generic_arg in &mut type_declaration.long_id.generic_args { - let anchor = BigInt::from(anchor_value); + let anchor = BigInt::from(old_value); match generic_arg { GenericArg::Value(return_value) if *return_value == anchor => { *return_value = BigInt::from(new_value); - anchor_counter += 1; + old_value_counter += 1; } _ => {} }; @@ -212,8 +211,8 @@ fn modify_starknet_contract(mut program: Program, anchor_value: u32, new_value: } assert!( - anchor_counter == 1, - "CONTRACT_MODIFICATION_ANCHOR was not found exactly once" + old_value_counter == 1, + "old_value was not found exactly once" ); program From a2e010c66f75d86e38dd642c8aa54a7849903990 Mon Sep 17 00:00:00 2001 From: Julian Gonzalez Calderon Date: Fri, 14 Jun 2024 11:07:19 -0300 Subject: [PATCH 26/26] Update src/bin/cairo-native-stress.rs Co-authored-by: fmoletta <99273364+fmoletta@users.noreply.github.com> --- src/bin/cairo-native-stress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/cairo-native-stress.rs b/src/bin/cairo-native-stress.rs index b3cb2171d..d1536635e 100644 --- a/src/bin/cairo-native-stress.rs +++ b/src/bin/cairo-native-stress.rs @@ -41,7 +41,7 @@ const UNIQUE_CONTRACT_VALUE: u32 = 835; /// A stress tester for Cairo Native /// -/// It Sierra programs compiles with Cairo Native, caches, and executes them with AOT runner. +/// It compiles Sierra programs with Cairo Native, caches, and executes them with AOT runner. /// The compiled dynamic libraries are stored in `AOT_CACHE_DIR` relative to the current working directory. #[derive(Parser, Debug)] struct StressTestCommand {