Skip to content

Commit

Permalink
chore: scaffolding and docs.
Browse files Browse the repository at this point in the history
  • Loading branch information
l-monninger committed Dec 7, 2023
1 parent 8b2f377 commit 6438553
Show file tree
Hide file tree
Showing 32 changed files with 465 additions and 89 deletions.
65 changes: 62 additions & 3 deletions movement-sdk/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions movement-sdk/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,16 @@ members = [

# execution
"execution/sui-block-executor",
"execution/aptos-block-executor",
"execution/canonical-block-executor",

# types
"types/sui-helper-types",
"types/canonical-types",
"types/aptos-helper-types",

# sui helpers
"sui-helpers/sui-block-authority-providers",

# clis
"clis/movement"
Expand All @@ -28,7 +35,14 @@ rust-version = "1.70"
[workspace.dependencies]
# internal
movement-sdk = { path = "movement-sdk" }
aptos-helper-types = { path = "types/aptos-helper-types" }
sui-helper-types = { path = "types/sui-helper-types" }
canonical-types = { path = "types/canonical-types" }
sui-block-executor = { path = "execution/sui-block-executor" }
aptos-block-executor = { path = "execution/aptos-block-executor" }
canonical-block-executor = { path = "execution/canonical-block-executor" }
sui-block-authority-providers = { path = "sui-helpers/sui-block-authority-providers" }



async-trait = { version = "0.1" }
Expand Down Expand Up @@ -64,6 +78,7 @@ aptos-types = { path = "../vendors/aptos-core/types" }
aptos-executor = { path = "../vendors/aptos-core/execution/executor" }
aptos-executor-types = { path = "../vendors/aptos-core/execution/executor-types" }
aptos-storage-interface = { path = "../vendors/aptos-core/storage/storage-interface" }
aptos-crypto = { path = "../vendors/aptos-core/crates/aptos-crypto" }

# sui
sui-adapter-latest = { path = "../vendors/sui/sui-execution/latest/sui-adapter" }
Expand Down
21 changes: 21 additions & 0 deletions movement-sdk/execution/aptos-block-executor/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
[package]
name = "aptos-block-executor"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
# internal
movement-sdk = { workspace = true }

# general
anyhow = { workspace = true }
async-trait = { workspace = true }
tokio = { workspace = true }

# aptos
aptos-executor = { workspace = true }
aptos-types = { workspace = true }
aptos-vm = { workspace = true }
aptos-helper-types = { workspace = true }
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
use tokio::sync::RwLock;
use std::sync::Arc;
use aptos_executor::block_executor::BlockExecutor;
use aptos_types::transaction::Transaction;
use aptos_vm::AptosVM;
use movement_sdk::{ExecutionLayer, Layer};
use aptos_helper_types::block::Block;


#[derive(Clone)]
pub struct AptosBlockExecutor {
pub executor: Arc<RwLock<BlockExecutor<AptosVM>>>,
}

impl std::fmt::Debug for AptosBlockExecutor {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AptosBlockExecutor")
.finish()
}
}

impl AptosBlockExecutor {
pub fn new(executor: Arc<RwLock<BlockExecutor<AptosVM>>>) -> Self {
AptosBlockExecutor { executor }
}
}

impl Layer for AptosBlockExecutor {}

#[async_trait::async_trait]
impl ExecutionLayer for AptosBlockExecutor {

type Block = Block;
type BlockId = String; // todo: change later
type ChangeSet = u64; // todo: change later

// Gets the next block from the previous layer.
async fn get_next_block(
&self
) -> Result<Option<Self::Block>, anyhow::Error> {
unimplemented!();
}

// Executes a block and produces a change set.
async fn execute_block(
&self,
block: Self::Block
) -> Result<Self::ChangeSet, anyhow::Error> {

let mut executor = self.executor.write().await;
// let parent_block_id_now = executor.committed_block_id();

// todo: update for aptos 1.70
/*// execute the block
let output = executor
.execute_block((block_id, block_tx.clone()), parent_block_id)
.unwrap();
// sign for the the ledger
let ledger_info = LedgerInfo::new(
BlockInfo::new(
next_epoch,
0,
block_id,
output.root_hash(),
output.version(),
ts,
output.epoch_state().clone(),
),
HashValue::zero(),
);
let li = generate_ledger_info_with_sig(&[self.signer.as_ref().unwrap().clone()], ledger_info);
executor.commit_blocks(vec![block_id], li.clone()).unwrap();*/

Ok(0)

}

// Sends a change set to the next layer, i.e., the storage layer.
async fn send_change_set(
&self,
change_set: Self::ChangeSet
) -> Result<(), anyhow::Error> {
unimplemented!();
}

// Gets an executed block
async fn get_block(
&self,
block_id: Self::BlockId
) -> Result<Option<Self::Block>, anyhow::Error> {
unimplemented!();
}

}
1 change: 1 addition & 0 deletions movement-sdk/execution/aptos-block-executor/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub mod aptos_block_executor;
16 changes: 16 additions & 0 deletions movement-sdk/execution/canonical-block-executor/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
[package]
name = "canonical-block-executor"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
# internal
movement-sdk = { workspace = true }

# general
anyhow = { workspace = true }
async-trait = { workspace = true }
tokio = { workspace = true }
aptos-helper-types = { workspace = true }
Empty file.
Empty file.
4 changes: 3 additions & 1 deletion movement-sdk/execution/sui-block-executor/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[package]
name = "sui-block"
name = "sui-block-executor"
version = "0.1.0"
edition = "2021"

Expand All @@ -13,11 +13,13 @@ movement-sdk = { workspace = true }
anyhow = { workspace = true }
async-trait = { workspace = true }
tokio = { workspace = true }
futures = { workspace = true }

# internal
sui-helper-types = { workspace = true }
sui-types = { workspace = true }


# sui
# todo: conflicting rocksdb means we can't use workspace
# todo: likely movement-sdk will move into its own workspace
Expand Down
12 changes: 8 additions & 4 deletions movement-sdk/execution/sui-block-executor/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ The `sui-block-executor` is a block-based execution layer providing Move VM exec
Essentially, the `sui-block-executor` performs the duty of...
1. Transforming [`Vec<SenderSignedData>`](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/crates/sui-types/src/transaction.rs#L2019) into a [`Vec<VerifiedExecutableTransaction>`](https://github.com/MystenLabs/sui/blob/main/crates/sui-types/src/executable_transaction.rs#L55), and then,
2. Executing each element in said vector in deterministic order via [`execute_transaction_to_effects`](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/sui-execution/src/latest.rs#L79).
3. `execute_transaction_to_effects` both returns the `TransactionEffects` and applies them to the backing store which the `AuthorityStore` implements (by implementing the [required traits](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/crates/sui-types/src/storage/mod.rs#L489)). Take a look here to see for example how they are not used directly in the [`AuthorityStore`'s execution_driver](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/crates/sui-core/src/execution_driver.rs#L103). However, there's additional complexity here that is addressed below with which we are not yet sure whether we will need to deal.
3. `execute_transaction_to_effects` both returns the `TransactionEffects` and applies them to a temporary store within the `AuthorityStore` (which implements `BackingStore` via [required traits](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/crates/sui-types/src/storage/mod.rs#L489)). These effects must be committed--which is discussed further below.

> Perhaps the most analogous path in the existing Sui source to what needs to be developed is the [`process_certificate` method from the `AuthorityStore`](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/crates/sui-core/src/authority.rs#L1339). This is a good point of reference for many parts of this project, hence it is included in the reading material.
Expand Down Expand Up @@ -48,7 +48,7 @@ fn execute_transaction_to_effects(
```

- `&dyn BackingStore`: arguably the most important of the parameters. We essentially want to do this over RocksDB on our own. There may be several crates for this developed over time, but `canonical-sui-backing-store` is where we will start.
- `protocol_config`: this is struct pops up all over the Sui crate and was originally a big part of the cause for hesitancy in implementing the Sui block-executor at this level--owing to the presumed entanglement of consensus in parts of the object runtime specifically attributable to this config. However, it turns out that consensus parameters are not in fact used. Instead, what this is [used for at this level](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/sui-execution/v1/sui-adapter/src/programmable_transactions/execution.rs#L639) are things such as...
- `protocol_config`: this is a struct that pops up all over the Sui crate and was originally a big part of the cause for hesitancy in implementing the Sui block-executor at this level--owing to the presumed entanglement of consensus in parts of the object runtime specifically attributable to this config. However, it turns out that consensus parameters are not in fact used. Instead, what this is [used for at this level](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/sui-execution/v1/sui-adapter/src/programmable_transactions/execution.rs#L639) are things such as...
- shared object deletion settings,
- Move binary format settings,
- Max package size.
Expand All @@ -68,15 +68,19 @@ fn execute_transaction_to_effects(
### Dealing with `TransactionEffects`
The [`AuthorityPerEpochStore`](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/crates/sui-core/src/authority/authority_per_epoch_store.rs#L628) is really the engine of a lot of how Sui execution comes together for the Sui network. However, it unfortunately is also very entangled and asynchronous.

Though `TransactionEffects` are applied via calling `execute_transaction_to_effects` the `AuthorityPerEpochStore` relies on these effects to inform processes which [affect execution within and between epochs](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/crates/sui-core/src/authority/authority_per_epoch_store.rs#L1172). For the most part, it seems like we can ignore this as we are not concerned with consensus details. However, a second glance should be made at the `AuthorityPerEpochStore` whenever it appears something is not working correctly concerning issues related to the absence, presence, etc. of checkpoint and epoch indicators.
The effects are then asynchronously committed to the epoch store using the returned [`TransactionEffects`](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/crates/sui-core/src/authority.rs#L1213) and later make their way into persistent state. This process involves a lot of interdependencies in Sui. However, we do not necessarily have to implement this directly. Take a look here to see for example how they are not used directly in the [`AuthorityStore`'s execution_driver](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/crates/sui-core/src/execution_driver.rs#L103). However, there's additional complexity here that is addressed below with which we are not yet sure whether we will need to deal.

To accomplish this, we must build a subsystem [similar to](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/crates/sui-core/src/authority.rs#L1213). However, ours can be synchronous.

This is an ongoing area of research.

## Sui Object Runtime
`execute_transaction_to_effects` takes care of the Sui object runtime provided, so long as you are constructing the executor with a MoveVM similar to that produced by [`new_move_vm`](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/sui-execution/src/latest.rs#L55C35-L55C35). In fact `new_move_vm` gives a good entry point for extending the function table, which should be enough for the appropriate integration.

## Sui-like Concurrency
We need to deterministically group transactions into execution groups that can be safely and maximally executed in parallel based on their sets of shared objects.

Obtaining shared objects can be accomplished via the transaction directly.
Obtaining shared objects can be accomplished via the [transaction data directly](https://github.com/MystenLabs/sui/blob/552158d9eae200314499809d8977f732f6c2cee7/crates/sui-core/src/authority.rs#L963C78-L963C78).

The simplest way to solve this problem is to brute-force through possible groupings in transaction order which is $O(n^2 * ||S||)$. However, practically the sizes of the sets will be much smaller than $||S||$ and the number of transactions in a block should be on the order of 100s.

Expand Down
Loading

0 comments on commit 6438553

Please sign in to comment.