Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/multi client #32

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+167 −11
Open

Feat/multi client #32

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3aa75d7
chore: geo_runs
namn-grg Aug 31, 2024
9b68562
feat: add 2 client support
namn-grg Aug 31, 2024
30bf2bd
chore: download both client logs
namn-grg Aug 31, 2024
ac3d61a
feat: add results
namn-grg Sep 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 58 additions & 1 deletion benchmark/geo_runs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,10 +425,67 @@ def update_batch_size(json_file_path, mechanism_name, new_batch_size):
except Exception as e:
print(f"An error occurred: {e}")

# def update_parameters(json_file_path, mechanism_name, new_tx_size=None, new_rate=None, new_batch_size=None):
# try:
# with open(json_file_path, "r") as file:
# data = json.load(file)

# if mechanism_name in data["remote"]:
# # Update tx_size
# if new_tx_size is not None:
# data["remote"][mechanism_name]["bench_params"]["tx_size"] = new_tx_size

# # Update rate
# if new_rate is not None:
# if isinstance(new_rate, list):
# data["remote"][mechanism_name]["bench_params"]["rate"] = new_rate
# else:
# data["remote"][mechanism_name]["bench_params"]["rate"] = [new_rate]

# # Update batch_size
# if new_batch_size is not None:
# if mechanism_name == "hotstuff":
# data["remote"][mechanism_name]["node_params"]["mempool"]["batch_size"] = new_batch_size
# elif mechanism_name == "bullshark" or mechanism_name == "cometbft":
# data["remote"][mechanism_name]["node_params"]["batch_size"] = new_batch_size

# with open(json_file_path, "w") as file:
# json.dump(data, file, indent=4)
# print(f"Parameters for {mechanism_name} updated successfully.")
# else:
# print(f"Mechanism {mechanism_name} not found in the JSON file.")

# except Exception as e:
# print(f"An error occurred: {e}")

# def update_batch_size_relative(json_file_path, mechanism_name, relative_tx_rate):
# try:
# with open(json_file_path, "r") as file:
# data = json.load(file)

# if mechanism_name in data["remote"]:
# if mechanism_name == "hotstuff":
# tx_rate = data["remote"][mechanism_name]["bench_params"]["rate"][0]
# tx_size = data["remote"][mechanism_name]["bench_params"]["tx_size"]
# data["remote"][mechanism_name]["node_params"]["mempool"]["batch_size"] = int(tx_size) * (int(tx_rate) + int(relative_tx_rate))
# elif mechanism_name == "bullshark":
# data["remote"][mechanism_name]["node_params"]["batch_size"] = relative_tx_rate
# with open(json_file_path, "w") as file:
# json.dump(data, file, indent=4)
# print(f"Batch count for {mechanism_name} updated to {tx_rate + relative_tx_rate}.")
# else:
# print(f"Mechanism {mechanism_name} not found in the JSON file.")

# except Exception as e:
# print(f"An error occurred: {e}")


if __name__ == "__main__":

batch_sizes = [512, 1024, 10000, 20000, 50000, 80000, 100000, 200000, 300000]
# batch_sizes = [512, 1024, 10000, 20000, 50000, 80000, 100000, 200000, 300000]
batch_sizes = [1000, 5000, 10000, 100000, 1000000, 5000000, 7500000, 10000000, 12500000, 15000000]
# batch_sizes = [-100, 100, -10, 0, 10, 20, 30, 40, 50, 70]

mechanism = ["hotstuff"]

print("Starting benchmarking tool")
Expand Down
29 changes: 23 additions & 6 deletions benchmark/remote.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,19 +249,35 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals
# for the faulty nodes to be online).
committee = Committee.load(PathMaker.committee_file())
addresses = [f'{x}:{self.settings.ports["front"]}' for x in hosts]
rate_share = ceil(rate / committee.size()) # Take faults into account.
rate_share = ceil(rate / (committee.size() * 2)) # Take faults into account.
timeout = node_parameters.timeout_delay
client_logs = [PathMaker.client_log_file(i) for i in range(len(hosts))]
for host, addr, log_file in zip(hosts, addresses, client_logs):
cmd = CommandMaker.run_client(

# Updated client logs for two clients per node
client_logs_a = [PathMaker.client_log_file(f"{i}-a") for i in range(len(hosts))]
client_logs_b = [PathMaker.client_log_file(f"{i}-b") for i in range(len(hosts))]

for host, addr, log_file_a, log_file_b in zip(hosts, addresses, client_logs_a, client_logs_b):
# Run first client (client-{i}-a)
cmd_a = CommandMaker.run_client(
addr,
bench_parameters.tx_size,
rate_share,
self.mechanism.name,
timeout,
nodes=addresses,
)
self._background_run(host, cmd, log_file)
self._background_run(host, cmd_a, log_file_a)

# Run second client (client-{i}-b)
cmd_b = CommandMaker.run_client(
addr,
bench_parameters.tx_size,
rate_share,
self.mechanism.name,
timeout,
nodes=addresses,
)
self._background_run(host, cmd_b, log_file_b)

# Run the nodes.
key_files = [PathMaker.key_file(i) for i in range(len(hosts))]
Expand Down Expand Up @@ -411,7 +427,8 @@ def _logs(self, hosts, faults, committee=[]): # , servers, run_id):
for i, host in enumerate(progress):
c = Connection(host, user=self.settings.key_name, connect_kwargs=self.connect)
c.get(PathMaker.node_log_file(i), local=PathMaker.node_log_file(i))
c.get(PathMaker.client_log_file(i), local=PathMaker.client_log_file(i))
c.get(PathMaker.client_log_file(f"{i}-a"), local=PathMaker.client_log_file(f"{i}-a"))
c.get(PathMaker.client_log_file(f"{i}-b"), local=PathMaker.client_log_file(f"{i}-b"))
if self.mechanism.name == "cometbft":
c.get(
PathMaker.latency_log_file(i),
Expand Down
2 changes: 1 addition & 1 deletion benchmark/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def node_log_file(i):

@staticmethod
def client_log_file(i):
assert isinstance(i, int) and i >= 0
# assert isinstance(i, int) and i >= 0
return join(PathMaker.logs_path(), f'client-{i}.log')

@staticmethod
Expand Down
6 changes: 3 additions & 3 deletions fab-params.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
4
],
"rate": [
200000
160000
],
"tx_size": 128,
"duration": 300,
Expand All @@ -22,8 +22,8 @@
"gc_depth": 50,
"sync_retry_delay": 5000,
"sync_retry_nodes": 3,
"batch_size": 300000,
"max_batch_delay": 100
"batch_size": 15000000,
"max_batch_delay": 5000
}
}
},
Expand Down
82 changes: 82 additions & 0 deletions results/metrics.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -683,3 +683,85 @@ run_id,name,faults,input_rate,committee_size,transaction_size,execution_time,bat
682,hotstuff,0.0,80000.0,4.0,128.0,3.0,15000000.0,1527777.0,195555483.0,36.0,18556.0,2375169.0,1608.0,
683,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
684,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,509259.0,65185161.0,12.0,6185.333333333333,791723.0,536.0,3.0
685,hotstuff,0.0,80000.0,4.0,128.0,299.0,7500000.0,80409.0,10292323.0,221.0,79807.0,10215267.0,2166.0,
686,hotstuff,0.0,160000.0,4.0,128.0,230.0,7500000.0,151399.0,19379072.0,1153.0,150213.0,19227306.0,3779.0,
687,hotstuff,0.0,160000.0,4.0,128.0,175.0,7500000.0,94760.0,12129221.0,21087.0,93641.0,11986029.0,28540.0,
688,hotstuff,0.0,160000.0,4.0,128.0,12.0,7500000.0,174740.0,22366695.0,396.0,141324.0,18089491.0,2960.0,
689,hotstuff,0.0,160000.0,4.0,128.0,230.0,7500000.0,140299.66666666666,17958329.333333332,7545.333333333333,128392.66666666669,16434275.333333334,11759.666666666666,3.0
690,hotstuff,0.0,80000.0,4.0,128.0,126.0,1000.0,39465.0,5051479.0,5675.0,39461.0,5051038.0,8319.0,
691,hotstuff,0.0,80000.0,4.0,128.0,181.0,1000.0,69051.0,8838542.0,2714.0,69045.0,8837761.0,4227.0,
692,hotstuff,0.0,80000.0,4.0,128.0,90.0,1000.0,30433.0,3895426.0,7108.0,30429.0,3894907.0,9650.0,
693,hotstuff,0.0,80000.0,4.0,128.0,126.0,1000.0,46316.333333333336,5928482.333333333,5165.666666666667,46311.66666666666,5927902.0,7398.666666666667,3.0
694,hotstuff,0.0,80000.0,4.0,128.0,301.0,5000.0,80007.0,10240851.0,41.0,80007.0,10240885.0,180.0,
695,hotstuff,0.0,80000.0,4.0,128.0,301.0,5000.0,79998.0,10239710.0,50.0,79996.0,10239540.0,177.0,
696,hotstuff,0.0,80000.0,4.0,128.0,299.0,5000.0,79988.0,10238512.0,28.0,79985.0,10238136.0,162.0,
697,hotstuff,0.0,80000.0,4.0,128.0,301.0,5000.0,79997.66666666667,10239691.0,39.66666666666666,79996.0,10239520.333333334,173.0,3.0
698,hotstuff,0.0,80000.0,4.0,128.0,301.0,10000.0,80057.0,10247240.0,19.0,80006.0,10240804.0,185.0,
699,hotstuff,0.0,80000.0,4.0,128.0,300.0,10000.0,79995.0,10239391.0,11.0,79993.0,10239153.0,75.0,
700,hotstuff,0.0,80000.0,4.0,128.0,300.0,10000.0,80010.0,10241257.0,13.0,80008.0,10241052.0,126.0,
701,hotstuff,0.0,80000.0,4.0,128.0,301.0,10000.0,80020.66666666667,10242629.333333334,14.333333333333334,80002.33333333333,10240336.333333334,128.66666666666666,3.0
702,hotstuff,0.0,80000.0,4.0,128.0,301.0,100000.0,80002.0,10240314.0,7.0,79999.0,10239871.0,93.0,
703,hotstuff,0.0,80000.0,4.0,128.0,301.0,100000.0,80008.0,10241087.0,6.0,80005.0,10240610.0,76.0,
704,hotstuff,0.0,80000.0,4.0,128.0,301.0,100000.0,80004.0,10240532.0,10.0,80001.0,10240157.0,133.0,
705,hotstuff,0.0,80000.0,4.0,128.0,301.0,100000.0,80004.66666666667,10240644.333333334,7.666666666666667,80001.66666666667,10240212.666666666,100.66666666666669,3.0
706,hotstuff,0.0,80000.0,4.0,128.0,300.0,1000000.0,80086.0,10250983.0,11.0,79983.0,10237814.0,268.0,
707,hotstuff,0.0,80000.0,4.0,128.0,300.0,1000000.0,80080.0,10250265.0,15.0,79981.0,10237575.0,367.0,
708,hotstuff,0.0,80000.0,4.0,128.0,300.0,1000000.0,80088.0,10251256.0,17.0,79985.0,10238120.0,425.0,
709,hotstuff,0.0,80000.0,4.0,128.0,300.0,1000000.0,80084.66666666667,10250834.666666666,14.333333333333334,79983.0,10237836.333333334,353.3333333333333,3.0
710,hotstuff,0.0,80000.0,4.0,128.0,298.0,5000000.0,80397.0,10290810.0,51.0,79813.0,10216111.0,1265.0,
711,hotstuff,0.0,80000.0,4.0,128.0,299.0,5000000.0,80503.0,10304377.0,154.0,79937.0,10231986.0,1729.0,
712,hotstuff,0.0,80000.0,4.0,128.0,299.0,5000000.0,80511.0,10305418.0,68.0,79953.0,10233937.0,1335.0,
713,hotstuff,0.0,80000.0,4.0,128.0,298.0,5000000.0,80470.33333333333,10300201.666666666,91.0,79901.0,10227344.666666666,1443.0,3.0
714,hotstuff,0.0,80000.0,4.0,128.0,299.0,7500000.0,80739.0,10334530.0,121.0,79903.0,10227556.0,2254.0,
715,hotstuff,0.0,80000.0,4.0,128.0,300.0,7500000.0,80401.0,10291293.0,97.0,79561.0,10183851.0,2087.0,
716,hotstuff,0.0,80000.0,4.0,128.0,299.0,7500000.0,80731.0,10333588.0,170.0,79896.0,10226633.0,2446.0,
717,hotstuff,0.0,80000.0,4.0,128.0,299.0,7500000.0,80623.66666666667,10319803.666666666,129.33333333333334,79786.66666666667,10212680.0,2262.333333333333,3.0
718,hotstuff,0.0,80000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
719,hotstuff,0.0,80000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
720,hotstuff,0.0,80000.0,4.0,128.0,3.0,10000000.0,2307690.0,295384355.0,26.0,18541.0,2373300.0,1760.0,
721,hotstuff,0.0,80000.0,4.0,128.0,0.0,10000000.0,769230.0,98461451.66666669,8.666666666666666,6180.333333333333,791100.0,586.6666666666666,3.0
722,hotstuff,0.0,80000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0,
723,hotstuff,0.0,80000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0,
724,hotstuff,0.0,80000.0,4.0,128.0,3.0,12500000.0,1999997.0,255999613.0,29.0,18471.0,2364331.0,1702.0,
725,hotstuff,0.0,80000.0,4.0,128.0,0.0,12500000.0,666665.6666666666,85333204.33333333,9.666666666666666,6157.0,788110.3333333334,567.3333333333334,3.0
726,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
727,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
728,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
729,hotstuff,0.0,80000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0,3.0
730,hotstuff,0.0,160000.0,4.0,128.0,20.0,1000.0,108427.0,13878697.0,1156.0,108358.0,13869837.0,1361.0,
731,hotstuff,0.0,160000.0,4.0,128.0,25.0,1000.0,79871.0,10223544.0,4682.0,79814.0,10216230.0,5551.0,
732,hotstuff,0.0,160000.0,4.0,128.0,57.0,1000.0,28409.0,3636357.0,9338.0,28406.0,3635910.0,9966.0,
733,hotstuff,0.0,160000.0,4.0,128.0,20.0,1000.0,72235.66666666667,9246199.333333334,5058.666666666667,72192.66666666667,9240659.0,5626.0,3.0
734,hotstuff,0.0,160000.0,4.0,128.0,300.0,5000.0,159981.0,20477529.0,28.0,159978.0,20477120.0,94.0,
735,hotstuff,0.0,160000.0,4.0,128.0,227.0,5000.0,118457.0,15162517.0,6577.0,118454.0,15162116.0,10296.0,
736,hotstuff,0.0,160000.0,4.0,128.0,300.0,5000.0,128076.0,16393719.0,2893.0,128074.0,16393445.0,4505.0,
737,hotstuff,0.0,160000.0,4.0,128.0,300.0,5000.0,135504.66666666666,17344588.333333332,3166.0,135502.0,17344227.0,4965.0,3.0
738,hotstuff,0.0,160000.0,4.0,128.0,300.0,10000.0,159963.0,20475215.0,246.0,159920.0,20469751.0,985.0,
739,hotstuff,0.0,160000.0,4.0,128.0,301.0,100000.0,160010.0,20481312.0,20.0,160004.0,20480562.0,180.0,
740,hotstuff,0.0,160000.0,4.0,128.0,300.0,100000.0,159976.0,20476964.0,38.0,159970.0,20476145.0,265.0,
741,hotstuff,0.0,160000.0,4.0,128.0,300.0,100000.0,160006.0,20480792.0,8.0,159999.0,20479836.0,37.0,
742,hotstuff,0.0,160000.0,4.0,128.0,301.0,100000.0,159997.33333333334,20479689.33333333,22.0,159991.0,20478847.666666668,160.66666666666666,3.0
743,hotstuff,0.0,160000.0,4.0,128.0,300.0,1000000.0,160039.0,20485029.0,26.0,159941.0,20472477.0,333.0,
744,hotstuff,0.0,160000.0,4.0,128.0,300.0,1000000.0,160076.0,20489712.0,22.0,159969.0,20475987.0,276.0,
745,hotstuff,0.0,160000.0,4.0,128.0,300.0,1000000.0,160081.0,20490395.0,16.0,159975.0,20476738.0,184.0,
746,hotstuff,0.0,160000.0,4.0,128.0,300.0,1000000.0,160065.33333333334,20488378.666666668,21.33333333333333,159961.66666666666,20475067.33333333,264.3333333333333,3.0
747,hotstuff,0.0,160000.0,4.0,128.0,299.0,5000000.0,158592.0,20299779.0,154.0,157999.0,20223857.0,1145.0,
748,hotstuff,0.0,160000.0,4.0,128.0,300.0,5000000.0,160359.0,20525906.0,183.0,159798.0,20454111.0,1128.0,
749,hotstuff,0.0,160000.0,4.0,128.0,291.0,5000000.0,159055.0,20359059.0,353.0,158450.0,20281655.0,1868.0,
750,hotstuff,0.0,160000.0,4.0,128.0,299.0,5000000.0,159335.33333333334,20394914.666666668,230.0,158749.0,20319874.33333333,1380.3333333333333,3.0
751,hotstuff,0.0,160000.0,4.0,128.0,300.0,7500000.0,157147.0,20114816.0,502.0,156269.0,20002489.0,2531.0,
752,hotstuff,0.0,160000.0,4.0,128.0,299.0,7500000.0,148600.0,19020780.0,663.0,147781.0,18916002.0,2529.0,
753,hotstuff,0.0,160000.0,4.0,128.0,301.0,7500000.0,160084.0,20490794.0,221.0,159176.0,20374491.0,1515.0,
754,hotstuff,0.0,160000.0,4.0,128.0,300.0,7500000.0,155277.0,19875463.33333333,462.0,154408.66666666666,19764327.33333333,2191.6666666666665,3.0
755,hotstuff,0.0,160000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
756,hotstuff,0.0,160000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
757,hotstuff,0.0,160000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
758,hotstuff,0.0,160000.0,4.0,128.0,0.0,10000000.0,0.0,0.0,0.0,0.0,0.0,0.0,3.0
759,hotstuff,0.0,160000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0,
760,hotstuff,0.0,160000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0,
761,hotstuff,0.0,160000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0,
762,hotstuff,0.0,160000.0,4.0,128.0,0.0,12500000.0,0.0,0.0,0.0,0.0,0.0,0.0,3.0
763,hotstuff,0.0,160000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
764,hotstuff,0.0,160000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
765,hotstuff,0.0,160000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0,
766,hotstuff,0.0,160000.0,4.0,128.0,0.0,15000000.0,0.0,0.0,0.0,0.0,0.0,0.0,3.0