Fluffy portal testnet support (#2383)

* Bump portal-mainnet repo. * Update command line arguments and parsing on startup. * Read in angelfood bootstrap nodes and update Fluffy guide. * Configure subnetwork protocol ids.
status-im · Jun 18, 2024 · e3d14bd · e3d14bd
1 parent 8926da0
commit e3d14bd
Show file tree

Hide file tree

Showing 16 changed files with 133 additions and 72 deletions.
diff --git a/fluffy/conf.nim b/fluffy/conf.nim
@@ -56,17 +56,6 @@ type
   PortalCmd* = enum
     noCommand
 
-  PortalNetwork* = enum
-    none
-    mainnet
-    testnet
-
-  # The networks alias Portal sub-protocols
-  Network* = enum
-    beacon
-    history
-    state
-
   PortalConf* = object
     logLevel* {.
       desc:
@@ -94,27 +83,37 @@ type
       name: "listen-address"
     .}: IpAddress
 
-    portalNetwork* {.
+    portalNetworkDeprecated* {.
+      hidden,
+      desc:
+        "DEPRECATED: The --portal-network flag will be removed in the future, " &
+        "please use the drop in replacement --network flag instead",
+      defaultValue: none(PortalNetwork),
+      name: "portal-network"
+    .}: Option[PortalNetwork]
+
+    network* {.
       desc:
         "Select which Portal network to join. This will set the " &
         "Portal network specific bootstrap nodes automatically",
       defaultValue: PortalNetwork.mainnet,
-      name: "portal-network"
+      name: "network"
     .}: PortalNetwork
 
-    portalNetworkDeprecated* {.
+    networksDeprecated* {.
       hidden,
       desc:
-        "DEPRECATED: The --network flag will be removed in the future, please use the drop in replacement --portal-network flag instead",
-      defaultValue: none(PortalNetwork),
-      name: "network"
-    .}: Option[PortalNetwork.mainnet]
+        "DEPRECATED: The --networks flag will be removed in the future, " &
+        "please use the drop in replacement --portal-subnetworks flag instead",
+      defaultValue: {},
+      name: "networks"
+    .}: set[PortalSubnetwork]
 
-    networks* {.
+    portalSubnetworks* {.
       desc: "Select which networks (Portal sub-protocols) to enable",
-      defaultValue: {Network.history},
-      name: "networks"
-    .}: set[Network]
+      defaultValue: {PortalSubnetwork.history},
+      name: "portal-subnetworks"
+    .}: set[PortalSubnetwork]
 
     # Note: This will add bootstrap nodes for both Discovery v5 network and each
     # enabled Portal network. No distinction is made on bootstrap nodes per
@@ -363,21 +362,23 @@ proc parseCmdArg*(T: type ClientConfig, p: string): T {.raises: [ValueError].} =
 proc completeCmdArg*(T: type ClientConfig, val: string): seq[string] =
   return @[]
 
-proc parseCmdArg*(T: type set[Network], p: string): T {.raises: [ValueError].} =
-  var res: set[Network] = {}
+proc parseCmdArg*(
+    T: type set[PortalSubnetwork], p: string
+): T {.raises: [ValueError].} =
+  var res: set[PortalSubnetwork] = {}
   let values = p.split({' ', ','})
   for value in values:
     let stripped = value.strip()
     let network =
       try:
-        parseEnum[Network](stripped)
+        parseEnum[PortalSubnetwork](stripped)
       except ValueError:
         raise newException(ValueError, "Invalid network: " & stripped)
 
     res.incl(network)
   res
 
-proc completeCmdArg*(T: type set[Network], val: string): seq[string] =
+proc completeCmdArg*(T: type set[PortalSubnetwork], val: string): seq[string] =
   return @[]
 
 chronicles.formatIt(InputDir):

diff --git a/fluffy/docs/the_fluffy_book/docs/connect-to-portal.md b/fluffy/docs/the_fluffy_book/docs/connect-to-portal.md
@@ -6,15 +6,14 @@ Connecting to the current Portal network is as easy as running following command
 ./build/fluffy --rpc
 ```
 
-This will connect to the public [Portal testnet](https://github.com/ethereum/portal-network-specs/blob/master/testnet.md#portal-network-testnet)
-which contains nodes of the different clients.
+This will connect to the public Portal mainnet which contains nodes of the different clients.
 
 !!! note
-    Default the Fluffy node will connect to the
-    [bootstrap nodes](https://github.com/ethereum/portal-network-specs/blob/master/testnet.md#bootnodes) of the public testnet.
+    By default the Fluffy node will connect to the
+    [bootstrap nodes](https://github.com/ethereum/portal-network-specs/blob/master/bootnodes.md#bootnodes-mainnet) of the public mainnet.
 
-    When testing locally the `--portal-network:none` option can be provided to avoid
-    connecting to any of the testnet bootstrap nodes.
+    When testing locally the `--network:none` option can be provided to avoid
+    connecting to any of the default bootstrap nodes.
 
 The `--rpc` option will also enable the different JSON-RPC interfaces through
 which you can access the Portal Network.
@@ -32,4 +31,3 @@ Fluffy also supports a small subset of the [Execution JSON-RPC API](https://ethe
 
 <!-- TODO: Explain some of the more important cli options here? Or in a separate
 page? -->
-
diff --git a/fluffy/fluffy.nim b/fluffy/fluffy.nim
@@ -35,7 +35,7 @@ import
   ./network/state/[state_network, state_content],
   ./network/history/[history_network, history_content],
   ./network/beacon/[beacon_init_loader, beacon_light_client],
-  ./network/wire/[portal_stream, portal_protocol_config],
+  ./network/wire/[portal_stream, portal_protocol_config, portal_protocol],
   ./eth_data/history_data_ssz_e2s,
   ./database/content_db,
   ./version,
@@ -101,24 +101,35 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
   loadBootstrapFile(string config.bootstrapNodesFile, bootstrapRecords)
   bootstrapRecords.add(config.bootstrapNodes)
 
-  var portalNetwork: PortalNetwork
-  if config.portalNetworkDeprecated.isSome():
-    warn "DEPRECATED: The --network flag will be removed in the future, please use the drop in replacement --portal-network flag instead"
-    portalNetwork = config.portalNetworkDeprecated.get()
-  else:
-    portalNetwork = config.portalNetwork
+  let portalNetwork =
+    if config.portalNetworkDeprecated.isNone():
+      config.network
+    else:
+      warn "DEPRECATED: The --portal-network flag will be removed in the future, " &
+        "please use the drop in replacement --network flag instead"
+      config.portalNetworkDeprecated.get()
+
+  let portalSubnetworks =
+    if config.networksDeprecated == {}:
+      config.portalSubnetworks
+    else:
+      warn "DEPRECATED: The --networks flag will be removed in the future, " &
+        "please use the drop in replacement --portal-subnetworks flag instead"
+      config.networksDeprecated
 
   case portalNetwork
-  of mainnet:
+  of PortalNetwork.none:
+    discard # don't connect to any network bootstrap nodes
+  of PortalNetwork.mainnet:
     for enrURI in mainnetBootstrapNodes:
       var record: Record
       if fromURI(record, enrURI):
         bootstrapRecords.add(record)
-  of testnet:
-    # TODO: add testnet repo with bootstrap file.
-    discard
-  else:
-    discard
+  of PortalNetwork.angelfood:
+    for enrURI in angelfoodBootstrapNodes:
+      var record: Record
+      if fromURI(record, enrURI):
+        bootstrapRecords.add(record)
 
   let
     discoveryConfig =
@@ -210,9 +221,10 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
         loadAccumulator()
 
     historyNetwork =
-      if Network.history in config.networks:
+      if PortalSubnetwork.history in portalSubnetworks:
         Opt.some(
           HistoryNetwork.new(
+            portalNetwork,
             d,
             db,
             streamManager,
@@ -225,9 +237,10 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
         Opt.none(HistoryNetwork)
 
     stateNetwork =
-      if Network.state in config.networks:
+      if PortalSubnetwork.state in portalSubnetworks:
         Opt.some(
           StateNetwork.new(
+            portalNetwork,
             d,
             db,
             streamManager,
@@ -243,12 +256,15 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
     beaconLightClient =
       # TODO: Currently disabled by default as it is not sufficiently polished.
       # Eventually this should be always-on functionality.
-      if Network.beacon in config.networks and config.trustedBlockRoot.isSome():
+      if PortalSubnetwork.beacon in portalSubnetworks and
+          config.trustedBlockRoot.isSome():
         let
           # Portal works only over mainnet data currently
+          # TODO: investigate this load network data function
           networkData = loadNetworkData("mainnet")
           beaconDb = BeaconDb.new(networkData, config.dataDir / "db" / "beacon_db")
           beaconNetwork = BeaconNetwork.new(
+            portalNetwork,
             d,
             beaconDb,
             streamManager,

diff --git a/fluffy/network/beacon/beacon_network.nim b/fluffy/network/beacon/beacon_network.nim
@@ -23,8 +23,6 @@ export beacon_content, beacon_db
 logScope:
   topics = "beacon_network"
 
-const lightClientProtocolId* = [byte 0x50, 0x1A]
-
 type BeaconNetwork* = ref object
   portalProtocol*: PortalProtocol
   beaconDb*: BeaconDb
@@ -183,6 +181,7 @@ proc getHistoricalSummaries*(
 
 proc new*(
     T: type BeaconNetwork,
+    portalNetwork: PortalNetwork,
     baseProtocol: protocol.Protocol,
     beaconDb: BeaconDb,
     streamManager: StreamManager,
@@ -206,7 +205,7 @@ proc new*(
 
     portalProtocol = PortalProtocol.new(
       baseProtocol,
-      lightClientProtocolId,
+      getProtocolId(portalNetwork, PortalSubnetwork.beacon),
       toContentIdHandler,
       createGetHandler(beaconDb),
       stream,

diff --git a/fluffy/network/history/history_network.nim b/fluffy/network/history/history_network.nim
@@ -47,8 +47,6 @@ export accumulator
 proc `$`(x: BlockHeader): string =
   $x
 
-const historyProtocolId* = [byte 0x50, 0x0B]
-
 type
   HistoryNetwork* = ref object
     portalProtocol*: PortalProtocol
@@ -703,6 +701,7 @@ proc validateContent(
 
 proc new*(
     T: type HistoryNetwork,
+    portalNetwork: PortalNetwork,
     baseProtocol: protocol.Protocol,
     contentDB: ContentDB,
     streamManager: StreamManager,
@@ -718,7 +717,7 @@ proc new*(
 
     portalProtocol = PortalProtocol.new(
       baseProtocol,
-      historyProtocolId,
+      getProtocolId(portalNetwork, PortalSubnetwork.history),
       toContentIdHandler,
       createGetHandler(contentDB),
       stream,

diff --git a/fluffy/network/network_seed.nim b/fluffy/network/network_seed.nim
@@ -33,8 +33,7 @@ const
   #TODO currently we are using value for history network, but this should be
   #caluculated per netowork basis
   maxItemsPerOfferBySize = getMaxOfferedContentKeys(
-    uint32(len(history_network.historyProtocolId)),
-    uint32(history_content.maxContentKeySize),
+    uint32(len(PortalProtocolId)), uint32(history_content.maxContentKeySize)
   )
 
   # Offering is restricted to max 64 items

diff --git a/fluffy/network/state/state_network.nim b/fluffy/network/state/state_network.nim
@@ -26,8 +26,6 @@ export results, state_content
 logScope:
   topics = "portal_state"
 
-const stateProtocolId* = [byte 0x50, 0x0A]
-
 type StateNetwork* = ref object
   portalProtocol*: PortalProtocol
   contentDB*: ContentDB
@@ -41,6 +39,7 @@ func toContentIdHandler(contentKey: ByteList): results.Opt[ContentId] =
 
 proc new*(
     T: type StateNetwork,
+    portalNetwork: PortalNetwork,
     baseProtocol: protocol.Protocol,
     contentDB: ContentDB,
     streamManager: StreamManager,
@@ -55,7 +54,7 @@ proc new*(
 
   let portalProtocol = PortalProtocol.new(
     baseProtocol,
-    stateProtocolId,
+    getProtocolId(portalNetwork, PortalSubnetwork.state),
     toContentIdHandler,
     createGetHandler(contentDB),
     s,

diff --git a/fluffy/network/wire/portal_protocol.nim b/fluffy/network/wire/portal_protocol.nim
@@ -254,6 +254,41 @@ func init*(
     nodesInterestedInContent: nodesInterestedInContent,
   )
 
+func getProtocolId*(
+    network: PortalNetwork, subnetwork: PortalSubnetwork
+): PortalProtocolId =
+  const portalPrefix = byte(0x50)
+
+  case network
+  of PortalNetwork.none, PortalNetwork.mainnet:
+    case subnetwork
+    of PortalSubnetwork.state:
+      [portalPrefix, 0x0A]
+    of PortalSubnetwork.history:
+      [portalPrefix, 0x0B]
+    of PortalSubnetwork.beacon:
+      [portalPrefix, 0x0C]
+    of PortalSubnetwork.transactionIndex:
+      [portalPrefix, 0x0D]
+    of PortalSubnetwork.verkleState:
+      [portalPrefix, 0x0E]
+    of PortalSubnetwork.transactionGossip:
+      [portalPrefix, 0x0F]
+  of PortalNetwork.angelfood:
+    case subnetwork
+    of PortalSubnetwork.state:
+      [portalPrefix, 0x4A]
+    of PortalSubnetwork.history:
+      [portalPrefix, 0x4B]
+    of PortalSubnetwork.beacon:
+      [portalPrefix, 0x4C]
+    of PortalSubnetwork.transactionIndex:
+      [portalPrefix, 0x4D]
+    of PortalSubnetwork.verkleState:
+      [portalPrefix, 0x4E]
+    of PortalSubnetwork.transactionGossip:
+      [portalPrefix, 0x4F]
+
 func `$`(id: PortalProtocolId): string =
   id.toHex()
 

diff --git a/fluffy/network/wire/portal_protocol_config.nim b/fluffy/network/wire/portal_protocol_config.nim
@@ -10,6 +10,20 @@
 import std/strutils, confutils, chronos, stint, eth/p2p/discoveryv5/routing_table
 
 type
+  PortalNetwork* = enum
+    none
+    mainnet
+    angelfood
+
+  # The Portal sub-protocols
+  PortalSubnetwork* = enum
+    state
+    history
+    beacon
+    transactionIndex
+    verkleState
+    transactionGossip
+
   RadiusConfigKind* = enum
     Static
     Dynamic

diff --git a/fluffy/network_metadata.nim b/fluffy/network_metadata.nim
@@ -44,6 +44,8 @@ const
   # rlp.rawData() in the enr code.
   mainnetBootstrapNodes* =
     loadCompileTimeBootstrapNodes(portalConfigDir / "bootstrap_nodes.txt")
+  angelfoodBootstrapNodes* =
+    loadCompileTimeBootstrapNodes(portalConfigDir / "bootstrap_nodes_angelfood.txt")
 
   finishedAccumulatorSSZ* = slurp(portalConfigDir / "finished_accumulator.ssz")