Added integration tests on htlc relaying in on-chain scenarios (ACINQ#51

)

* using scodec for failure messages

* we now update routing info and retry payments in case of Update failure

* properly implemented BOLT4/'Receiving Failure Codes' (also fixed a bug with UnknownPaymentHash)

* added integration tests on htlc timeout

* relayer now sends an UpdateFail to downstream when an htlc timeout hits the blockchain

* added integration tests on extracting preimage from blockchain

* cleaning up of htlc bindings in relayer

* router now doesn't send a 2nd request to bitcoind when it is notified of a new channel twice in a short amount of time

* peerclient now asks for witness when retrieving blocks from bitcoind

* getTxBlockHash now returns None instead of an error when tx is in the mempool

* we now look into the mempool when checking for WatchSpent

* fixed FundingSigned been sent twice when deferring an early FundingLocked

* added zeromq listener, removed predefined eclair.conf files in test, limited concurrent rpc calls to 5 for the router

* Update README.md with ZMQ configuration

* Added a warning and link to alpha2 install instructions

* removed PeerClient

* improved PeerWatcher performance

README.md

@@ -34,17 +34,21 @@ This software follows the [Lightning Network Specifications (BOLTs)](https://git
 
 ## Installation
 
+:warning: **Those are valid for the most up-to-date, unreleased, version of eclair. Here are the [instructions for Eclair 0.2-alpha1](https://github.com/ACINQ/eclair/blob/v0.2-alpha1/README.md#installation)**.
+
 ### Configuring Bitcoin Core
 
-Eclair needs a _synchronized_, _segwit-ready_, _non-pruning_, _tx-indexing_ [Bitcoin-core](https://github.com/bitcoin/bitcoin) node.
+Eclair needs a _synchronized_, _segwit-ready_, **_zeromq-enabled_**, _non-pruning_, _tx-indexing_ [Bitcoin Core](https://github.com/bitcoin/bitcoin) node. This means that on Windows you will need Bitcoin Core 0.14.0.
 
-Run bitcoind with the following `bitcoin.conf`:
+Run bitcoind with the following minimal `bitcoin.conf`:
 ```
 regtest=1
 server=1
 rpcuser=XXX
 rpcpassword=XXX
 txindex=1
+zmqpubrawblock=tcp://127.0.0.1:29000
+zmqpubrawtx=tcp://127.0.0.1:29000
 ```
 
 ### Installing Eclair
@@ -85,6 +89,7 @@ option                       | description               | default value
  eclair.api.port             | HTTP port                 | 8080
  eclair.bitcoind.rpcuser     | Bitcoin Core RPC user     | foo
  eclair.bitcoind.rpcpassword | Bitcoin Core RPC password | bar
+ eclair.bitcoind.zmq         | Bitcoin Core ZMQ address  | tcp://127.0.0.1:29000
 
 → see [`application.conf`](eclair-node/src/main/resources/application.conf) for full reference.
 

eclair-node/pom.xml

@@ -183,6 +183,11 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.zeromq</groupId>
+            <artifactId>jeromq</artifactId>
+            <version>0.4.0</version>
+        </dependency>
         <!-- SERIALIZATION -->
         <dependency>
             <groupId>org.scodec</groupId>

eclair-node/src/main/resources/application.conf

@@ -10,10 +10,10 @@ eclair {
   }
   bitcoind {
     host = "localhost"
-    port = 18333
     rpcport = 18332
     rpcuser = "foo"
     rpcpassword = "bar"
+    zmq = "tcp://127.0.0.1:29000"
   }
 
   node-alias = "eclair"
@@ -52,4 +52,10 @@ akka {
       fsm = on
     }
   }
+
+  http {
+    host-connection-pool {
+      max-open-requests = 64
+    }
+  }
 }

eclair-node/src/main/scala/fr/acinq/eclair/Boot.scala

@@ -13,10 +13,10 @@ import ch.qos.logback.classic.spi.ILoggingEvent
 import ch.qos.logback.classic.{Logger, LoggerContext}
 import ch.qos.logback.core.FileAppender
 import com.sun.javafx.application.LauncherImpl
-import fr.acinq.bitcoin.{Base58Check, Message, OP_CHECKSIG, OP_DUP, OP_EQUALVERIFY, OP_HASH160, OP_PUSHDATA, Script}
+import fr.acinq.bitcoin.{Base58Check, OP_CHECKSIG, OP_DUP, OP_EQUALVERIFY, OP_HASH160, OP_PUSHDATA, Script}
 import fr.acinq.eclair.api.Service
-import fr.acinq.eclair.blockchain.peer.PeerClient
 import fr.acinq.eclair.blockchain.rpc.BitcoinJsonRPCClient
+import fr.acinq.eclair.blockchain.zmq.ZeroMQClient
 import fr.acinq.eclair.blockchain.{ExtendedBitcoinClient, PeerWatcher}
 import fr.acinq.eclair.channel.Register
 import fr.acinq.eclair.gui.{FxApp, FxPreloader}
@@ -53,7 +53,7 @@ object Boot extends App with Logging {
   }
 }
 
-class Setup(datadir: String) extends Logging {
+class Setup(datadir: String, actorSystemName: String = "default") extends Logging {
 
   LogSetup.logTo(datadir)
 
@@ -63,7 +63,7 @@ class Setup(datadir: String) extends Logging {
   val nodeParams = NodeParams.makeNodeParams(new File(datadir), config)
   logger.info(s"nodeid=${nodeParams.privateKey.publicKey.toBin} alias=${nodeParams.alias}")
 
-  implicit lazy val system = ActorSystem()
+  implicit lazy val system = ActorSystem(actorSystemName)
   implicit val materializer = ActorMaterializer()
   implicit val timeout = Timeout(30 seconds)
 
@@ -77,9 +77,8 @@ class Setup(datadir: String) extends Logging {
   implicit val ec = ExecutionContext.Implicits.global
   val (chain, blockCount, progress) = Await.result(bitcoin_client.client.invoke("getblockchaininfo").map(json => ((json \ "chain").extract[String], (json \ "blocks").extract[Long], (json \ "verificationprogress").extract[Double])), 10 seconds)
   logger.info(s"using chain=$chain")
-  val magic = chain match {
-    case "test" => Message.MagicTestnet3
-    case "regtest" => Message.MagicTestNet
+  chain match {
+    case "test" | "regtest" => {}
     case _ => throw new RuntimeException("only regtest and testnet are supported for now")
   }
   assert(progress > 0.99, "bitcoind should be synchronized")
@@ -100,7 +99,7 @@ class Setup(datadir: String) extends Logging {
   //val finalScriptPubKey = OP_0 :: OP_PUSHDATA(Base58Check.decode(finalAddress)._2) :: Nil
   val finalScriptPubKey = Script.write(OP_DUP :: OP_HASH160 :: OP_PUSHDATA(Base58Check.decode(finalAddress)._2) :: OP_EQUALVERIFY :: OP_CHECKSIG :: Nil)
 
-  val peer = system.actorOf(SimpleSupervisor.props(PeerClient.props(new InetSocketAddress(config.getString("bitcoind.host"), config.getInt("bitcoind.port")), magic), "bitcoin-peer", SupervisorStrategy.Restart))
+  val zmq = new ZeroMQClient(config.getString("bitcoind.zmq"), system.eventStream)
   val watcher = system.actorOf(SimpleSupervisor.props(PeerWatcher.props(nodeParams, bitcoin_client), "watcher", SupervisorStrategy.Resume))
   val paymentHandler = system.actorOf(SimpleSupervisor.props(config.getString("payment-handler") match {
     case "local" => Props[LocalPaymentHandler]

eclair-node/src/main/scala/fr/acinq/eclair/blockchain/peer/BlockchainEvents.scala → eclair-node/src/main/scala/fr/acinq/eclair/blockchain/BlockchainEvents.scala

@@ -1,4 +1,4 @@
-package fr.acinq.eclair.blockchain.peer
+package fr.acinq.eclair.blockchain
 
 import fr.acinq.bitcoin.{Block, Transaction}
 
@@ -15,3 +15,5 @@ case class NewTransaction(tx: Transaction) extends BlockchainEvent
 case class CurrentBlockCount(blockCount: Long) extends BlockchainEvent
 
 case class CurrentFeerate(feeratePerKw: Long) extends BlockchainEvent
+
+case class MempoolTransaction(tx: Transaction)

eclair-node/src/main/scala/fr/acinq/eclair/blockchain/ExtendedBitcoinClient.scala

@@ -35,11 +35,20 @@ class ExtendedBitcoinClient(val client: BitcoinJsonRPCClient) {
 
   def getTxBlockHash(txId: String)(implicit ec: ExecutionContext): Future[Option[String]] =
     client.invoke("getrawtransaction", txId, 1) // we choose verbose output to get the number of confirmations
-      .map(json => Some((json \ "blockhash").extract[String]))
+      .map(json => (json \ "blockhash").extractOpt[String])
       .recover {
         case t: JsonRPCError if t.error.code == -5 => None
       }
 
+  def getBlockHashesSinceBlockHash(blockHash: String, previous: Seq[String] = Nil)(implicit ec: ExecutionContext): Future[Seq[String]] =
+    for {
+      nextblockhash_opt <- client.invoke("getblock", blockHash).map(json => ((json \ "nextblockhash").extractOpt[String]))
+      res <- nextblockhash_opt match {
+        case Some(nextBlockHash) => getBlockHashesSinceBlockHash(nextBlockHash, previous :+ nextBlockHash)
+        case None => Future.successful(previous)
+      }
+    } yield res
+
   def getTxsSinceBlockHash(blockHash: String, previous: Seq[Transaction] = Nil)(implicit ec: ExecutionContext): Future[Seq[Transaction]] =
     for {
       (nextblockhash_opt, txids) <- client.invoke("getblock", blockHash).map(json => ((json \ "nextblockhash").extractOpt[String], (json \ "tx").extract[List[String]]))
@@ -50,6 +59,11 @@ class ExtendedBitcoinClient(val client: BitcoinJsonRPCClient) {
       }
     } yield res
 
+  def getMempool()(implicit ec: ExecutionContext): Future[Seq[Transaction]] =
+    for {
+      txids <- client.invoke("getrawmempool").map(json => json.extract[List[String]])
+      txs <- Future.sequence(txids.map(getTransaction(_)))
+    } yield txs
 
   /**
     * *used in interop test*
@@ -219,3 +233,19 @@ object ExtendedBitcoinClient {
 
 }
 
+
+/*object Test extends App {
+
+  import scala.concurrent.duration._
+  import ExecutionContext.Implicits.global
+  implicit val system = ActorSystem()
+  implicit val timeout = Timeout(30 seconds)
+
+  val bitcoin_client = new ExtendedBitcoinClient(new BitcoinJsonRPCClient(
+    user = "foo",
+    password = "bar",
+    host = "localhost",
+    port = 28332))
+
+  println(Await.result(bitcoin_client.getTxBlockHash("dcb0abfa822402ce379fedd7bbbb2c824e53ef300313594c39282da1efd35f17"), 10 seconds))
+}*/