From c0852c24afe45da08b452c820ac0f8f5ec50d675 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Tue, 18 Jun 2024 21:12:21 +0200 Subject: [PATCH 01/32] Publish branch --- .vscode/launch.json | 4 +- CHANGELOG.md | 5 ++ CODE_OF_CONDUCT.md | 3 +- LICENSE | 2 +- SECURITY.md | 2 +- cmd/psweb/handlers.go | 59 ++++++++++----------- cmd/psweb/ln/lnd.go | 9 ++-- cmd/psweb/main.go | 2 +- cmd/psweb/templates/config.gohtml | 79 ++++++++++++++++++----------- cmd/psweb/templates/reusable.gohtml | 2 +- 10 files changed, 91 insertions(+), 76 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ab05dd0..75eeb8e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,11 +9,11 @@ "type": "go", "request": "launch", "mode": "auto", - "buildFlags": "-tags lnd", + "buildFlags": "-tags cln", "program": "${workspaceFolder}/cmd/psweb/", "showLog": false, "envFile": "${workspaceFolder}/.env", - "args": ["-datadir", "/home/vlad/.peerswap2"] + // "args": ["-datadir", "/home/vlad/.peerswap2"] } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 573311f..8a89537 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Versions +## 1.5.4 + +- Speedup saving Auto Fees settings +- Hide HTTPS option for Umbrel + ## 1.5.3 - Add automatic channel fees management diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 361b549..1540dfd 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -60,8 +60,7 @@ representative at an online or offline event. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at -t.me/vladgoryachev. -All complaints will be reviewed and investigated promptly and fairly. +https://peerswap.dev. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. diff --git a/LICENSE b/LICENSE index 6af2943..5f55d71 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2023-2024 Vlad Goryachev +Copyright (c) 2023-2024 Impa10r Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/SECURITY.md b/SECURITY.md index d66cdbf..028a42d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,4 +12,4 @@ Getblock runs a publicly available Bitcoin Core server. It is used as a fallback ## Reporting a Vulnerability -If you discover any vulnerability, please contact me directly at t.me/vladgoryachev +If you discover any vulnerability, please report it discretely to contributors. diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index 543b18c..2ae3da7 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -1155,8 +1155,6 @@ func configHandler(w http.ResponseWriter, r *http.Request) { IsPossibleHTTPS: os.Getenv("NO_HTTPS") == "", } - log.Println(os.Getenv("NO_HTTPS")) - // executing template named "config" err := templates.ExecuteTemplate(w, "config", data) if err != nil { @@ -1502,9 +1500,6 @@ func submitHandler(w http.ResponseWriter, r *http.Request) { } } - // apply the new fees - ln.ApplyAutoFeeAll() - // all done, display confirmation http.Redirect(w, r, "/af?id="+r.FormValue("channelId")+"&msg="+msg, http.StatusSeeOther) return @@ -1566,8 +1561,6 @@ func submitHandler(w http.ResponseWriter, r *http.Request) { if isEnabled { msg += "Enabled" - // apply the new fees - ln.ApplyAutoFeeAll() } else { msg += "Disabled" } @@ -1939,36 +1932,38 @@ func saveConfigHandler(w http.ResponseWriter, r *http.Request) { return } - secureConnection, err := strconv.ParseBool(r.FormValue("secureConnection")) - if err != nil { - redirectWithError(w, r, "/config?", err) - return - } + if os.Getenv("NO_HTTPS") != "true" { + secureConnection, err := strconv.ParseBool(r.FormValue("secureConnection")) + if err != nil { + redirectWithError(w, r, "/config?", err) + return + } - // display CA certificate installation instructions - if secureConnection && !config.Config.SecureConnection { - config.Config.ServerIPs = r.FormValue("serverIPs") - config.Save() - http.Redirect(w, r, "/ca", http.StatusSeeOther) - return - } + // display CA certificate installation instructions + if secureConnection && !config.Config.SecureConnection { + config.Config.ServerIPs = r.FormValue("serverIPs") + config.Save() + http.Redirect(w, r, "/ca", http.StatusSeeOther) + return + } - if r.FormValue("serverIPs") != config.Config.ServerIPs { - config.Config.ServerIPs = r.FormValue("serverIPs") - if secureConnection { - if err := config.GenerateServerCertificate(); err == nil { - restart(w, r, true, config.Config.Password) - } else { - log.Println("GenereateServerCertificate:", err) - redirectWithError(w, r, "/config?", err) - return + if r.FormValue("serverIPs") != config.Config.ServerIPs { + config.Config.ServerIPs = r.FormValue("serverIPs") + if secureConnection { + if err := config.GenerateServerCertificate(); err == nil { + restart(w, r, true, config.Config.Password) + } else { + log.Println("GenereateServerCertificate:", err) + redirectWithError(w, r, "/config?", err) + return + } } } - } - if !secureConnection && config.Config.SecureConnection { - // restart to listen on HTTP only - restart(w, r, false, "") + if !secureConnection && config.Config.SecureConnection { + // restart to listen on HTTP only + restart(w, r, false, "") + } } allowSwapRequests, err := strconv.ParseBool(r.FormValue("allowSwapRequests")) diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index f49ecbb..0ceff90 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -1050,8 +1050,7 @@ func SubscribeAll() { downloadForwards(client) // subscribe to Forwards for { - if err := subscribeForwards(ctx, routerClient); err != nil { - log.Println("Forwards subscription error:", err) + if subscribeForwards(ctx, routerClient) != nil { time.Sleep(60 * time.Second) } } @@ -1063,8 +1062,7 @@ func SubscribeAll() { // subscribe to Payments for { - if err := subscribePayments(ctx, routerClient); err != nil { - log.Println("Payments subscription error:", err) + if subscribePayments(ctx, routerClient) != nil { time.Sleep(60 * time.Second) } } @@ -1074,8 +1072,7 @@ func SubscribeAll() { // subscribe to Invoices for { - if err := subscribeInvoices(ctx, client); err != nil { - log.Println("Invoices subscription error:", err) + if subscribeInvoices(ctx, client) != nil { time.Sleep(60 * time.Second) } } diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index 16bb01b..7d9b873 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -34,7 +34,7 @@ import ( const ( // App version tag - version = "v1.5.3" + version = "v1.5.4" ) type SwapParams struct { diff --git a/cmd/psweb/templates/config.gohtml b/cmd/psweb/templates/config.gohtml index ba30fa9..540f2a3 100644 --- a/cmd/psweb/templates/config.gohtml +++ b/cmd/psweb/templates/config.gohtml @@ -117,22 +117,39 @@
- -
-
- -
-
-
- +
+ {{if .IsPossibleHTTPS}} +
+
+ +
+
+
+ +
-
+ {{end}}
+ {{if not .IsPossibleHTTPS}} +
+
+ +
+
+
+ +
+
+
+ {{end}}
-
-
- -
-
-
- + {{if .IsPossibleHTTPS}} +
+
+ +
+
+
+ +
-
-
-
- -
-
- +
+
+ +
+
+ +
-
+ {{end}}
diff --git a/cmd/psweb/templates/reusable.gohtml b/cmd/psweb/templates/reusable.gohtml index 940ec06..0b47b6f 100644 --- a/cmd/psweb/templates/reusable.gohtml +++ b/cmd/psweb/templates/reusable.gohtml @@ -122,7 +122,7 @@

- PeerSwap Web UI by Vlad Goryachev + PeerSwap Web UI by Impa10r

From d2cbaedeba7b5e9d3546741e1179bdf83a606412 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Wed, 19 Jun 2024 17:20:09 +0200 Subject: [PATCH 02/32] Migrate auto fee database --- CHANGELOG.md | 2 + cmd/psweb/handlers.go | 2 +- cmd/psweb/ln/cln.go | 20 ++-- cmd/psweb/ln/common.go | 54 ++++++++++- cmd/psweb/ln/lnd.go | 201 +++++++++++++++++++++++------------------ cmd/psweb/main.go | 23 ++++- 6 files changed, 191 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a89537..15275b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - Speedup saving Auto Fees settings - Hide HTTPS option for Umbrel +- AF: apply HTLC Fail Bumps only when Balance % < Low Liq % +- AF: log full fee change history, including inbound ## 1.5.3 diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index 2ae3da7..d2947f9 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -375,7 +375,7 @@ func peerHandler(w http.ResponseWriter, r *http.Request) { rates = "*" + rates } - feeLog := ln.AutoFeeLog[ch.ChannelId] + feeLog := ln.LastAutoFeeLog(ch.ChannelId, false) if feeLog != nil { rates += ", last update " + timePassedAgo(time.Unix(feeLog.TimeStamp, 0)) rates += " from " + formatWithThousandSeparators(uint64(feeLog.OldRate)) diff --git a/cmd/psweb/ln/cln.go b/cmd/psweb/ln/cln.go index 2fd125c..0342e61 100644 --- a/cmd/psweb/ln/cln.go +++ b/cmd/psweb/ln/cln.go @@ -14,7 +14,6 @@ import ( "peerswap-web/cmd/psweb/bitcoin" "peerswap-web/cmd/psweb/config" - "peerswap-web/cmd/psweb/db" "peerswap-web/cmd/psweb/internet" "github.com/btcsuite/btcd/chaincfg" @@ -1106,10 +1105,12 @@ func HasInboundFees() bool { } func ApplyAutoFeeAll() { - if !AutoFeeEnabledAll { + if !AutoFeeEnabledAll || autoFeeApplyAllIsRunning { return } + autoFeeApplyAllIsRunning = true + CacheForwards() client, cleanup, err := GetClient() @@ -1150,9 +1151,10 @@ func ApplyAutoFeeAll() { oldFee := int(channelMap["fee_proportional_millionths"].(float64)) newFee := oldFee + liqPct := int(channelMap["to_us_msat"].(float64) * 100 / channelMap["total_msat"].(float64)) // check 10 minutes back to be sure - if params.FailedBumpPPM > 0 { + if params.FailedBumpPPM > 0 && liqPct < params.LowLiqPct { if failedForwardTS[channelId] > time.Now().Add(-time.Duration(10*time.Minute)).Unix() { // bump fee newFee += params.FailedBumpPPM @@ -1161,8 +1163,8 @@ func ApplyAutoFeeAll() { } } + // if no Fail Bump if newFee == oldFee { - liqPct := int(channelMap["to_us_msat"].(float64) * 100 / channelMap["total_msat"].(float64)) newFee = calculateAutoFee(channelId, params, liqPct, oldFee) } @@ -1171,16 +1173,12 @@ func ApplyAutoFeeAll() { peerId := channelMap["peer_id"].(string) if SetFeeRate(peerId, channelId, int64(newFee), false, false) == nil { // log the last change - AutoFeeLog[channelId] = &AutoFeeEvent{ - TimeStamp: time.Now().Unix(), - OldRate: oldFee, - NewRate: newFee, - } - // persist to db - db.Save("AutoFees", "AutoFeeLog", AutoFeeLog) + LogAutoFee(channelId, oldFee, newFee, false) } } } + + autoFeeApplyAllIsRunning = false } func ApplyAutoFee() { diff --git a/cmd/psweb/ln/common.go b/cmd/psweb/ln/common.go index 24d0950..e775e16 100644 --- a/cmd/psweb/ln/common.go +++ b/cmd/psweb/ln/common.go @@ -1,6 +1,7 @@ package ln import ( + "reflect" "strconv" "strings" "time" @@ -113,6 +114,7 @@ type AutoFeeEvent struct { TimeStamp int64 OldRate int NewRate int + IsInbound bool } var ( @@ -124,7 +126,7 @@ var ( AutoFeeEnabledAll bool // maps to LND channel Id AutoFee = make(map[uint64]*AutoFeeParams) - AutoFeeLog = make(map[uint64]*AutoFeeEvent) + AutoFeeLog = make(map[uint64][]*AutoFeeEvent) AutoFeeEnabled = make(map[uint64]bool) AutoFeeDefaults = AutoFeeParams{ FailedBumpPPM: 10, @@ -142,6 +144,9 @@ var ( // track timestamp of the last outbound forward per channel lastForwardTS = make(map[uint64]int64) + + // prevents starting another fee update while the first still running + autoFeeApplyAllIsRunning = false ) func toSats(amount float64) int64 { @@ -224,7 +229,24 @@ func LoadAutoFees() { db.Load("AutoFees", "AutoFeeEnabled", &AutoFeeEnabled) db.Load("AutoFees", "AutoFee", &AutoFee) db.Load("AutoFees", "AutoFeeDefaults", &AutoFeeDefaults) - db.Load("AutoFees", "AutoFeeLog", &AutoFeeLog) + + // drop non-array legacy log + var log map[uint64]interface{} + db.Load("AutoFees", "AutoFeeLog", &log) + + if len(log) == 0 { + return + } + + // Use reflection to determine the type + for _, data := range log { + v := reflect.ValueOf(data) + if v.Kind() == reflect.Slice { + // Type is map[uint64][]*AutoFeeEvent, load again + db.Load("AutoFees", "AutoFeeLog", &AutoFeeLog) + return + } + } } func calculateAutoFee(channelId uint64, params *AutoFeeParams, liqPct int, oldFee int) int { @@ -232,8 +254,9 @@ func calculateAutoFee(channelId uint64, params *AutoFeeParams, liqPct int, oldFe if liqPct > params.LowLiqPct { // normal or high liquidity regime, check if fee can be dropped lastUpdate := int64(0) - if AutoFeeLog[channelId] != nil { - lastUpdate = AutoFeeLog[channelId].TimeStamp + lastLog := LastAutoFeeLog(channelId, false) + if lastLog != nil { + lastUpdate = lastLog.TimeStamp } if lastUpdate < time.Now().Add(-time.Duration(params.CoolOffHours)*time.Hour).Unix() { // check the last outbound timestamp @@ -267,3 +290,26 @@ func msatToSatUp(msat uint64) uint64 { } return sat } + +// returns last log entry +func LastAutoFeeLog(channelId uint64, isInbound bool) *AutoFeeEvent { + // Loop backwards through the array + for i := len(AutoFeeLog[channelId]) - 1; i >= 0; i-- { + if AutoFeeLog[channelId][i].IsInbound == isInbound { + return AutoFeeLog[channelId][i] + } + } + + return nil +} + +func LogAutoFee(channelId uint64, oldRate int, newRate int, isInbound bool) { + AutoFeeLog[channelId] = append(AutoFeeLog[channelId], &AutoFeeEvent{ + TimeStamp: time.Now().Unix(), + OldRate: oldRate, + NewRate: newRate, + IsInbound: true, + }) + // persist to db + db.Save("AutoFees", "AutoFeeLog", AutoFeeLog) +} diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index 0ceff90..b5547be 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -19,7 +19,6 @@ import ( "peerswap-web/cmd/psweb/bitcoin" "peerswap-web/cmd/psweb/config" - "peerswap-web/cmd/psweb/db" "github.com/elementsproject/peerswap/peerswaprpc" @@ -74,8 +73,9 @@ var ( lastInvoiceSettleIndex uint64 // last timestamps for downloads - lastforwardCreationTs uint64 + lastForwardCreationTs uint64 lastPaymentCreationTs int64 + lastInvoiceCreationTs int64 // default lock id used by LND internalLockId = []byte{ @@ -694,14 +694,65 @@ func GetMyAlias() string { return myNodeAlias } +func downloadInvoices(client lnrpc.LightningClient) { + // only go back 6 months for itinial download + start := uint64(time.Now().AddDate(0, -6, 0).Unix()) + offset := uint64(0) + totalInvoices := uint64(0) + + // incremental download + if lastInvoiceCreationTs > 0 { + start = uint64(lastInvoiceCreationTs) + 1 + } + + for { + res, err := client.ListInvoices(context.Background(), &lnrpc.ListInvoiceRequest{ + CreationDateStart: start, + Reversed: false, + IndexOffset: offset, + NumMaxInvoices: 100, // bolt11 fields can be long + }) + if err != nil { + if !strings.HasPrefix(fmt.Sprint(err), "rpc error: code = Unknown desc = waiting to start") { + log.Println("ListInvoices:", err) + } + return + } + + for _, invoice := range res.Invoices { + appendInvoice(invoice) + } + + n := len(res.Invoices) + totalInvoices += uint64(n) + + if n > 0 { + // settle index for subscription + lastInvoiceSettleIndex = res.Invoices[n-1].SettleIndex + } + if n < 100 { + // all invoices retrieved + break + } + + // next pull start from the next index + offset = res.LastIndexOffset + } + + if totalInvoices > 0 { + log.Printf("Cached %d invoices", totalInvoices) + } + +} + func downloadForwards(client lnrpc.LightningClient) { // only go back 6 months start := uint64(time.Now().AddDate(0, -6, 0).Unix()) // incremental download if substription was interrupted - if lastforwardCreationTs > 0 { + if lastForwardCreationTs > 0 { // continue from the last timestamp in seconds - start = lastforwardCreationTs + 1 + start = lastForwardCreationTs + 1 } // download forwards @@ -734,7 +785,7 @@ func downloadForwards(client lnrpc.LightningClient) { if n > 0 { // store the last timestamp - lastforwardCreationTs = res.ForwardingEvents[n-1].TimestampNs / 1_000_000_000 + lastForwardCreationTs = res.ForwardingEvents[n-1].TimestampNs / 1_000_000_000 } if n < 50000 { // all events retrieved @@ -932,7 +983,7 @@ func subscribeForwards(ctx context.Context, client routerrpc.RouterClient) error // settled htlcEvent has no Outgoing info, take from queue forwardsOut[htlc.OutgoingChannelId] = append(forwardsOut[htlc.OutgoingChannelId], htlc.forwardingEvent) // store the last timestamp - lastforwardCreationTs = htlc.forwardingEvent.TimestampNs / 1_000_000_000 + lastForwardCreationTs = htlc.forwardingEvent.TimestampNs / 1_000_000_000 // delete from queue removeInflightHTLC(htlcEvent.IncomingChannelId, htlcEvent.IncomingHtlcId) @@ -948,6 +999,7 @@ func subscribeForwards(ctx context.Context, client routerrpc.RouterClient) error } defer cleanup() + // calculate with new balance ApplyAutoFee(client, htlc.forwardingEvent.ChanIdOut, false) break } @@ -1000,70 +1052,34 @@ func SubscribeAll() { client := lnrpc.NewLightningClient(conn) ctx := context.Background() - // only go back 6 months for itinial download - start := uint64(time.Now().AddDate(0, -6, 0).Unix()) - offset := uint64(0) - totalInvoices := uint64(0) - - for { - res, err := client.ListInvoices(ctx, &lnrpc.ListInvoiceRequest{ - CreationDateStart: start, - Reversed: false, - IndexOffset: offset, - NumMaxInvoices: 100, // bolt11 fields can be long - }) - if err != nil { - if !strings.HasPrefix(fmt.Sprint(err), "rpc error: code = Unknown desc = waiting to start") { - log.Println("ListInvoices:", err) - } - return - } - - for _, invoice := range res.Invoices { - appendInvoice(invoice) - } - - n := len(res.Invoices) - totalInvoices += uint64(n) - - if n > 0 { - // settle index for subscription - lastInvoiceSettleIndex = res.Invoices[n-1].SettleIndex - } - if n < 100 { - // all invoices retrieved - break - } - - // next pull start from the next index - offset = res.LastIndexOffset - } - - if totalInvoices > 0 { - log.Printf("Cached %d invoices", totalInvoices) - } + // initial download + downloadInvoices(client) routerClient := routerrpc.NewRouterClient(conn) go func() { - // initial or incremental download forwards + // initial download forwards downloadForwards(client) // subscribe to Forwards for { if subscribeForwards(ctx, routerClient) != nil { time.Sleep(60 * time.Second) + // incremental download after error + downloadForwards(client) } } }() go func() { - // initial or incremental download payments + // initial download payments downloadPayments(client) // subscribe to Payments for { if subscribePayments(ctx, routerClient) != nil { time.Sleep(60 * time.Second) + // incremental download after error + downloadPayments(client) } } }() @@ -1074,6 +1090,8 @@ func SubscribeAll() { for { if subscribeInvoices(ctx, client) != nil { time.Sleep(60 * time.Second) + // incremental download after error + downloadInvoices(client) } } } @@ -1083,6 +1101,10 @@ func appendInvoice(invoice *lnrpc.Invoice) { // precaution return } + + // save for incremental downloads + lastInvoiceCreationTs = invoice.CreationDate + // only append settled htlcs if invoice.State == lnrpc.Invoice_SETTLED { if parts := strings.Split(invoice.Memo, " "); len(parts) > 4 { @@ -1683,44 +1705,37 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo oldFee := int(policy.FeeRateMilliMsat) newFee := oldFee + // get balances + bytePeer, err := hex.DecodeString(peerId) + if err != nil { + return + } - if failedHTLC { - // increase fee to help prevent further failed HTLCs - newFee += params.FailedBumpPPM - } else { - // get balances - bytePeer, err := hex.DecodeString(peerId) - if err != nil { - return - } - - res, err := client.ListChannels(ctx, &lnrpc.ListChannelsRequest{ - PublicOnly: true, - Peer: bytePeer, - }) - if err != nil { - return - } + res, err := client.ListChannels(ctx, &lnrpc.ListChannelsRequest{ + Peer: bytePeer, + }) + if err != nil { + return + } - localBalance := int64(0) - for _, ch := range res.Channels { - if ch.ChanId == channelId { - if ch.UnsettledBalance != 0 { - // skip af while htlcs are pending - return - } - localBalance = ch.LocalBalance - break + localBalance := int64(0) + for _, ch := range res.Channels { + if ch.ChanId == channelId { + if ch.UnsettledBalance != 0 { + // skip af while htlcs are pending + return } + localBalance = ch.LocalBalance + break } + } - liqPct := int(localBalance * 100 / r.Capacity) - - if HasInboundFees() && liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat != int32(params.LowLiqDiscount) { - // set discount - SetFeeRate(peerId, channelId, int64(params.LowLiqDiscount), true, false) - } + liqPct := int(localBalance * 100 / r.Capacity) + if failedHTLC && liqPct < params.LowLiqPct { + // increase fee to help prevent further failed HTLCs + newFee += params.FailedBumpPPM + } else { newFee = calculateAutoFee(channelId, params, liqPct, oldFee) } @@ -1728,23 +1743,27 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo if newFee != oldFee { if SetFeeRate(peerId, channelId, int64(newFee), false, false) == nil { // log the last change - AutoFeeLog[channelId] = &AutoFeeEvent{ - TimeStamp: time.Now().Unix(), - OldRate: oldFee, - NewRate: newFee, - } - // persist to db - db.Save("AutoFees", "AutoFeeLog", AutoFeeLog) + LogAutoFee(channelId, oldFee, newFee, false) } } + + // set inbound fee + if HasInboundFees() && liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat != int32(params.LowLiqDiscount) { + // set discount + SetFeeRate(peerId, channelId, int64(params.LowLiqDiscount), true, false) + // log the last change + LogAutoFee(channelId, int(policy.InboundFeeRateMilliMsat), params.LowLiqDiscount, true) + } } func ApplyAutoFeeAll() { - if !AutoFeeEnabledAll { + if !AutoFeeEnabledAll || autoFeeApplyAllIsRunning { return } + autoFeeApplyAllIsRunning = true + client, cleanup, err := GetClient() if err != nil { return @@ -1762,4 +1781,6 @@ func ApplyAutoFeeAll() { for _, ch := range res.Channels { ApplyAutoFee(client, ch.ChanId, false) } + + autoFeeApplyAllIsRunning = false } diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index 7d9b873..ae6c0fa 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -348,7 +348,7 @@ func onTimer() { go ln.SubscribeAll() // execute auto fee - ln.ApplyAutoFeeAll() + go ln.ApplyAutoFeeAll() } func liquidBackup(force bool) { @@ -1459,14 +1459,27 @@ func feeInputField(peerNodeId string, channelId uint64, direction string, feePer rates = "*" + rates } - feeLog := ln.AutoFeeLog[channelId] + change := " " + feeLog := ln.LastAutoFeeLog(channelId, direction == "inbound") if feeLog != nil { rates += "\nLast update " + timePassedAgo(time.Unix(feeLog.TimeStamp, 0)) - rates += "\nFrom " + formatWithThousandSeparators(uint64(feeLog.OldRate)) - rates += " to " + formatWithThousandSeparators(uint64(feeLog.NewRate)) + rates += "\nFrom " + formatSigned(int64(feeLog.OldRate)) + rates += " to " + formatSigned(int64(feeLog.NewRate)) + if feeLog.TimeStamp > time.Now().Add(-24*time.Hour).Unix() { + if feeLog.NewRate > feeLog.OldRate { + if config.Config.ColorScheme == "dark" { + change = `⬆` + } else { + change = `⬆` + } + + } else { + change = `⬇` + } + } } - t += "" + formatSigned(feePerMil) + "" + t += "" + formatSigned(feePerMil) + "" + change } else { t += `
` t += `` From 15cf0447a5a0d34544aaa403b2075f2bd58a79ee Mon Sep 17 00:00:00 2001 From: Impa10r Date: Wed, 19 Jun 2024 17:32:10 +0200 Subject: [PATCH 03/32] fix bug --- CHANGELOG.md | 2 +- cmd/psweb/ln/common.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15275b9..bd08206 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - Speedup saving Auto Fees settings - Hide HTTPS option for Umbrel - AF: apply HTLC Fail Bumps only when Balance % < Low Liq % -- AF: log full fee change history, including inbound +- AF: log full fee changes history, including inbound ## 1.5.3 diff --git a/cmd/psweb/ln/common.go b/cmd/psweb/ln/common.go index e775e16..8a17589 100644 --- a/cmd/psweb/ln/common.go +++ b/cmd/psweb/ln/common.go @@ -308,7 +308,7 @@ func LogAutoFee(channelId uint64, oldRate int, newRate int, isInbound bool) { TimeStamp: time.Now().Unix(), OldRate: oldRate, NewRate: newRate, - IsInbound: true, + IsInbound: isInbound, }) // persist to db db.Save("AutoFees", "AutoFeeLog", AutoFeeLog) From bfa1ab8ae50360b07fb37261eaaf4dae42118ac5 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Wed, 19 Jun 2024 18:30:13 +0200 Subject: [PATCH 04/32] Auto move low liq threshold --- cmd/psweb/handlers.go | 6 ++++++ cmd/psweb/ln/cln.go | 15 +++++++++----- cmd/psweb/ln/common.go | 21 ++++++++++++++++++- cmd/psweb/ln/lnd.go | 16 ++++++++++----- cmd/psweb/templates/af.gohtml | 38 +++++++++++++++++++++-------------- 5 files changed, 70 insertions(+), 26 deletions(-) diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index d2947f9..a6c2f91 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -1422,6 +1422,12 @@ func submitHandler(w http.ResponseWriter, r *http.Request) { return } + rule.FailedMoveThreshold, err = strconv.Atoi(r.FormValue("failedMoveThreshold")) + if err != nil { + redirectWithError(w, r, "/af?", err) + return + } + rule.LowLiqPct, err = strconv.Atoi(r.FormValue("lowLiqPct")) if err != nil { redirectWithError(w, r, "/af?", err) diff --git a/cmd/psweb/ln/cln.go b/cmd/psweb/ln/cln.go index 0342e61..010ea2b 100644 --- a/cmd/psweb/ln/cln.go +++ b/cmd/psweb/ln/cln.go @@ -1154,12 +1154,17 @@ func ApplyAutoFeeAll() { liqPct := int(channelMap["to_us_msat"].(float64) * 100 / channelMap["total_msat"].(float64)) // check 10 minutes back to be sure - if params.FailedBumpPPM > 0 && liqPct < params.LowLiqPct { - if failedForwardTS[channelId] > time.Now().Add(-time.Duration(10*time.Minute)).Unix() { + if failedForwardTS[channelId] > time.Now().Add(-time.Duration(10*time.Minute)).Unix() { + // forget failed HTLC to prevent duplicate action + failedForwardTS[channelId] = 0 + + if liqPct <= params.LowLiqPct { // bump fee newFee += params.FailedBumpPPM - // forget failed HTLC - failedForwardTS[channelId] = 0 + } else { + // move threshold or do nothing + moveLowLiqThreshold(channelId, params.FailedMoveThreshold) + return } } @@ -1173,7 +1178,7 @@ func ApplyAutoFeeAll() { peerId := channelMap["peer_id"].(string) if SetFeeRate(peerId, channelId, int64(newFee), false, false) == nil { // log the last change - LogAutoFee(channelId, oldFee, newFee, false) + LogFee(channelId, oldFee, newFee, false, false) } } } diff --git a/cmd/psweb/ln/common.go b/cmd/psweb/ln/common.go index 8a17589..82e3e2c 100644 --- a/cmd/psweb/ln/common.go +++ b/cmd/psweb/ln/common.go @@ -88,6 +88,8 @@ type AutoFeeStatus struct { type AutoFeeParams struct { // fee rate ppm increase after each "Insufficient Balance" HTLC failure FailedBumpPPM int + // Move Low Liq % theshold after each 'Insufficient Balance' HTLC failure above it + FailedMoveThreshold int // low local balance threshold where fee rates stay high LowLiqPct int // ppm rate when liquidity is below LowLiqPct @@ -115,6 +117,7 @@ type AutoFeeEvent struct { OldRate int NewRate int IsInbound bool + IsManual bool } var ( @@ -303,13 +306,29 @@ func LastAutoFeeLog(channelId uint64, isInbound bool) *AutoFeeEvent { return nil } -func LogAutoFee(channelId uint64, oldRate int, newRate int, isInbound bool) { +func LogFee(channelId uint64, oldRate int, newRate int, isInbound bool, isManual bool) { AutoFeeLog[channelId] = append(AutoFeeLog[channelId], &AutoFeeEvent{ TimeStamp: time.Now().Unix(), OldRate: oldRate, NewRate: newRate, IsInbound: isInbound, + IsManual: isManual, }) // persist to db db.Save("AutoFees", "AutoFeeLog", AutoFeeLog) } + +func moveLowLiqThreshold(channelId uint64, bump int) { + if bump == 0 { + return + } + + if AutoFee[channelId] == nil { + // add custom parameters + AutoFee[channelId] = new(AutoFeeParams) + } + + AutoFee[channelId].LowLiqPct += bump + // persist to db + db.Save("AutoFees", "AutoFee", AutoFee) +} diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index b5547be..5c5b7d3 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -1732,9 +1732,15 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo liqPct := int(localBalance * 100 / r.Capacity) - if failedHTLC && liqPct < params.LowLiqPct { - // increase fee to help prevent further failed HTLCs - newFee += params.FailedBumpPPM + if failedHTLC { + if liqPct <= params.LowLiqPct { + // increase fee to help prevent further failed HTLCs + newFee += params.FailedBumpPPM + } else { + // move threshold or do nothing + moveLowLiqThreshold(channelId, params.FailedMoveThreshold) + return + } } else { newFee = calculateAutoFee(channelId, params, liqPct, oldFee) } @@ -1743,7 +1749,7 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo if newFee != oldFee { if SetFeeRate(peerId, channelId, int64(newFee), false, false) == nil { // log the last change - LogAutoFee(channelId, oldFee, newFee, false) + LogFee(channelId, oldFee, newFee, false, false) } } @@ -1752,7 +1758,7 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo // set discount SetFeeRate(peerId, channelId, int64(params.LowLiqDiscount), true, false) // log the last change - LogAutoFee(channelId, int(policy.InboundFeeRateMilliMsat), params.LowLiqDiscount, true) + LogFee(channelId, int(policy.InboundFeeRateMilliMsat), params.LowLiqDiscount, true, false) } } diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 619ce89..4c9ff13 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -127,7 +127,19 @@
- + +
+ + +
+ +
+ + + + +
+
@@ -135,6 +147,16 @@
+ +
+ +
+ + +
+ +
+ @@ -180,20 +202,6 @@
- {{if .HasInboundFees}} - - -
- -
- - -
- -
- - - {{end}}
From 964807442282980f069d8feb724f630dfaaaffea Mon Sep 17 00:00:00 2001 From: Impa10r Date: Wed, 19 Jun 2024 19:25:23 +0200 Subject: [PATCH 05/32] Show rates in AF table --- .vscode/launch.json | 4 ++-- CHANGELOG.md | 3 ++- cmd/psweb/handlers.go | 27 ++++++++++++++++++--------- cmd/psweb/ln/common.go | 28 +++++++++++++++++----------- cmd/psweb/templates/af.gohtml | 14 ++++++++------ 5 files changed, 47 insertions(+), 29 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 75eeb8e..ab05dd0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,11 +9,11 @@ "type": "go", "request": "launch", "mode": "auto", - "buildFlags": "-tags cln", + "buildFlags": "-tags lnd", "program": "${workspaceFolder}/cmd/psweb/", "showLog": false, "envFile": "${workspaceFolder}/.env", - // "args": ["-datadir", "/home/vlad/.peerswap2"] + "args": ["-datadir", "/home/vlad/.peerswap2"] } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index bd08206..07b51a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ - Speedup saving Auto Fees settings - Hide HTTPS option for Umbrel -- AF: apply HTLC Fail Bumps only when Balance % < Low Liq % +- AF: apply HTLC Fail Bumps only when Local % <= Low Liq % +- AF: for HTLC Fails above Low Liq % allow increasing Low Liq % threshold - AF: log full fee changes history, including inbound ## 1.5.3 diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index a6c2f91..f3c16bd 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -831,19 +831,28 @@ func afHandler(w http.ResponseWriter, r *http.Request) { return } + // get fee rates for all channels + outboundFeeRates := make(map[uint64]int64) + inboundFeeRates := make(map[uint64]int64) + + ln.FeeReport(cl, outboundFeeRates, inboundFeeRates) + for _, peer := range res.GetPeers() { alias := getNodeAlias(peer.NodeId) for _, ch := range peer.Channels { - rates, custom := ln.AutoFeeRatesSummary(ch.ChannelId) + rule, custom := ln.AutoFeeRatesSummary(ch.ChannelId) + af, _ := ln.AutoFeeRule(ch.ChannelId) channelList = append(channelList, &ln.AutoFeeStatus{ - Enabled: ln.AutoFeeEnabled[ch.ChannelId], - LocalBalance: ch.LocalBalance, - RemoteBalance: ch.RemoteBalance, - Alias: alias, - LocalPct: ch.LocalBalance * 100 / (ch.LocalBalance + ch.RemoteBalance), - Rates: rates, - Custom: custom, - ChannelId: ch.ChannelId, + Enabled: ln.AutoFeeEnabled[ch.ChannelId], + Capacity: ch.LocalBalance + ch.RemoteBalance, + Alias: alias, + LocalPct: ch.LocalBalance * 100 / (ch.LocalBalance + ch.RemoteBalance), + Rule: rule, + Custom: custom, + AutoFee: af, + FeeRate: outboundFeeRates[ch.ChannelId], + InboundRate: inboundFeeRates[ch.ChannelId], + ChannelId: ch.ChannelId, }) if ch.ChannelId == channelId { diff --git a/cmd/psweb/ln/common.go b/cmd/psweb/ln/common.go index 82e3e2c..9823a08 100644 --- a/cmd/psweb/ln/common.go +++ b/cmd/psweb/ln/common.go @@ -75,14 +75,16 @@ type ChanneInfo struct { } type AutoFeeStatus struct { - Alias string - ChannelId uint64 - LocalBalance uint64 - LocalPct uint64 - RemoteBalance uint64 - Enabled bool - Rates string - Custom bool + Alias string + ChannelId uint64 + Capacity uint64 + LocalPct uint64 + Enabled bool + Rule string + AutoFee *AutoFeeParams + Custom bool + FeeRate int64 + InboundRate int64 } type AutoFeeParams struct { @@ -328,7 +330,11 @@ func moveLowLiqThreshold(channelId uint64, bump int) { AutoFee[channelId] = new(AutoFeeParams) } - AutoFee[channelId].LowLiqPct += bump - // persist to db - db.Save("AutoFees", "AutoFee", AutoFee) + // do not alow exeeding high liquidity threshold + if AutoFee[channelId].LowLiqPct+bump < AutoFee[channelId].ExcessPct { + AutoFee[channelId].LowLiqPct += bump + // persist to db + db.Save("AutoFees", "AutoFee", AutoFee) + } + } diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 4c9ff13..6b29a66 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -240,9 +240,10 @@ Alias - Local + Cap % - Remote + Rate + In Rules ☑ @@ -252,10 +253,11 @@ {{range .ChannelList}} {{.Alias}} - {{m .LocalBalance}} - {{fmt .LocalPct}} - {{m .RemoteBalance}} - {{if .Custom}}*{{end}}{{.Rates}} + {{m .Capacity}} + {{fmt .LocalPct}} + {{fs .FeeRate}} + {{fs .InboundRate}} + {{if .Custom}}*{{end}}{{.Rule}} From 5ac7f870154c4bcb4fab41ae264b7307e7767097 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Wed, 19 Jun 2024 19:39:25 +0200 Subject: [PATCH 06/32] Clone defaults when move threshold --- cmd/psweb/ln/common.go | 2 ++ cmd/psweb/templates/af.gohtml | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/psweb/ln/common.go b/cmd/psweb/ln/common.go index 9823a08..2af0b15 100644 --- a/cmd/psweb/ln/common.go +++ b/cmd/psweb/ln/common.go @@ -328,6 +328,8 @@ func moveLowLiqThreshold(channelId uint64, bump int) { if AutoFee[channelId] == nil { // add custom parameters AutoFee[channelId] = new(AutoFeeParams) + // clone default values + *AutoFee[channelId] = AutoFeeDefaults } // do not alow exeeding high liquidity threshold diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 6b29a66..fce0ba3 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -240,12 +240,12 @@ Alias - Cap + Cap % - Rate + Rate In Rules +* indicates custom rule" style="text-align: center;">AF Rules ☑ @@ -255,9 +255,9 @@ {{.Alias}} {{m .Capacity}} {{fmt .LocalPct}} - {{fs .FeeRate}} + {{fs .FeeRate}} {{fs .InboundRate}} - {{if .Custom}}*{{end}}{{.Rule}} + {{if .Enabled}}{{if .Custom}}*{{end}}{{.Rule}}{{end}} From 4833f4d91fd1fded31d93a03b713a05e3576e718 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Wed, 19 Jun 2024 20:06:31 +0200 Subject: [PATCH 07/32] download error on lnd startup --- cmd/psweb/ln/lnd.go | 9 ++++++--- cmd/psweb/templates/af.gohtml | 8 +++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index 5c5b7d3..f5a1927 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -694,7 +694,7 @@ func GetMyAlias() string { return myNodeAlias } -func downloadInvoices(client lnrpc.LightningClient) { +func downloadInvoices(client lnrpc.LightningClient) error { // only go back 6 months for itinial download start := uint64(time.Now().AddDate(0, -6, 0).Unix()) offset := uint64(0) @@ -716,7 +716,7 @@ func downloadInvoices(client lnrpc.LightningClient) { if !strings.HasPrefix(fmt.Sprint(err), "rpc error: code = Unknown desc = waiting to start") { log.Println("ListInvoices:", err) } - return + return err } for _, invoice := range res.Invoices { @@ -743,6 +743,7 @@ func downloadInvoices(client lnrpc.LightningClient) { log.Printf("Cached %d invoices", totalInvoices) } + return nil } func downloadForwards(client lnrpc.LightningClient) { @@ -1053,7 +1054,9 @@ func SubscribeAll() { ctx := context.Background() // initial download - downloadInvoices(client) + if downloadInvoices(client) != nil { + return + } routerClient := routerrpc.NewRouterClient(conn) diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index fce0ba3..0ba4e27 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -254,7 +254,13 @@ {{.Alias}} {{m .Capacity}} - {{fmt .LocalPct}} + {{fmt .LocalPct}} {{fs .FeeRate}} {{fs .InboundRate}} {{if .Enabled}}{{if .Custom}}*{{end}}{{.Rule}}{{end}} From 3bcd8ac3321195d3927dd55e2ad1e5eeaf97ad9c Mon Sep 17 00:00:00 2001 From: Impa10r Date: Wed, 19 Jun 2024 20:48:03 +0200 Subject: [PATCH 08/32] optimize autofee --- cmd/psweb/ln/cln.go | 15 +++--- cmd/psweb/ln/common.go | 2 +- cmd/psweb/ln/lnd.go | 98 ++++++++++++++++++++++++++++------- cmd/psweb/main.go | 2 +- cmd/psweb/templates/af.gohtml | 12 ++--- 5 files changed, 95 insertions(+), 34 deletions(-) diff --git a/cmd/psweb/ln/cln.go b/cmd/psweb/ln/cln.go index 010ea2b..6172a06 100644 --- a/cmd/psweb/ln/cln.go +++ b/cmd/psweb/ln/cln.go @@ -1104,17 +1104,18 @@ func HasInboundFees() bool { return false } -func ApplyAutoFeeAll() { - if !AutoFeeEnabledAll || autoFeeApplyAllIsRunning { +func ApplyAutoFees() { + if !AutoFeeEnabledAll || autoFeeIsRunning { return } - autoFeeApplyAllIsRunning = true + autoFeeIsRunning = true CacheForwards() client, cleanup, err := GetClient() if err != nil { + autoFeeIsRunning = false return } defer cleanup() @@ -1122,6 +1123,7 @@ func ApplyAutoFeeAll() { var response map[string]interface{} if client.Request(&ListPeerChannelsRequest{}, &response) != nil { + autoFeeIsRunning = false return } @@ -1164,6 +1166,7 @@ func ApplyAutoFeeAll() { } else { // move threshold or do nothing moveLowLiqThreshold(channelId, params.FailedMoveThreshold) + autoFeeIsRunning = false return } } @@ -1183,9 +1186,5 @@ func ApplyAutoFeeAll() { } } - autoFeeApplyAllIsRunning = false -} - -func ApplyAutoFee() { - // not implemented + autoFeeIsRunning = false } diff --git a/cmd/psweb/ln/common.go b/cmd/psweb/ln/common.go index 2af0b15..27ce252 100644 --- a/cmd/psweb/ln/common.go +++ b/cmd/psweb/ln/common.go @@ -151,7 +151,7 @@ var ( lastForwardTS = make(map[uint64]int64) // prevents starting another fee update while the first still running - autoFeeApplyAllIsRunning = false + autoFeeIsRunning = false ) func toSats(amount float64) int64 { diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index f5a1927..b4d62c7 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -972,7 +972,7 @@ func subscribeForwards(ctx context.Context, client routerrpc.RouterClient) error } defer cleanup() - ApplyAutoFee(client, htlcEvent.OutgoingChannelId, true) + applyAutoFee(client, htlcEvent.OutgoingChannelId, true) } case *routerrpc.HtlcEvent_SettleEvent: @@ -1001,7 +1001,7 @@ func subscribeForwards(ctx context.Context, client routerrpc.RouterClient) error defer cleanup() // calculate with new balance - ApplyAutoFee(client, htlc.forwardingEvent.ChanIdOut, false) + applyAutoFee(client, htlc.forwardingEvent.ChanIdOut, false) break } } @@ -1670,12 +1670,15 @@ func SetHtlcSize(peerNodeId string, return nil } -func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC bool) { +// for failed HTLC only +func applyAutoFee(client lnrpc.LightningClient, channelId uint64, htlcFail bool) { - if !AutoFeeEnabledAll || !AutoFeeEnabled[channelId] { + if !AutoFeeEnabledAll || !AutoFeeEnabled[channelId] || autoFeeIsRunning { return } + autoFeeIsRunning = true + params := &AutoFeeDefaults if AutoFee[channelId] != nil { // channel has custom parameters @@ -1687,6 +1690,7 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo // get my node id res, err := client.GetInfo(ctx, &lnrpc.GetInfoRequest{}) if err != nil { + autoFeeIsRunning = false return } myNodeId = res.GetIdentityPubkey() @@ -1695,6 +1699,7 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo ChanId: channelId, }) if err != nil { + autoFeeIsRunning = false return } @@ -1708,9 +1713,11 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo oldFee := int(policy.FeeRateMilliMsat) newFee := oldFee + // get balances bytePeer, err := hex.DecodeString(peerId) if err != nil { + autoFeeIsRunning = false return } @@ -1718,6 +1725,7 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo Peer: bytePeer, }) if err != nil { + autoFeeIsRunning = false return } @@ -1726,6 +1734,7 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo if ch.ChanId == channelId { if ch.UnsettledBalance != 0 { // skip af while htlcs are pending + autoFeeIsRunning = false return } localBalance = ch.LocalBalance @@ -1734,14 +1743,14 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo } liqPct := int(localBalance * 100 / r.Capacity) - - if failedHTLC { + if htlcFail { if liqPct <= params.LowLiqPct { // increase fee to help prevent further failed HTLCs newFee += params.FailedBumpPPM } else { // move threshold or do nothing moveLowLiqThreshold(channelId, params.FailedMoveThreshold) + autoFeeIsRunning = false return } } else { @@ -1756,40 +1765,93 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo } } - // set inbound fee - if HasInboundFees() && liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat != int32(params.LowLiqDiscount) { - // set discount - SetFeeRate(peerId, channelId, int64(params.LowLiqDiscount), true, false) - // log the last change - LogFee(channelId, int(policy.InboundFeeRateMilliMsat), params.LowLiqDiscount, true, false) - } + autoFeeIsRunning = false } -func ApplyAutoFeeAll() { +func ApplyAutoFees() { - if !AutoFeeEnabledAll || autoFeeApplyAllIsRunning { + if !AutoFeeEnabledAll || autoFeeIsRunning { return } - autoFeeApplyAllIsRunning = true + autoFeeIsRunning = true client, cleanup, err := GetClient() if err != nil { + autoFeeIsRunning = false return } defer cleanup() ctx := context.Background() + if myNodeId == "" { + // get my node id + res, err := client.GetInfo(ctx, &lnrpc.GetInfoRequest{}) + if err != nil { + autoFeeIsRunning = false + return + } + myNodeId = res.GetIdentityPubkey() + } + res, err := client.ListChannels(ctx, &lnrpc.ListChannelsRequest{ ActiveOnly: true, }) if err != nil { + autoFeeIsRunning = false return } for _, ch := range res.Channels { - ApplyAutoFee(client, ch.ChanId, false) + if ch.UnsettledBalance != 0 { + // skip af while htlcs are pending + continue + } + + params := &AutoFeeDefaults + if AutoFee[ch.ChanId] != nil { + // channel has custom parameters + params = AutoFee[ch.ChanId] + } + + r, err := client.GetChanInfo(ctx, &lnrpc.ChanInfoRequest{ + ChanId: ch.ChanId, + }) + if err != nil { + autoFeeIsRunning = false + return + } + + policy := r.Node1Policy + peerId := r.Node2Pub + if r.Node1Pub != myNodeId { + // the first policy is not ours, use the second + policy = r.Node2Policy + peerId = r.Node1Pub + } + + oldFee := int(policy.FeeRateMilliMsat) + newFee := oldFee + liqPct := int(ch.LocalBalance * 100 / r.Capacity) + + newFee = calculateAutoFee(ch.ChanId, params, liqPct, oldFee) + + // set the new rate + if newFee != oldFee { + if SetFeeRate(peerId, ch.ChanId, int64(newFee), false, false) == nil { + // log the last change + LogFee(ch.ChanId, oldFee, newFee, false, false) + } + } + + // set inbound fee + if HasInboundFees() && liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat != int32(params.LowLiqDiscount) { + // set discount + SetFeeRate(peerId, ch.ChanId, int64(params.LowLiqDiscount), true, false) + // log the last change + LogFee(ch.ChanId, int(policy.InboundFeeRateMilliMsat), params.LowLiqDiscount, true, false) + } } - autoFeeApplyAllIsRunning = false + autoFeeIsRunning = false } diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index ae6c0fa..b84c024 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -348,7 +348,7 @@ func onTimer() { go ln.SubscribeAll() // execute auto fee - go ln.ApplyAutoFeeAll() + go ln.ApplyAutoFees() } func liquidBackup(force bool) { diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 0ba4e27..2046a5f 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -240,10 +240,10 @@ Alias - Cap - % - Rate - In + Cap + % + Rate + In AF Rules ☑ @@ -253,8 +253,8 @@ {{range .ChannelList}} {{.Alias}} - {{m .Capacity}} - {{m .Capacity}} + Date: Wed, 19 Jun 2024 21:45:19 +0200 Subject: [PATCH 11/32] Allow setting manual fee discounts --- cmd/psweb/ln/lnd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index 923ee07..5d41efb 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -1851,8 +1851,8 @@ func ApplyAutoFees() { } } - // set inbound fee - if HasInboundFees() && liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat != int32(params.LowLiqDiscount) { + // set inbound fee discount if larger than current + if HasInboundFees() && liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat > int32(params.LowLiqDiscount) { // set discount _, err := SetFeeRate(peerId, ch.ChanId, int64(params.LowLiqDiscount), true, false) if err == nil { From feec602389b2d4a482518b3a9e3021fe79d89d38 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Wed, 19 Jun 2024 21:55:32 +0200 Subject: [PATCH 12/32] Keep manual inbound fee --- cmd/psweb/ln/lnd.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index 5d41efb..32a8573 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -1851,13 +1851,27 @@ func ApplyAutoFees() { } } - // set inbound fee discount if larger than current - if HasInboundFees() && liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat > int32(params.LowLiqDiscount) { - // set discount - _, err := SetFeeRate(peerId, ch.ChanId, int64(params.LowLiqDiscount), true, false) - if err == nil { - // log the last change - LogFee(ch.ChanId, int(policy.InboundFeeRateMilliMsat), params.LowLiqDiscount, true, false) + if HasInboundFees() { + toSet := false + discountRate := int64(0) + + if liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat > int32(params.LowLiqDiscount) { + // set inbound fee discount if larger than current + discountRate = int64(params.LowLiqDiscount) + toSet = true + } else if liqPct >= params.LowLiqPct { + // remove discount unless it was manually set + if !LastAutoFeeLog(ch.ChanId, true).IsManual { + toSet = true + } + } + + if toSet { + oldRate, err := SetFeeRate(peerId, ch.ChanId, discountRate, true, false) + if err == nil { + // log the last change + LogFee(ch.ChanId, oldRate, int(discountRate), true, false) + } } } } From 878a1052fcce569e8bfa2d4a04a686c282dcc742 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Wed, 19 Jun 2024 21:57:18 +0200 Subject: [PATCH 13/32] fix bug --- cmd/psweb/ln/lnd.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index 32a8573..71a4f74 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -1855,11 +1855,11 @@ func ApplyAutoFees() { toSet := false discountRate := int64(0) - if liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat > int32(params.LowLiqDiscount) { - // set inbound fee discount if larger than current + if liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat != int32(params.LowLiqDiscount) { + // set inbound fee discount discountRate = int64(params.LowLiqDiscount) toSet = true - } else if liqPct >= params.LowLiqPct { + } else if liqPct >= params.LowLiqPct && policy.InboundFeeRateMilliMsat < 0 { // remove discount unless it was manually set if !LastAutoFeeLog(ch.ChanId, true).IsManual { toSet = true From 21e0164ce8cf7864663a358d799e8da4e0f27f1b Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 00:09:30 +0200 Subject: [PATCH 14/32] Add ppm chart --- .vscode/launch.json | 2 +- cmd/psweb/handlers.go | 14 ++++- cmd/psweb/ln/common.go | 10 ++++ cmd/psweb/ln/lnd.go | 19 +++++++ cmd/psweb/main.go | 6 +++ cmd/psweb/templates/af.gohtml | 82 ++++++++++++++++++++++++++++- cmd/psweb/templates/reusable.gohtml | 4 ++ 7 files changed, 133 insertions(+), 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 1cda95b..06ff138 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,7 @@ "type": "go", "request": "launch", "mode": "auto", - "buildFlags": "-tags cln", + "buildFlags": "-tags lnd", "program": "${workspaceFolder}/cmd/psweb/", "showLog": false, "envFile": "${workspaceFolder}/.env", diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index e02f336..b9b9fcb 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -867,7 +867,7 @@ func afHandler(w http.ResponseWriter, r *http.Request) { // sort by LocalPct descending sort.Slice(channelList, func(i, j int) bool { - return channelList[i].LocalPct > channelList[j].LocalPct + return channelList[i].LocalPct < channelList[j].LocalPct }) //check for error errorMessage to display errorMessage := "" @@ -883,6 +883,16 @@ func afHandler(w http.ResponseWriter, r *http.Request) { popupMessage = keys[0] } + chart, maxAmount := ln.PlotPPM(channelId) + if maxAmount > 0 { + // calculate bobble radius and apply labels + // max R = 20 + for _, p := range *chart { + p.R = 20 * p.Amount / maxAmount + p.Label = "Routed: " + formatWithThousandSeparators(p.Amount) + "\nFee: " + formatWithThousandSeparators(p.Fee) + "\nPPM: " + formatWithThousandSeparators(p.PPM) + } + } + type Page struct { Authenticated bool ErrorMessage string @@ -899,6 +909,7 @@ func afHandler(w http.ResponseWriter, r *http.Request) { Enabled bool // for the displayed channel AnyEnabled bool // for any channel HasInboundFees bool + Chart *[]ln.DataPoint } data := Page{ @@ -917,6 +928,7 @@ func afHandler(w http.ResponseWriter, r *http.Request) { Enabled: ln.AutoFeeEnabled[channelId], AnyEnabled: anyEnabled, HasInboundFees: ln.HasInboundFees(), + Chart: chart, } // executing template named "af" diff --git a/cmd/psweb/ln/common.go b/cmd/psweb/ln/common.go index dc8925e..fedae2c 100644 --- a/cmd/psweb/ln/common.go +++ b/cmd/psweb/ln/common.go @@ -122,6 +122,16 @@ type AutoFeeEvent struct { IsManual bool } +// for chart plotting +type DataPoint struct { + TS uint64 + Amount uint64 + Fee uint64 + PPM uint64 + R uint64 + Label string +} + var ( // lightning payments from swap out initiator to receiver SwapRebates = make(map[string]int64) diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index 71a4f74..fe6faa0 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -1878,3 +1878,22 @@ func ApplyAutoFees() { autoFeeIsRunning = false } + +func PlotPPM(channelId uint64) (*[]DataPoint, uint64) { + var plot []DataPoint + maxAmount := uint64(0) + + for _, e := range forwardsOut[channelId] { + if maxAmount < e.AmtOut { + maxAmount = e.AmtOut + } + plot = append(plot, DataPoint{ + TS: e.TimestampNs / 1_000_000_000, + Amount: e.AmtOut, + Fee: e.Fee, + PPM: e.FeeMsat * 1_000_000 / e.AmtOutMsat, + }) + } + + return &plot, maxAmount +} diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index b84c024..b33d894 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -133,6 +133,7 @@ func main() { "fmt": formatWithThousandSeparators, "fs": formatSigned, "m": toMil, + "last": last, }). ParseFS(tplFolder, templateNames...)) @@ -1496,3 +1497,8 @@ func feeInputField(peerNodeId string, channelId uint64, direction string, feePer return t } + +// Template function to check if the element is the last one in the slice +func last(x int, a interface{}) bool { + return x == len(*(a.(*[]ln.DataPoint)))-1 +} diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 2046a5f..88a6c92 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -211,8 +211,15 @@ {{end}}
+ + + {{if ne .ChannelId 0}} +
+
+ +
- + {{end}}
@@ -259,7 +266,7 @@ color:lightgreen {{end}} {{if gt .AutoFee.LowLiqPct .LocalPct}} - color:red + color:pink {{end}}{{end}}">{{fmt .LocalPct}} {{fs .FeeRate}} {{fs .InboundRate}} @@ -296,6 +303,77 @@ } } } + + // Get the context of the canvas element we want to select + var ctx = document.getElementById('myScatterChart').getContext('2d'); + + // Create the chart + var myScatterChart = new Chart(ctx, { + type: 'bubble', // The type of chart we want to create + data: { + datasets: [{ + label: 'Realized Routing PPM', + data: [ + {{range $index, $element := .Chart}} + { x: new Date({{$element.TS}} * 1000), y: {{$element.PPM}}, r: {{$element.R}}, label: "{{$element.Label}}" }{{if not (last $index $.Chart)}},{{end}} + {{end}} + ], + backgroundColor: 'rgba(54, 162, 235, 0.8)', + borderColor: 'rgba(75, 192, 192, 1)', + borderWidth: 1 + }] + }, + options: { + scales: { + x: { + type: 'time', + time: { + unit: 'month' + }, + position: 'bottom', + ticks: { + color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' + }, + title: { + display: false, + text: 'Date', + color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' + } + }, + y: { + beginAtZero: false, + ticks: { + color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' + }, + title: { + display: true, + text: 'PPM', + color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' + } + } + }, + plugins: { + tooltip: { + callbacks: { + label: function(context) { + let label = context.raw.label || ''; + return label; + }, + }, + backgroundColor: 'rgba(0, 0, 0, 0.8)', + titleColor: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}', + bodyColor: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}', + borderColor: '{{if eq .ColorScheme "dark"}}rgba(255, 255, 255, 0.8){{else}}rgba(0, 0, 0, 0.8){{end}}', + borderWidth: 1, + }, + legend: { + labels: { + color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' + } + } + } + } + });
{{template "footer" .}} diff --git a/cmd/psweb/templates/reusable.gohtml b/cmd/psweb/templates/reusable.gohtml index 0b47b6f..dad8a96 100644 --- a/cmd/psweb/templates/reusable.gohtml +++ b/cmd/psweb/templates/reusable.gohtml @@ -94,6 +94,10 @@ } } + + + +
{{if eq .ErrorMessage "welcome"}} From 6733b6805dad49ac0a5d691f08d8857789bcf597 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 00:20:28 +0200 Subject: [PATCH 15/32] test --- cmd/psweb/handlers.go | 3 ++- cmd/psweb/ln/lnd.go | 21 ++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index b9b9fcb..0c728d3 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -884,8 +884,9 @@ func afHandler(w http.ResponseWriter, r *http.Request) { } chart, maxAmount := ln.PlotPPM(channelId) + log.Println(maxAmount) if maxAmount > 0 { - // calculate bobble radius and apply labels + // calculate bubble radii and apply labels // max R = 20 for _, p := range *chart { p.R = 20 * p.Amount / maxAmount diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index fe6faa0..aa867f9 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -1884,15 +1884,18 @@ func PlotPPM(channelId uint64) (*[]DataPoint, uint64) { maxAmount := uint64(0) for _, e := range forwardsOut[channelId] { - if maxAmount < e.AmtOut { - maxAmount = e.AmtOut - } - plot = append(plot, DataPoint{ - TS: e.TimestampNs / 1_000_000_000, - Amount: e.AmtOut, - Fee: e.Fee, - PPM: e.FeeMsat * 1_000_000 / e.AmtOutMsat, - }) + // ignore small forwards + if e.AmtOut > 1000 { + if maxAmount < e.AmtOut { + maxAmount = e.AmtOut + } + plot = append(plot, DataPoint{ + TS: e.TimestampNs / 1_000_000_000, + Amount: e.AmtOut, + Fee: e.Fee, + PPM: e.FeeMsat * 1_000_000 / e.AmtOutMsat, + }) + } } return &plot, maxAmount From 10093085c9a893177c8f0bd4238245ee5d66f9fe Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 00:26:35 +0200 Subject: [PATCH 16/32] test --- cmd/psweb/handlers.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index 0c728d3..96152e0 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -884,16 +884,18 @@ func afHandler(w http.ResponseWriter, r *http.Request) { } chart, maxAmount := ln.PlotPPM(channelId) - log.Println(maxAmount) if maxAmount > 0 { // calculate bubble radii and apply labels // max R = 20 for _, p := range *chart { p.R = 20 * p.Amount / maxAmount p.Label = "Routed: " + formatWithThousandSeparators(p.Amount) + "\nFee: " + formatWithThousandSeparators(p.Fee) + "\nPPM: " + formatWithThousandSeparators(p.PPM) + log.Println(p.R) } } + log.Println(*chart) + type Page struct { Authenticated bool ErrorMessage string From 1a7f09292071cf28b607d3fd88adde7de3d8a27c Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 00:29:23 +0200 Subject: [PATCH 17/32] test --- cmd/psweb/handlers.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index 96152e0..96fb2bf 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -887,10 +887,9 @@ func afHandler(w http.ResponseWriter, r *http.Request) { if maxAmount > 0 { // calculate bubble radii and apply labels // max R = 20 - for _, p := range *chart { - p.R = 20 * p.Amount / maxAmount - p.Label = "Routed: " + formatWithThousandSeparators(p.Amount) + "\nFee: " + formatWithThousandSeparators(p.Fee) + "\nPPM: " + formatWithThousandSeparators(p.PPM) - log.Println(p.R) + for i, p := range *chart { + (*chart)[i].R = 20 * p.Amount / maxAmount + (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + "\nFee: " + formatWithThousandSeparators(p.Fee) + "\nPPM: " + formatWithThousandSeparators(p.PPM) } } From 762ff35c8829759d78f2f40af5cb61921945bc67 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 00:34:35 +0200 Subject: [PATCH 18/32] test --- cmd/psweb/handlers.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index 96fb2bf..38e1a9d 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -889,12 +889,10 @@ func afHandler(w http.ResponseWriter, r *http.Request) { // max R = 20 for i, p := range *chart { (*chart)[i].R = 20 * p.Amount / maxAmount - (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + "\nFee: " + formatWithThousandSeparators(p.Fee) + "\nPPM: " + formatWithThousandSeparators(p.PPM) + (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + "
Fee: " + formatWithThousandSeparators(p.Fee) + "
PPM: " + formatWithThousandSeparators(p.PPM) } } - log.Println(*chart) - type Page struct { Authenticated bool ErrorMessage string From 31779498a2d05539645514ce7476a983320dd703 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 00:41:35 +0200 Subject: [PATCH 19/32] test --- cmd/psweb/handlers.go | 2 +- cmd/psweb/templates/af.gohtml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index 38e1a9d..db94ef9 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -889,7 +889,7 @@ func afHandler(w http.ResponseWriter, r *http.Request) { // max R = 20 for i, p := range *chart { (*chart)[i].R = 20 * p.Amount / maxAmount - (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + "
Fee: " + formatWithThousandSeparators(p.Fee) + "
PPM: " + formatWithThousandSeparators(p.PPM) + (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + ", Fee: " + formatWithThousandSeparators(p.Fee) + ", PPM: " + formatWithThousandSeparators(p.PPM) } } diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 88a6c92..6db5842 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -357,6 +357,7 @@ callbacks: { label: function(context) { let label = context.raw.label || ''; + label += '
' + moment(context.raw.x).format('YYYY-MM-DD') return label; }, }, From 33204eb460ce6852c32c60efc76b8954b8190259 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 00:43:40 +0200 Subject: [PATCH 20/32] test --- cmd/psweb/templates/af.gohtml | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 6db5842..1616c19 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -366,6 +366,7 @@ bodyColor: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}', borderColor: '{{if eq .ColorScheme "dark"}}rgba(255, 255, 255, 0.8){{else}}rgba(0, 0, 0, 0.8){{end}}', borderWidth: 1, + displayColors: false // Hide color indicators in tooltip }, legend: { labels: { From e88a3e887d2faf145ec8df9493805aa658290af8 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 00:48:50 +0200 Subject: [PATCH 21/32] test --- cmd/psweb/handlers.go | 2 +- cmd/psweb/templates/af.gohtml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index db94ef9..38e1a9d 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -889,7 +889,7 @@ func afHandler(w http.ResponseWriter, r *http.Request) { // max R = 20 for i, p := range *chart { (*chart)[i].R = 20 * p.Amount / maxAmount - (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + ", Fee: " + formatWithThousandSeparators(p.Fee) + ", PPM: " + formatWithThousandSeparators(p.PPM) + (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + "
Fee: " + formatWithThousandSeparators(p.Fee) + "
PPM: " + formatWithThousandSeparators(p.PPM) } } diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 1616c19..49e8c6a 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -358,6 +358,7 @@ label: function(context) { let label = context.raw.label || ''; label += '
' + moment(context.raw.x).format('YYYY-MM-DD') + label = label.replace(/\
/g, '\n'); // Replace '
' with actual line breaks return label; }, }, From 8fab90a2282917e420290a4e371423e76d7589d2 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 01:06:10 +0200 Subject: [PATCH 22/32] test --- cmd/psweb/handlers.go | 13 +++++-------- cmd/psweb/ln/lnd.go | 8 ++------ cmd/psweb/templates/af.gohtml | 12 +++++++++--- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index 38e1a9d..5d87c95 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -883,14 +883,11 @@ func afHandler(w http.ResponseWriter, r *http.Request) { popupMessage = keys[0] } - chart, maxAmount := ln.PlotPPM(channelId) - if maxAmount > 0 { - // calculate bubble radii and apply labels - // max R = 20 - for i, p := range *chart { - (*chart)[i].R = 20 * p.Amount / maxAmount - (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + "
Fee: " + formatWithThousandSeparators(p.Fee) + "
PPM: " + formatWithThousandSeparators(p.PPM) - } + chart := ln.PlotPPM(channelId) + // calculate bubble radii in 100k + for i, p := range *chart { + (*chart)[i].R = p.Amount / 100_000 + (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + "\nFee: " + formatWithThousandSeparators(p.Fee) + "\nPPM: " + formatWithThousandSeparators(p.PPM) } type Page struct { diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index aa867f9..9b6afa2 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -1879,16 +1879,12 @@ func ApplyAutoFees() { autoFeeIsRunning = false } -func PlotPPM(channelId uint64) (*[]DataPoint, uint64) { +func PlotPPM(channelId uint64) *[]DataPoint { var plot []DataPoint - maxAmount := uint64(0) for _, e := range forwardsOut[channelId] { // ignore small forwards if e.AmtOut > 1000 { - if maxAmount < e.AmtOut { - maxAmount = e.AmtOut - } plot = append(plot, DataPoint{ TS: e.TimestampNs / 1_000_000_000, Amount: e.AmtOut, @@ -1898,5 +1894,5 @@ func PlotPPM(channelId uint64) (*[]DataPoint, uint64) { } } - return &plot, maxAmount + return &plot } diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 49e8c6a..2b4aaa0 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -315,7 +315,7 @@ label: 'Realized Routing PPM', data: [ {{range $index, $element := .Chart}} - { x: new Date({{$element.TS}} * 1000), y: {{$element.PPM}}, r: {{$element.R}}, label: "{{$element.Label}}" }{{if not (last $index $.Chart)}},{{end}} + { x: new Date({{$element.TS}} * 1000), y: {{$element.PPM}}, r: {{$element.R}}, label: `{{$element.Label | jsStr}}` }{{if not (last $index $.Chart)}},{{end}} {{end}} ], backgroundColor: 'rgba(54, 162, 235, 0.8)', @@ -357,8 +357,6 @@ callbacks: { label: function(context) { let label = context.raw.label || ''; - label += '
' + moment(context.raw.x).format('YYYY-MM-DD') - label = label.replace(/\
/g, '\n'); // Replace '
' with actual line breaks return label; }, }, @@ -377,6 +375,14 @@ } } }); + + // Custom function to handle JavaScript string escaping for Go templates + function jsStr(s) { + return s.replace(/\\/g, '\\\\').replace(/`/g, '\\`').replace(/\$/g, '\\$').replace(/\n/g, '\\n').replace(/\r/g, '\\r').replace(/\t/g, '\\t'); + } + + // Add custom function to the template context + {{define "jsStr"}}jsStr{{end}}
{{template "footer" .}} From cc798d432de3bffeb2caa7797356b2ae450042f2 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 01:09:47 +0200 Subject: [PATCH 23/32] test --- cmd/psweb/templates/af.gohtml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 2b4aaa0..fcd7a75 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -375,14 +375,6 @@ } } }); - - // Custom function to handle JavaScript string escaping for Go templates - function jsStr(s) { - return s.replace(/\\/g, '\\\\').replace(/`/g, '\\`').replace(/\$/g, '\\$').replace(/\n/g, '\\n').replace(/\r/g, '\\r').replace(/\t/g, '\\t'); - } - - // Add custom function to the template context - {{define "jsStr"}}jsStr{{end}} {{template "footer" .}} From f58dd6421ad752b06aa6512257f7ddaa93c5ea0b Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 01:10:54 +0200 Subject: [PATCH 24/32] test --- cmd/psweb/templates/af.gohtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index fcd7a75..c403633 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -315,7 +315,7 @@ label: 'Realized Routing PPM', data: [ {{range $index, $element := .Chart}} - { x: new Date({{$element.TS}} * 1000), y: {{$element.PPM}}, r: {{$element.R}}, label: `{{$element.Label | jsStr}}` }{{if not (last $index $.Chart)}},{{end}} + { x: new Date({{$element.TS}} * 1000), y: {{$element.PPM}}, r: {{$element.R}}, label: `{{$element.Label}}` }{{if not (last $index $.Chart)}},{{end}} {{end}} ], backgroundColor: 'rgba(54, 162, 235, 0.8)', From 8414adb071c46618897faa44c21127faa287804c Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 01:18:40 +0200 Subject: [PATCH 25/32] Test --- cmd/psweb/handlers.go | 2 +- cmd/psweb/templates/af.gohtml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index 5d87c95..271ca86 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -887,7 +887,7 @@ func afHandler(w http.ResponseWriter, r *http.Request) { // calculate bubble radii in 100k for i, p := range *chart { (*chart)[i].R = p.Amount / 100_000 - (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + "\nFee: " + formatWithThousandSeparators(p.Fee) + "\nPPM: " + formatWithThousandSeparators(p.PPM) + (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + ", Fee: " + formatWithThousandSeparators(p.Fee) + ", PPM: " + formatWithThousandSeparators(p.PPM) } type Page struct { diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index c403633..87c4cac 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -375,6 +375,7 @@ } } }); + {{template "footer" .}} From 1c3483e538a2c469b8c169c80809a1aa7922956a Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 01:28:58 +0200 Subject: [PATCH 26/32] test --- cmd/psweb/handlers.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index 271ca86..9779bda 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -6,10 +6,16 @@ import ( "fmt" "io" "log" + "math" "net" "net/http" "os" "path/filepath" + "sort" + "strconv" + "strings" + "time" + "peerswap-web/cmd/psweb/bitcoin" "peerswap-web/cmd/psweb/config" "peerswap-web/cmd/psweb/db" @@ -17,10 +23,6 @@ import ( "peerswap-web/cmd/psweb/liquid" "peerswap-web/cmd/psweb/ln" "peerswap-web/cmd/psweb/ps" - "sort" - "strconv" - "strings" - "time" "github.com/elementsproject/peerswap/peerswaprpc" "github.com/gorilla/sessions" @@ -884,9 +886,9 @@ func afHandler(w http.ResponseWriter, r *http.Request) { } chart := ln.PlotPPM(channelId) - // calculate bubble radii in 100k + // bubble square area reflects amount for i, p := range *chart { - (*chart)[i].R = p.Amount / 100_000 + (*chart)[i].R = uint64(math.Sqrt(float64(p.Amount) / 100_000)) (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + ", Fee: " + formatWithThousandSeparators(p.Fee) + ", PPM: " + formatWithThousandSeparators(p.PPM) } From de4bf18fe17314ffd7a32a4c6d201e8edc3ff03b Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 01:29:54 +0200 Subject: [PATCH 27/32] test --- cmd/psweb/handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index 9779bda..339c23e 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -888,7 +888,7 @@ func afHandler(w http.ResponseWriter, r *http.Request) { chart := ln.PlotPPM(channelId) // bubble square area reflects amount for i, p := range *chart { - (*chart)[i].R = uint64(math.Sqrt(float64(p.Amount) / 100_000)) + (*chart)[i].R = uint64(math.Sqrt(float64(p.Amount) / 10_000)) (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + ", Fee: " + formatWithThousandSeparators(p.Fee) + ", PPM: " + formatWithThousandSeparators(p.PPM) } From e748184f4e73b473d72f42675126a222c19a7966 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 01:35:11 +0200 Subject: [PATCH 28/32] test --- cmd/psweb/templates/af.gohtml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 87c4cac..f48ec1e 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -360,12 +360,6 @@ return label; }, }, - backgroundColor: 'rgba(0, 0, 0, 0.8)', - titleColor: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}', - bodyColor: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}', - borderColor: '{{if eq .ColorScheme "dark"}}rgba(255, 255, 255, 0.8){{else}}rgba(0, 0, 0, 0.8){{end}}', - borderWidth: 1, - displayColors: false // Hide color indicators in tooltip }, legend: { labels: { From bb30fb0fba68c8cb7c0e17178daf676d2e005dee Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 01:43:57 +0200 Subject: [PATCH 29/32] Remove legend --- cmd/psweb/templates/af.gohtml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index f48ec1e..dc2c504 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -215,6 +215,7 @@ {{if ne .ChannelId 0}}
+

Realized Routing PPM

@@ -312,7 +313,6 @@ type: 'bubble', // The type of chart we want to create data: { datasets: [{ - label: 'Realized Routing PPM', data: [ {{range $index, $element := .Chart}} { x: new Date({{$element.TS}} * 1000), y: {{$element.PPM}}, r: {{$element.R}}, label: `{{$element.Label}}` }{{if not (last $index $.Chart)}},{{end}} @@ -346,7 +346,7 @@ color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' }, title: { - display: true, + display: false, text: 'PPM', color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' } @@ -362,10 +362,8 @@ }, }, legend: { - labels: { - color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' - } - } + display: false + }, } } }); From f0ab4eee52a70d067a9d07674ebea901bdbb225a Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 01:57:31 +0200 Subject: [PATCH 30/32] Add grid --- cmd/psweb/templates/af.gohtml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index dc2c504..009190d 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -331,24 +331,28 @@ unit: 'month' }, position: 'bottom', - ticks: { - color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' - }, + {{if eq .ColorScheme "dark"}} + grid: { + color: 'rgba(255, 255, 255, 0.1)', // Set grid line color (white with reduced opacity) + borderColor: 'rgba(255, 255, 255, 0.2)', // Set grid border color (white with slightly higher opacity) + borderWidth: 1 // Grid border width + }, + {{end}} title: { display: false, - text: 'Date', - color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' } }, y: { beginAtZero: false, - ticks: { - color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' - }, + {{if eq .ColorScheme "dark"}} + grid: { + color: 'rgba(255, 255, 255, 0.1)', // Set grid line color (white with reduced opacity) + borderColor: 'rgba(255, 255, 255, 0.2)', // Set grid border color (white with slightly higher opacity) + borderWidth: 1 // Grid border width + }, + {{end}} title: { display: false, - text: 'PPM', - color: '{{if eq .ColorScheme "dark"}}white{{else}}black{{end}}' } } }, From b1bf77d0cb3c1a6f4604a505079cdbfaf86dc03c Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 02:00:51 +0200 Subject: [PATCH 31/32] white font --- cmd/psweb/templates/af.gohtml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 009190d..b88e8d0 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -337,6 +337,9 @@ borderColor: 'rgba(255, 255, 255, 0.2)', // Set grid border color (white with slightly higher opacity) borderWidth: 1 // Grid border width }, + ticks: { + color: 'white' // Set x-axis font color to white + }, {{end}} title: { display: false, @@ -350,6 +353,9 @@ borderColor: 'rgba(255, 255, 255, 0.2)', // Set grid border color (white with slightly higher opacity) borderWidth: 1 // Grid border width }, + ticks: { + color: 'white' // Set x-axis font color to white + }, {{end}} title: { display: false, From 36ed30a81222a66d77256481bb1053caa883a4f8 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 21 Jun 2024 02:17:47 +0200 Subject: [PATCH 32/32] Complete changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24a3ae8..1c288ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - AF: for HTLC Fails above Low Liq % allow increasing Low Liq % threshold - AF: log full fee changes history, including inbound - AF: reduce LND load when applying auto fees +- AF: add realized PPM chart for the channel to help decide AF parameters ## 1.5.3