From f550fb2ad4c755748289cf29312a77e63258dde9 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Wed, 26 Jul 2023 14:59:44 +0200 Subject: [PATCH 01/12] add reconciliation script --- lnd/lnd.go | 4 ++ reconciliation_lost_invoices/main.go | 59 +++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/lnd/lnd.go b/lnd/lnd.go index c1640e95..1b3ea057 100644 --- a/lnd/lnd.go +++ b/lnd/lnd.go @@ -137,3 +137,7 @@ func (wrapper *LNDWrapper) IsIdentityPubkey(pubkey string) (isOurPubkey bool) { func (wrapper *LNDWrapper) GetMainPubkey() (pubkey string) { return wrapper.IdentityPubkey } + +func (wrapper *LNDWrapper) ListInvoices(ctx context.Context, req *lnrpc.ListInvoiceRequest, options ...grpc.CallOption) (*lnrpc.ListInvoiceResponse, error) { + return wrapper.client.ListInvoices(ctx, req, options...) +} diff --git a/reconciliation_lost_invoices/main.go b/reconciliation_lost_invoices/main.go index bdd41ec2..87c5b779 100644 --- a/reconciliation_lost_invoices/main.go +++ b/reconciliation_lost_invoices/main.go @@ -2,17 +2,23 @@ package main import ( "context" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" "fmt" "log" + "time" "github.com/getAlby/lndhub.go/db" + "github.com/getAlby/lndhub.go/db/models" "github.com/getAlby/lndhub.go/lib" "github.com/getAlby/lndhub.go/lib/service" "github.com/getAlby/lndhub.go/lnd" - "github.com/getsentry/sentry-go" "github.com/joho/godotenv" "github.com/kelseyhightower/envconfig" "github.com/labstack/echo/v4" + "github.com/lightningnetwork/lnd/lnrpc" ) // script to reconcile pending payments between the backup node and the database @@ -67,9 +73,52 @@ func main() { InvoicePubSub: service.NewPubsub(), } - err = svc.CheckAllPendingOutgoingPayments(startupCtx) - if err != nil { - sentry.CaptureException(err) - svc.Logger.Error(err) + numDays := 30 + numMaxInvoices := 100 + ctx := context.Background() + //for loop: + offset := uint64(0) + // - fetch next 100 invoices from LND + startTime := time.Now() + counter := 0 + for { + + invoiceResp, err := lndClient.ListInvoices(ctx, &lnrpc.ListInvoiceRequest{ + PendingOnly: false, + IndexOffset: offset, + NumMaxInvoices: uint64(numMaxInvoices), + Reversed: true, + }) + if err != nil { + svc.Logger.Fatal(err) + } + // for all invoices: + for _, lndInvoice := range invoiceResp.Invoices { + creationDate := time.Unix(lndInvoice.CreationDate, 0) + // - return if invoice older than time X + if creationDate.Before(time.Now().Add(-1 * time.Duration(numDays) * 24 * time.Hour)) { + fmt.Printf("time elapsed: %f total invoices %d \n", time.Now().Sub(startTime).Seconds(), counter) + return + } + // - get payment hash and do a db query + var dbInvoice models.Invoice + counter += 1 + + err := svc.DB.NewSelect().Model(&dbInvoice).Where("invoice.r_hash = ?", hex.EncodeToString(lndInvoice.RHash)).Limit(1).Scan(ctx) + if err != nil { + // - if not found, dump invoice json + if errors.Is(err, sql.ErrNoRows) { + marshalled, err := json.Marshal(lndInvoice) + if err != nil { + svc.Logger.Fatal(err) + } + fmt.Println(string(marshalled)) + fmt.Println() + continue + } + svc.Logger.Fatal(err) + } + } + offset = invoiceResp.FirstIndexOffset } } From e1954e7b9a8b1b335fa758329dc6f6151df7998b Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Wed, 26 Jul 2023 14:59:52 +0200 Subject: [PATCH 02/12] add reconciliation script --- reconciliation_lost_payments/main.go | 75 ++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 reconciliation_lost_payments/main.go diff --git a/reconciliation_lost_payments/main.go b/reconciliation_lost_payments/main.go new file mode 100644 index 00000000..bdd41ec2 --- /dev/null +++ b/reconciliation_lost_payments/main.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/getAlby/lndhub.go/db" + "github.com/getAlby/lndhub.go/lib" + "github.com/getAlby/lndhub.go/lib/service" + "github.com/getAlby/lndhub.go/lnd" + "github.com/getsentry/sentry-go" + "github.com/joho/godotenv" + "github.com/kelseyhightower/envconfig" + "github.com/labstack/echo/v4" +) + +// script to reconcile pending payments between the backup node and the database +func main() { + + c := &service.Config{} + + // Load configruation from environment variables + err := godotenv.Load(".env") + if err != nil { + fmt.Println("Failed to load .env file") + } + err = envconfig.Process("", c) + if err != nil { + log.Fatalf("Error loading environment variables: %v", err) + } + + // Setup logging to STDOUT or a configrued log file + logger := lib.Logger(c.LogFilePath) + + // Open a DB connection based on the configured DATABASE_URI + dbConn, err := db.Open(c) + if err != nil { + logger.Fatalf("Error initializing db connection: %v", err) + } + + // Migrate the DB + //Todo: use timeout for startupcontext + startupCtx := context.Background() + + // New Echo app + e := echo.New() + + // Init new LND client + lndClient, err := lnd.NewLNDclient(lnd.LNDoptions{ + Address: c.LNDAddress, + MacaroonFile: c.LNDMacaroonFile, + MacaroonHex: c.LNDMacaroonHex, + CertFile: c.LNDCertFile, + CertHex: c.LNDCertHex, + }, startupCtx) + if err != nil { + e.Logger.Fatalf("Error initializing the LND connection: %v", err) + } + logger.Infof("Connected to LND: %s ", lndClient.GetMainPubkey()) + + svc := &service.LndhubService{ + Config: c, + DB: dbConn, + LndClient: lndClient, + Logger: logger, + InvoicePubSub: service.NewPubsub(), + } + + err = svc.CheckAllPendingOutgoingPayments(startupCtx) + if err != nil { + sentry.CaptureException(err) + svc.Logger.Error(err) + } +} From c542259aa776056c202632601da7399cd0c0e163 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Thu, 27 Jul 2023 15:29:36 +0200 Subject: [PATCH 03/12] add hex dump r hash --- reconciliation_lost_invoices/main.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/reconciliation_lost_invoices/main.go b/reconciliation_lost_invoices/main.go index 87c5b779..180f20c1 100644 --- a/reconciliation_lost_invoices/main.go +++ b/reconciliation_lost_invoices/main.go @@ -21,10 +21,16 @@ import ( "github.com/lightningnetwork/lnd/lnrpc" ) +type ReconciliationConfig struct { + NumDays int `envconfig:"NUM_DAYS" default:"30"` + NumMaxInvoices int `envconfig:"NUM_MAX_INVOICES" default:"100"` +} + // script to reconcile pending payments between the backup node and the database func main() { c := &service.Config{} + rc := &ReconciliationConfig{} // Load configruation from environment variables err := godotenv.Load(".env") @@ -36,6 +42,11 @@ func main() { log.Fatalf("Error loading environment variables: %v", err) } + err = envconfig.Process("", rc) + if err != nil { + log.Fatalf("Error loading environment variables: %v", err) + } + // Setup logging to STDOUT or a configrued log file logger := lib.Logger(c.LogFilePath) @@ -73,20 +84,16 @@ func main() { InvoicePubSub: service.NewPubsub(), } - numDays := 30 - numMaxInvoices := 100 ctx := context.Background() //for loop: offset := uint64(0) // - fetch next 100 invoices from LND - startTime := time.Now() - counter := 0 for { invoiceResp, err := lndClient.ListInvoices(ctx, &lnrpc.ListInvoiceRequest{ PendingOnly: false, IndexOffset: offset, - NumMaxInvoices: uint64(numMaxInvoices), + NumMaxInvoices: uint64(rc.NumMaxInvoices), Reversed: true, }) if err != nil { @@ -96,18 +103,17 @@ func main() { for _, lndInvoice := range invoiceResp.Invoices { creationDate := time.Unix(lndInvoice.CreationDate, 0) // - return if invoice older than time X - if creationDate.Before(time.Now().Add(-1 * time.Duration(numDays) * 24 * time.Hour)) { - fmt.Printf("time elapsed: %f total invoices %d \n", time.Now().Sub(startTime).Seconds(), counter) + if creationDate.Before(time.Now().Add(-1 * time.Duration(rc.NumDays) * 24 * time.Hour)) { return } // - get payment hash and do a db query var dbInvoice models.Invoice - counter += 1 err := svc.DB.NewSelect().Model(&dbInvoice).Where("invoice.r_hash = ?", hex.EncodeToString(lndInvoice.RHash)).Limit(1).Scan(ctx) if err != nil { // - if not found, dump invoice json if errors.Is(err, sql.ErrNoRows) { + fmt.Printf("hex: %s\n", hex.EncodeToString(lndInvoice.RHash)) marshalled, err := json.Marshal(lndInvoice) if err != nil { svc.Logger.Fatal(err) From d56cc02bc7d5a36c118d9ed974649d361c02f498 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Thu, 27 Jul 2023 15:36:30 +0200 Subject: [PATCH 04/12] add reconciliation build workflow --- .github/workflows/build_reconciliation.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/build_reconciliation.yaml diff --git a/.github/workflows/build_reconciliation.yaml b/.github/workflows/build_reconciliation.yaml new file mode 100644 index 00000000..a130eff0 --- /dev/null +++ b/.github/workflows/build_reconciliation.yaml @@ -0,0 +1,21 @@ +name: Docker build & push +on: + push: +jobs: + build: + env: + REGISTRY: ghcr.io + IMAGENAME: lndhub-reconciliation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + name: Check out code + - name: Docker build + uses: mr-smithers-excellent/docker-build-push@v5 + id: build + dockerfile: reconciliation.Dockerfile + with: + image: ${{ env.IMAGENAME }} + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 845a288ef7faf5aa96ed9ad26b190b452935c2ab Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Thu, 27 Jul 2023 15:36:39 +0200 Subject: [PATCH 05/12] add reconciliation build workflow --- reconciliation.Dockerfile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 reconciliation.Dockerfile diff --git a/reconciliation.Dockerfile b/reconciliation.Dockerfile new file mode 100644 index 00000000..0888b7c2 --- /dev/null +++ b/reconciliation.Dockerfile @@ -0,0 +1,24 @@ +FROM golang:1.20-alpine as builder + +# Move to working directory /build +WORKDIR /build + +# Copy and download dependency using go mod +COPY go.mod . +COPY go.sum . +RUN go mod download + +# Copy the code into the container +COPY . . + +# Build the application +WORKDIR /build/reconciliation_lost_invoices +RUN go build -o main + +# Start a new, final image to reduce size. +FROM alpine as final + +# Copy the binaries and entrypoint from the builder image. +COPY --from=builder /build/reconciliation_lost_invoices/main /bin/ + +ENTRYPOINT [ "/bin/main" ] From 230d951a52874ef3e89254670b34aee68dfd3c79 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Thu, 27 Jul 2023 15:37:11 +0200 Subject: [PATCH 06/12] add reconciliation build workflow --- .github/workflows/build_reconciliation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_reconciliation.yaml b/.github/workflows/build_reconciliation.yaml index a130eff0..c9e234c4 100644 --- a/.github/workflows/build_reconciliation.yaml +++ b/.github/workflows/build_reconciliation.yaml @@ -1,4 +1,4 @@ -name: Docker build & push +name: Reconciliation Docker build & push on: push: jobs: From e7fcf3fa40c45b69b8de9dfb1e97cb9ff6f7de46 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Thu, 27 Jul 2023 15:37:46 +0200 Subject: [PATCH 07/12] add reconciliation build workflow --- .github/workflows/build_reconciliation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_reconciliation.yaml b/.github/workflows/build_reconciliation.yaml index c9e234c4..04091e51 100644 --- a/.github/workflows/build_reconciliation.yaml +++ b/.github/workflows/build_reconciliation.yaml @@ -13,8 +13,8 @@ jobs: - name: Docker build uses: mr-smithers-excellent/docker-build-push@v5 id: build - dockerfile: reconciliation.Dockerfile with: + dockerfile: reconciliation.Dockerfile image: ${{ env.IMAGENAME }} registry: ${{ env.REGISTRY }} username: ${{ github.repository_owner }} From 7f873c36cf826051712da5bfa0e549bbead7b5a3 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 28 Jul 2023 10:42:25 +0200 Subject: [PATCH 08/12] fix query, add keysend log --- reconciliation_lost_invoices/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/reconciliation_lost_invoices/main.go b/reconciliation_lost_invoices/main.go index 180f20c1..d15e84b8 100644 --- a/reconciliation_lost_invoices/main.go +++ b/reconciliation_lost_invoices/main.go @@ -10,6 +10,7 @@ import ( "log" "time" + "github.com/getAlby/lndhub.go/common" "github.com/getAlby/lndhub.go/db" "github.com/getAlby/lndhub.go/db/models" "github.com/getAlby/lndhub.go/lib" @@ -109,11 +110,11 @@ func main() { // - get payment hash and do a db query var dbInvoice models.Invoice - err := svc.DB.NewSelect().Model(&dbInvoice).Where("invoice.r_hash = ?", hex.EncodeToString(lndInvoice.RHash)).Limit(1).Scan(ctx) + err := svc.DB.NewSelect().Model(&dbInvoice).Where("invoice.r_hash = ? AND state = ?", hex.EncodeToString(lndInvoice.RHash), common.InvoiceStateSettled).Limit(1).Scan(ctx) if err != nil { // - if not found, dump invoice json if errors.Is(err, sql.ErrNoRows) { - fmt.Printf("hex: %s\n", hex.EncodeToString(lndInvoice.RHash)) + fmt.Printf("keysend: %t hex: %s\n", lndInvoice.IsKeysend, hex.EncodeToString(lndInvoice.RHash)) marshalled, err := json.Marshal(lndInvoice) if err != nil { svc.Logger.Fatal(err) From 7b88e40494b5519b47a422ec26cb8e30c65762f5 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 28 Jul 2023 10:55:14 +0200 Subject: [PATCH 09/12] don't process unsettled invoices --- reconciliation_lost_invoices/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/reconciliation_lost_invoices/main.go b/reconciliation_lost_invoices/main.go index d15e84b8..73b66513 100644 --- a/reconciliation_lost_invoices/main.go +++ b/reconciliation_lost_invoices/main.go @@ -107,10 +107,14 @@ func main() { if creationDate.Before(time.Now().Add(-1 * time.Duration(rc.NumDays) * 24 * time.Hour)) { return } + //non-settled invoices don't matter + if !lndInvoice.Settled { + continue + } // - get payment hash and do a db query var dbInvoice models.Invoice - err := svc.DB.NewSelect().Model(&dbInvoice).Where("invoice.r_hash = ? AND state = ?", hex.EncodeToString(lndInvoice.RHash), common.InvoiceStateSettled).Limit(1).Scan(ctx) + err := svc.DB.NewSelect().Model(&dbInvoice).Where("type = ? AND invoice.r_hash = ? AND state = ?", common.InvoiceTypeIncoming, hex.EncodeToString(lndInvoice.RHash), common.InvoiceStateSettled).Limit(1).Scan(ctx) if err != nil { // - if not found, dump invoice json if errors.Is(err, sql.ErrNoRows) { From 8c1294572e97b812e5269e9d22a8069adb71bd96 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 28 Jul 2023 16:11:36 +0200 Subject: [PATCH 10/12] add decent logging --- reconciliation_lost_invoices/main.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/reconciliation_lost_invoices/main.go b/reconciliation_lost_invoices/main.go index 73b66513..c2691283 100644 --- a/reconciliation_lost_invoices/main.go +++ b/reconciliation_lost_invoices/main.go @@ -118,7 +118,22 @@ func main() { if err != nil { // - if not found, dump invoice json if errors.Is(err, sql.ErrNoRows) { - fmt.Printf("keysend: %t hex: %s\n", lndInvoice.IsKeysend, hex.EncodeToString(lndInvoice.RHash)) + + firstHtlc := &lnrpc.InvoiceHTLC{} + if len(lndInvoice.Htlcs) > 0 { + firstHtlc = lndInvoice.Htlcs[0] + } + messageJson := map[string]interface{}{ + "message": "reconciliation: potential missing invoice", + "r_hash": hex.EncodeToString(lndInvoice.RHash), + "keysend": lndInvoice.IsKeysend, + "custom_records": firstHtlc.CustomRecords, + } + msgBytes, err := json.Marshal(messageJson) + if err != nil { + svc.Logger.Fatal(err) + } + fmt.Printf(string(msgBytes)) marshalled, err := json.Marshal(lndInvoice) if err != nil { svc.Logger.Fatal(err) From 1cc0e2b1510be3d9522b3fdf54ea89ea5c25e265 Mon Sep 17 00:00:00 2001 From: Michael Bumann Date: Wed, 27 Dec 2023 17:19:26 +0200 Subject: [PATCH 11/12] Consistent naming for log entries --- reconciliation_lost_invoices/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reconciliation_lost_invoices/main.go b/reconciliation_lost_invoices/main.go index c2691283..44a418a4 100644 --- a/reconciliation_lost_invoices/main.go +++ b/reconciliation_lost_invoices/main.go @@ -125,7 +125,7 @@ func main() { } messageJson := map[string]interface{}{ "message": "reconciliation: potential missing invoice", - "r_hash": hex.EncodeToString(lndInvoice.RHash), + "payment_hash": hex.EncodeToString(lndInvoice.RHash), "keysend": lndInvoice.IsKeysend, "custom_records": firstHtlc.CustomRecords, } From 6de0c664ed11dd394648161b993dc5faa65ea465 Mon Sep 17 00:00:00 2001 From: Fmar Date: Wed, 21 Feb 2024 14:47:02 +0100 Subject: [PATCH 12/12] use InitLNClient instead of NewLNDclient + GetAllPendingPayments into CheckPendingOutgoingPayments --- .../main.go | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) rename {reconciliation_lost_payments => reconciliation_lost_invoices}/main.go (80%) diff --git a/reconciliation_lost_payments/main.go b/reconciliation_lost_invoices/main.go similarity index 80% rename from reconciliation_lost_payments/main.go rename to reconciliation_lost_invoices/main.go index bdd41ec2..a2285f9b 100644 --- a/reconciliation_lost_payments/main.go +++ b/reconciliation_lost_invoices/main.go @@ -46,14 +46,13 @@ func main() { // New Echo app e := echo.New() - // Init new LND client - lndClient, err := lnd.NewLNDclient(lnd.LNDoptions{ - Address: c.LNDAddress, - MacaroonFile: c.LNDMacaroonFile, - MacaroonHex: c.LNDMacaroonHex, - CertFile: c.LNDCertFile, - CertHex: c.LNDCertHex, - }, startupCtx) + //// Init new LND client + lnCfg, err := lnd.LoadConfig() + if err != nil { + logger.Fatalf("Error loading LN config: %v", err) + } + lndClient, err := lnd.InitLNClient(lnCfg, logger, startupCtx) + if err != nil { e.Logger.Fatalf("Error initializing the LND connection: %v", err) } @@ -67,7 +66,12 @@ func main() { InvoicePubSub: service.NewPubsub(), } - err = svc.CheckAllPendingOutgoingPayments(startupCtx) + pendingPayments, err := svc.GetAllPendingPayments(startupCtx) + if err != nil { + return + } + + err = svc.CheckPendingOutgoingPayments(startupCtx, pendingPayments) if err != nil { sentry.CaptureException(err) svc.Logger.Error(err)