Merge branch 'main' into feature/reconciliation-invoices

# Conflicts:
#	cmd/payment-reconciliation/main.go

.github/workflows/integration_tests.yaml

@@ -43,6 +43,4 @@ jobs:
           - name: Run tests
             run:  go test -p 1 -v -covermode=atomic -coverprofile=coverage.out -cover -coverpkg=./... ./...
             env:
-              RABBITMQ_URI: amqp://root:password@localhost:5672
-          - name: Upload coverage reports to Codecov
-            uses: codecov/codecov-action@v3
+              RABBITMQ_URI: amqp://root:password@localhost:5672

.github/workflows/workflow.yaml

@@ -26,12 +26,12 @@ jobs:
         repository: getalby/alby-deployment
         path: infrastructure
         token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-    # Always update dev environment
+    # Always update dev environment.
     - name: Update dev environment
       if: ${{ github.ref == 'refs/heads/main' }}
       uses: fjogeleit/[email protected]
       with:
-        valueFile: 'alby-simnet-deployment/values.yaml'
+        valueFile: 'alby-simnet-deployment/values/lndhub.yaml'
         propertyPath: 'lndhub.image.tag'
         value: ${{ steps.build.outputs.tags }}
         repository: getalby/alby-deployment

Dockerfile

@@ -12,12 +12,18 @@ RUN go mod download
 COPY . .
 
 # Build the application
-RUN go build -o main
+RUN go build -o main ./cmd/server
+
+# Build the utility scripts
+RUN go build ./cmd/invoice-republishing
+RUN go build ./cmd/payment-reconciliation
 
 # Start a new, final image to reduce size.
 FROM alpine as final
 
 # Copy the binaries and entrypoint from the builder image.
 COPY --from=builder /build/main /bin/
+COPY --from=builder /build/invoice-republishing /bin/
+COPY --from=builder /build/payment-reconciliation /bin/
 
 ENTRYPOINT [ "/bin/main" ]

Makefile

@@ -2,4 +2,7 @@
 	cp .env_example .env
 
 build:
-	CGO_ENABLED=0 go build -o lndhub
+	CGO_ENABLED=0 go build -o lndhub ./cmd/server
+
+test:
+	go test -p 1 -v -covermode=atomic -coverprofile=coverage.out -cover -coverpkg=./... ./...

README.md

@@ -54,6 +54,10 @@ vim .env # edit your config
 + `MAX_RECEIVE_AMOUNT`: (default: 0 = no limit) Set maximum amount (in satoshi) for which an invoice can be created
 + `MAX_SEND_AMOUNT`: (default: 0 = no limit) Set maximum amount (in satoshi) of an invoice that can be paid
 + `MAX_ACCOUNT_BALANCE`: (default: 0 = no limit) Set maximum balance (in satoshi) for each account
++ `MAX_SEND_VOLUME`: (default: 0 = no limit) Set maximum volume (in satoshi) for sending for each account
++ `MAX_RECEIVE_VOLUME`: (default: 0 = no limit) Set maximum volume (in satoshi) for receiving for each account
++ `SERVICE_FEE`: (default: 0 = no service fee) Set the service fee for each outgoing transaction in 1/1000 (e.g. 1 means a fee of 1sat for 1000sats - rounded up to the next bigger integer)
++ `NO_SERVICE_FEE_UP_TO_AMOUNT` (default: 0 = no free transactions) the amount in sats up to which no service fee should be charged
 
 ### Macaroon
 
@@ -79,9 +83,9 @@ lncli bakemacaroon --save_to=lndhub.macaroon info:read invoices:read invoices:wr
 ```
 
 ## Developing
-
+To run the server
 ```shell
-go run .
+go run cmd/server/main.go
 ```
 
 ### Building

cmd/invoice-republishing/main.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/getAlby/lndhub.go/db"
+	"github.com/getAlby/lndhub.go/db/models"
+	"github.com/getAlby/lndhub.go/lib"
+	"github.com/getAlby/lndhub.go/lib/service"
+	"github.com/getAlby/lndhub.go/rabbitmq"
+	"github.com/joho/godotenv"
+	"github.com/kelseyhightower/envconfig"
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+
+	c := &service.Config{}
+	// Load configruation from environment variables
+	err := godotenv.Load(".env")
+	if err != nil {
+		fmt.Println("Failed to load .env file")
+	}
+	logger := lib.Logger(c.LogFilePath)
+	startDate, endDate, err := loadStartAndEndIdFromEnv()
+	if err != nil {
+		logger.Fatalf("Could not load start and end id from env %v", err)
+	}
+	err = envconfig.Process("", c)
+	if err != nil {
+		logger.Fatalf("Error loading environment variables: %v", err)
+	}
+	// Open a DB connection based on the configured DATABASE_URI
+	dbConn, err := db.Open(c)
+	if err != nil {
+		logger.Fatalf("Error initializing db connection: %v", err)
+	}
+	amqpClient, err := rabbitmq.DialAMQP(c.RabbitMQUri, rabbitmq.WithAmqpLogger(logger))
+	if err != nil {
+		logger.Fatal(err)
+	}
+
+	defer amqpClient.Close()
+
+	rabbitmqClient, err := rabbitmq.NewClient(amqpClient,
+		rabbitmq.WithLogger(logger),
+		rabbitmq.WithLndInvoiceExchange(c.RabbitMQLndInvoiceExchange),
+		rabbitmq.WithLndHubInvoiceExchange(c.RabbitMQLndhubInvoiceExchange),
+		rabbitmq.WithLndInvoiceConsumerQueueName(c.RabbitMQInvoiceConsumerQueueName),
+		rabbitmq.WithLndPaymentExchange(c.RabbitMQLndPaymentExchange),
+		rabbitmq.WithLndPaymentConsumerQueueName(c.RabbitMQPaymentConsumerQueueName),
+	)
+	if err != nil {
+		logger.Fatal(err)
+	}
+	//small hack to get the script to work decently
+	defaultClient := rabbitmqClient.(*rabbitmq.DefaultClient)
+
+	// close the connection gently at the end of the runtime
+	defer rabbitmqClient.Close()
+
+	result := []models.Invoice{}
+	err = dbConn.NewSelect().Model(&result).Where("settled_at > ?", startDate).Where("settled_at < ?", endDate).Scan(context.Background())
+	if err != nil {
+		logger.Fatal(err)
+	}
+	logrus.Infof("Found %d invoices", len(result))
+	svc := &service.LndhubService{
+		Config:         c,
+		DB:             dbConn,
+		Logger:         logger,
+		RabbitMQClient: rabbitmqClient,
+		InvoicePubSub:  service.NewPubsub(),
+	}
+	ctx := context.Background()
+	dryRun := os.Getenv("DRY_RUN") == "true"
+	errCount := 0
+	for _, inv := range result {
+		logger.Infof("Publishing invoice with hash %s", inv.RHash)
+		if dryRun {
+			continue
+		}
+		err = defaultClient.PublishToLndhubExchange(ctx, inv, svc.EncodeInvoiceWithUserLogin)
+		if err != nil {
+			logrus.WithError(err).Error("errror publishing to lndhub exchange")
+		}
+	}
+	logger.Infof("Published %d invoices, # errors %d", len(result), errCount)
+
+}
+
+func loadStartAndEndIdFromEnv() (start, end time.Time, err error) {
+	start, err = time.Parse(time.RFC3339, os.Getenv("START_DATE"))
+	if err != nil {
+		return
+	}
+	end, err = time.Parse(time.RFC3339, os.Getenv("END_DATE"))
+	return
+}

cmd/payment-reconciliation/main.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/getAlby/lndhub.go/db"
+	"github.com/getAlby/lndhub.go/lib"
+	"github.com/getAlby/lndhub.go/lib/service"
+	"github.com/getAlby/lndhub.go/lnd"
+	"github.com/getsentry/sentry-go"
+	"github.com/joho/godotenv"
+	"github.com/kelseyhightower/envconfig"
+	"github.com/labstack/echo/v4"
+)
+
+// script to reconcile pending payments between the backup node and the database
+// normally, this reconciliation should happen through rabbitmq but there are
+// cases where it doesn't happen and in that case this script can be run as a a
+// cron job as a redundant reconcilation mechanism.
+func main() {
+
+	c := &service.Config{}
+
+	// Load configruation from environment variables
+	err := godotenv.Load(".env")
+	if err != nil {
+		fmt.Println("Failed to load .env file")
+	}
+	err = envconfig.Process("", c)
+	if err != nil {
+		log.Fatalf("Error loading environment variables: %v", err)
+	}
+
+	// Setup logging to STDOUT or a configrued log file
+	logger := lib.Logger(c.LogFilePath)
+
+	// Open a DB connection based on the configured DATABASE_URI
+	dbConn, err := db.Open(c)
+	if err != nil {
+		logger.Fatalf("Error initializing db connection: %v", err)
+	}
+
+	// Migrate the DB
+	//Todo: use timeout for startupcontext
+	startupCtx := context.Background()
+
+	// New Echo app
+	e := echo.New()
+
+	// Init new LND client
+	lnCfg, err := lnd.LoadConfig()
+	if err != nil {
+		logger.Fatalf("Failed to load lnd config %v", err)
+	}
+	lndClient, err := lnd.NewLNDclient(lnd.LNDoptions{
+		Address:      lnCfg.LNDAddress,
+		MacaroonFile: lnCfg.LNDMacaroonFile,
+		MacaroonHex:  lnCfg.LNDMacaroonHex,
+		CertFile:     lnCfg.LNDCertFile,
+		CertHex:      lnCfg.LNDCertHex,
+	}, startupCtx)
+	if err != nil {
+		e.Logger.Fatalf("Error initializing the LND connection: %v", err)
+	}
+	logger.Infof("Connected to LND: %s ", lndClient.GetMainPubkey())
+
+	svc := &service.LndhubService{
+		Config:        c,
+		DB:            dbConn,
+		LndClient:     lndClient,
+		Logger:        logger,
+		InvoicePubSub: service.NewPubsub(),
+	}
+
+	//for this job, we only search for payments older than a day to avoid current in-flight payments
+	ts := time.Now().Add(-1 * 24 * time.Hour)
+	pending, err := svc.GetPendingPaymentsUntil(startupCtx, ts)
+	if err != nil {
+		sentry.CaptureException(err)
+		svc.Logger.Fatal(err)
+	}
+	svc.Logger.Infof("Found %d pending payments", len(pending))
+	startupCtx, cancel := context.WithTimeout(startupCtx, 2*time.Minute)
+	defer cancel()
+	err = svc.CheckPendingOutgoingPayments(startupCtx, pending)
+	if err != nil {
+		sentry.CaptureException(err)
+		svc.Logger.Fatal(err)
+	}
+}