feat(failover): return 503 to batcher when eigenda is down

chore: go mod tidy to generate go.mod

go.mod

@@ -5,7 +5,10 @@ go 1.22
 toolchain go1.22.0
 
 require (
-	github.com/Layr-Labs/eigenda v0.8.5-0.20241028201743-5fe3e910a22d
+	// this is the latest commit of https://github.com/Layr-Labs/eigenda/pull/828
+	// TODO: merge upstream first and the use the latest commit of the main branch
+	github.com/Layr-Labs/eigenda v0.8.5-0.20241025144504-69ff616c1e5b
+	github.com/avast/retry-go/v4 v4.6.0
 	github.com/consensys/gnark-crypto v0.12.1
 	github.com/ethereum-optimism/optimism v1.9.4-0.20240927020138-a9c7f349d10b
 	github.com/ethereum/go-ethereum v1.14.11
@@ -39,7 +42,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.31.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/kms v1.31.0 // indirect

go.sum

@@ -22,8 +22,8 @@ github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE=
 github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
-github.com/Layr-Labs/eigenda v0.8.5-0.20241028201743-5fe3e910a22d h1:s5U1TaWhC1J2rwc9IQdU/COGvuXALCKMd9GbONUZMxc=
-github.com/Layr-Labs/eigenda v0.8.5-0.20241028201743-5fe3e910a22d/go.mod h1:sqUNf9Ak+EfAX82jDxrb4QbT/g3DViWD3b7YIk36skk=
+github.com/Layr-Labs/eigenda v0.8.5-0.20241025144504-69ff616c1e5b h1:wQQhuxeFIK069pKIPcjNWI+ak/wuxlBybt/n787YUBQ=
+github.com/Layr-Labs/eigenda v0.8.5-0.20241025144504-69ff616c1e5b/go.mod h1:sqUNf9Ak+EfAX82jDxrb4QbT/g3DViWD3b7YIk36skk=
 github.com/Layr-Labs/eigensdk-go v0.1.7-0.20240507215523-7e4891d5099a h1:L/UsJFw9M31FD/WgXTPFB0oxbq9Cu4Urea1xWPMQS7Y=
 github.com/Layr-Labs/eigensdk-go v0.1.7-0.20240507215523-7e4891d5099a/go.mod h1:OF9lmS/57MKxS0xpSpX0qHZl0SKkDRpvJIvsGvMN1y8=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
@@ -45,6 +45,8 @@ github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer5
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
+github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA=
+github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE=
 github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
 github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=

server/errors.go

@@ -1,9 +1,12 @@
 package server
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/Layr-Labs/eigenda-proxy/commitments"
+	"github.com/Layr-Labs/eigenda-proxy/store"
+	"github.com/Layr-Labs/eigenda/api"
 )
 
 // MetaError includes both an error and commitment metadata
@@ -22,3 +25,10 @@ func (me MetaError) Error() string {
 func (me MetaError) Unwrap() error {
 	return me.Err
 }
+
+func is400(err error) bool {
+	var errAPI api.ErrorAPI
+	// TODO: the ErrProxyOversizedBlob
+	return errors.Is(err, store.ErrProxyOversizedBlob) ||
+		errors.As(err, &errAPI) && errAPI.ErrorFault() == api.ErrorFaultClient
+}

server/handlers.go

@@ -9,7 +9,7 @@ import (
 	"net/http"
 
 	"github.com/Layr-Labs/eigenda-proxy/commitments"
-	"github.com/Layr-Labs/eigenda-proxy/store"
+	"github.com/Layr-Labs/eigenda/api"
 	"github.com/gorilla/mux"
 )
 
@@ -180,11 +180,14 @@ func (svr *Server) handlePostShared(w http.ResponseWriter, r *http.Request, comm
 			Err:  fmt.Errorf("put request failed with commitment %v (commitment mode %v): %w", comm, meta.Mode, err),
 			Meta: meta,
 		}
-		if errors.Is(err, store.ErrEigenDAOversizedBlob) || errors.Is(err, store.ErrProxyOversizedBlob) {
-			// we add here any error that should be returned as a 400 instead of a 500.
-			// currently only includes oversized blob requests
+		var errAPI api.ErrorAPI
+		switch {
+		case is400(err):
 			http.Error(w, err.Error(), http.StatusBadRequest)
-		} else {
+		case errors.As(err, &errAPI) && errAPI.ErrorCode() == http.StatusServiceUnavailable:
+			// this tells the caller (batcher) to failover to ethda b/c eigenda is temporarily down
+			http.Error(w, err.Error(), http.StatusServiceUnavailable)
+		default:
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 		}
 		return err

store/generated_key/eigenda/eigenda.go

@@ -2,12 +2,17 @@ package eigenda
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
 	"github.com/Layr-Labs/eigenda-proxy/store"
 	"github.com/Layr-Labs/eigenda-proxy/verify"
+	"github.com/Layr-Labs/eigenda/api"
 	"github.com/Layr-Labs/eigenda/api/clients"
+	"github.com/Layr-Labs/eigenda/api/grpc/disperser"
+
+	"github.com/avast/retry-go/v4"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 )
@@ -70,7 +75,27 @@ func (e Store) Put(ctx context.Context, value []byte) ([]byte, error) {
 		return nil, fmt.Errorf("%w: blob length %d, max blob size %d", store.ErrProxyOversizedBlob, len(value), e.cfg.MaxBlobSizeBytes)
 	}
 
-	blobInfo, err := e.client.PutBlob(ctx, value)
+	// We attempt to disperse the blob to EigenDA up to 3 times, unless we get a 400 error on any attempt.
+	blobInfo, err := retry.DoWithData(
+		func() (*disperser.BlobInfo, error) {
+			return e.client.PutBlob(ctx, value)
+		},
+		retry.RetryIf(func(err error) bool {
+			var errAPI api.ErrorAPI
+			if errors.As(err, &errAPI) {
+				if errAPI.ErrorFault() == api.ErrorFaultClient {
+					// we don't retry 400 errors because there is no point,
+					// we are passing invalid data
+					return false
+				}
+			}
+			return true
+		}),
+		// only return the last error. If it is a 503, then the handler will convert it to an http 503
+		// to signify to the client (batcher) to failover to ethda b/c eigenda is temporarily down.
+		retry.LastErrorOnly(true),
+		retry.Attempts(3),
+	)
 	if err != nil {
 		// TODO: we will want to filter for errors here and return a 503 when needed
 		// ie when dispersal itself failed, or that we timed out waiting for batch to land onchain

store/store.go

@@ -18,8 +18,7 @@ const (
 )
 
 var (
-	ErrProxyOversizedBlob   = fmt.Errorf("encoded blob is larger than max blob size")
-	ErrEigenDAOversizedBlob = fmt.Errorf("blob size cannot exceed")
+	ErrProxyOversizedBlob = fmt.Errorf("encoded blob is larger than max blob size")
 )
 
 func (b BackendType) String() string {