From 4625e9668a29bbaf4170c95e958d0d5e823b2018 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:53:03 -0500 Subject: [PATCH 01/18] Write GET tests Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/mock/BlobCodec.go | 68 ++++++ api/clients/eigenda_client_v2.go | 142 +++++++++++++ api/clients/eigenda_client_v2_test.go | 294 ++++++++++++++++++++++++++ api/clients/mock/relay_client.go | 2 +- common/testutils/test_utils.go | 5 + 5 files changed, 510 insertions(+), 1 deletion(-) create mode 100644 api/clients/codecs/mock/BlobCodec.go create mode 100644 api/clients/eigenda_client_v2.go create mode 100644 api/clients/eigenda_client_v2_test.go diff --git a/api/clients/codecs/mock/BlobCodec.go b/api/clients/codecs/mock/BlobCodec.go new file mode 100644 index 000000000..973669bff --- /dev/null +++ b/api/clients/codecs/mock/BlobCodec.go @@ -0,0 +1,68 @@ +package mock + +import mock "github.com/stretchr/testify/mock" + +// BlobCodec is an autogenerated mock type for the BlobCodec type +type BlobCodec struct { + mock.Mock +} + +// DecodeBlob provides a mock function with given fields: encodedData +func (_m *BlobCodec) DecodeBlob(encodedData []byte) ([]byte, error) { + ret := _m.Called(encodedData) + + if len(ret) == 0 { + panic("no return value specified for DecodeBlob") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func([]byte) ([]byte, error)); ok { + return rf(encodedData) + } + if rf, ok := ret.Get(0).(func([]byte) []byte); ok { + r0 = rf(encodedData) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func([]byte) error); ok { + r1 = rf(encodedData) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// EncodeBlob provides a mock function with given fields: rawData +func (_m *BlobCodec) EncodeBlob(rawData []byte) ([]byte, error) { + ret := _m.Called(rawData) + + if len(ret) == 0 { + panic("no return value specified for EncodeBlob") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func([]byte) ([]byte, error)); ok { + return rf(rawData) + } + if rf, ok := ret.Get(0).(func([]byte) []byte); ok { + r0 = rf(rawData) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func([]byte) error); ok { + r1 = rf(rawData) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/api/clients/eigenda_client_v2.go b/api/clients/eigenda_client_v2.go new file mode 100644 index 000000000..2d92f1654 --- /dev/null +++ b/api/clients/eigenda_client_v2.go @@ -0,0 +1,142 @@ +package clients + +import ( + "context" + "fmt" + "github.com/Layr-Labs/eigenda/api/clients/codecs" + corev2 "github.com/Layr-Labs/eigenda/core/v2" + "github.com/Layr-Labs/eigensdk-go/logging" + "github.com/hashicorp/go-multierror" + "math/rand" +) + +// EigenDAClientV2 provides the ability to get blobs from the relay subsystem, and to send new blobs to the disperser. +type EigenDAClientV2 interface { + // GetBlob iteratively attempts to retrieve a given blob with key blobKey from the relays listed in the blobCertificate. + GetBlob(ctx context.Context, blobKey corev2.BlobKey, blobCertificate corev2.BlobCertificate) ([]byte, error) + GetCodec() codecs.BlobCodec + Close() error +} + +type eigenDAClientV2 struct { + clientConfig EigenDAClientConfig + log logging.Logger + relayClient RelayClient + codec codecs.BlobCodec +} + +var _ EigenDAClientV2 = &eigenDAClientV2{} + +// BuildEigenDAClientV2 builds an EigenDAClientV2 from config structs. +func BuildEigenDAClientV2( + log logging.Logger, + clientConfig EigenDAClientConfig, + relayClientConfig RelayClientConfig) (EigenDAClientV2, error) { + + err := clientConfig.CheckAndSetDefaults() + if err != nil { + return nil, err + } + + relayClient, err := NewRelayClient(&relayClientConfig, log) + + lowLevelCodec, err := codecs.BlobEncodingVersionToCodec(clientConfig.PutBlobEncodingVersion) + if err != nil { + return nil, fmt.Errorf("create low level codec: %w", err) + } + var codec codecs.BlobCodec + if clientConfig.DisablePointVerificationMode { + codec = codecs.NewNoIFFTCodec(lowLevelCodec) + } else { + codec = codecs.NewIFFTCodec(lowLevelCodec) + } + + return NewEigenDAClientV2(log, clientConfig, relayClient, codec) +} + +// NewEigenDAClientV2 assembles an EigenDAClientV2 from subcomponents that have already been constructed and initialized. +func NewEigenDAClientV2( + log logging.Logger, + clientConfig EigenDAClientConfig, + relayClient RelayClient, + codec codecs.BlobCodec) (EigenDAClientV2, error) { + + return &eigenDAClientV2{ + clientConfig: clientConfig, + log: log, + relayClient: relayClient, + codec: codec, + }, nil +} + +// GetBlob iteratively attempts to retrieve a given blob with key blobKey from the relays listed in the blobCertificate. +// +// The relays are attempted in random order. +// +// The returned blob is decoded. +func (c *eigenDAClientV2) GetBlob(ctx context.Context, blobKey corev2.BlobKey, blobCertificate corev2.BlobCertificate) ([]byte, error) { + // create a randomized array of indices, so that it isn't always the first relay in the list which gets hit + random := rand.New(rand.NewSource(rand.Int63())) + relayKeyCount := len(blobCertificate.RelayKeys) + var indices []int + for i := 0; i < relayKeyCount; i++ { + indices = append(indices, i) + } + random.Shuffle(len(indices), func(i int, j int) { + indices[i], indices[j] = indices[j], indices[i] + }) + + // TODO (litt3): consider creating a utility which can deprioritize relays that fail to respond (or respond maliciously) + + // iterate over relays in random order, until we are able to get the blob from someone + for _, val := range indices { + relayKey := blobCertificate.RelayKeys[val] + + data, err := c.relayClient.GetBlob(ctx, relayKey, blobKey) + + // if GetBlob returned an error, try calling a different relay + if err != nil { + // TODO: should this log type be downgraded to debug to avoid log spam? I'm not sure how frequent retrieval + // from a relay will fail in practice (?) + c.log.Info("blob couldn't be retrieved from relay", "blobKey", blobKey, "relayKey", relayKey) + continue + } + + // An honest relay should never send an empty blob + if len(data) == 0 { + c.log.Warn("blob received from relay had length 0", "blobKey", blobKey, "relayKey", relayKey) + continue + } + + // An honest relay should never send a blob which cannot be decoded + decodedData, err := c.codec.DecodeBlob(data) + if err != nil { + c.log.Warn("error decoding blob", "blobKey", blobKey, "relayKey", relayKey) + continue + } + + return decodedData, nil + } + + return nil, fmt.Errorf("unable to retrieve blob from any relay") +} + +func (c *eigenDAClientV2) GetCodec() codecs.BlobCodec { + return c.codec +} + +func (c *eigenDAClientV2) Close() error { + var errList *multierror.Error + + // TODO: this is using a multierror, since there will be more subcomponents requiring closing after adding PUT functionality + relayClientErr := c.relayClient.Close() + if relayClientErr != nil { + errList = multierror.Append(errList, relayClientErr) + } + + if errList != nil { + return errList.ErrorOrNil() + } + + return nil +} diff --git a/api/clients/eigenda_client_v2_test.go b/api/clients/eigenda_client_v2_test.go new file mode 100644 index 000000000..ae0c1c432 --- /dev/null +++ b/api/clients/eigenda_client_v2_test.go @@ -0,0 +1,294 @@ +package clients_test + +import ( + "context" + "fmt" + "github.com/Layr-Labs/eigenda/api/clients" + "github.com/Layr-Labs/eigenda/api/clients/codecs" + codecsmock "github.com/Layr-Labs/eigenda/api/clients/codecs/mock" + clientsmock "github.com/Layr-Labs/eigenda/api/clients/mock" + tu "github.com/Layr-Labs/eigenda/common/testutils" + v2 "github.com/Layr-Labs/eigenda/core/v2" + "github.com/Layr-Labs/eigensdk-go/logging" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "golang.org/x/exp/rand" + "testing" + "time" +) + +type ClientV2Tester struct { + ClientV2 clients.EigenDAClientV2 + MockRelayClient *clientsmock.MockRelayClient + MockCodec *codecsmock.BlobCodec +} + +func (c *ClientV2Tester) assertExpectations(t *testing.T) { + c.MockRelayClient.AssertExpectations(t) + c.MockCodec.AssertExpectations(t) +} + +// buildClientV2Tester sets up a V2 client, with mocks necessary for testing +func buildClientV2Tester(t *testing.T) ClientV2Tester { + tu.InitializeRandom() + logger := logging.NewNoopLogger() + clientConfig := clients.EigenDAClientConfig{} + + mockRelayClient := clientsmock.MockRelayClient{} + mockCodec := codecsmock.BlobCodec{} + + client, err := clients.NewEigenDAClientV2( + logger, + clientConfig, + &mockRelayClient, + &mockCodec) + + assert.NotNil(t, client) + assert.Nil(t, err) + + return ClientV2Tester{ + ClientV2: client, + MockRelayClient: &mockRelayClient, + MockCodec: &mockCodec, + } +} + +// TestGetBlobSuccess tests that a blob is received without error in the happy case +func TestGetBlobSuccess(t *testing.T) { + tester := buildClientV2Tester(t) + + blobKey := v2.BlobKey(tu.RandomBytes(32)) + blobBytes := tu.RandomBytes(100) + + relayKeys := make([]v2.RelayKey, 1) + relayKeys[0] = tu.RandomUint16() + blobCert := v2.BlobCertificate{ + RelayKeys: relayKeys, + } + + tester.MockRelayClient.On("GetBlob", mock.Anything, relayKeys[0], blobKey).Return(blobBytes, nil).Once() + tester.MockCodec.On("DecodeBlob", blobBytes).Return(tu.RandomBytes(50), nil).Once() + + blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + + assert.NotNil(t, blob) + assert.Nil(t, err) + + tester.assertExpectations(t) +} + +// TestRandomRelayRetries verifies correct behavior when some relays from the certificate do not respond with the blob, +// requiring the client to retry with other relays. +func TestRandomRelayRetries(t *testing.T) { + tester := buildClientV2Tester(t) + + blobKey := v2.BlobKey(tu.RandomBytes(32)) + blobBytes := tu.RandomBytes(100) + + relayCount := 100 + relayKeys := make([]v2.RelayKey, relayCount) + for i := 0; i < relayCount; i++ { + relayKeys[i] = tu.RandomUint16() + } + blobCert := v2.BlobCertificate{ + RelayKeys: relayKeys, + } + + // for this test, only a single relay is online + // we will be asserting that it takes a different amount of retries to dial this relay, since the array of relay keys to try is randomized + onlineRelayKey := relayKeys[rand.Intn(len(relayKeys))] + + offlineKeyMatcher := func(relayKey v2.RelayKey) bool { return relayKey != onlineRelayKey } + onlineKeyMatcher := func(relayKey v2.RelayKey) bool { return relayKey == onlineRelayKey } + var failedCallCount int + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.MatchedBy(offlineKeyMatcher), blobKey).Return(nil, fmt.Errorf("offline relay")).Run(func(args mock.Arguments) { + failedCallCount++ + }) + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.MatchedBy(onlineKeyMatcher), blobKey).Return(blobBytes, nil) + tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tu.RandomBytes(50), nil) + + // keep track of how many tries various blob retrievals require + // this allows us to assert that there is variability, i.e. that relay call order is actually random + requiredTries := map[int]bool{} + + for i := 0; i < relayCount; i++ { + failedCallCount = 0 + blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + assert.NotNil(t, blob) + assert.Nil(t, err) + + requiredTries[failedCallCount] = true + } + + // with 100 random tries, with possible values between 1 and 100, we can very confidently assert that there are at least 10 unique values + assert.Greater(t, len(requiredTries), 10) + + tester.assertExpectations(t) +} + +// TestNoRelayResponse tests functionality when none of the relays listed in the blob certificate respond +func TestNoRelayResponse(t *testing.T) { + tester := buildClientV2Tester(t) + + blobKey := v2.BlobKey(tu.RandomBytes(32)) + + relayCount := 10 + relayKeys := make([]v2.RelayKey, relayCount) + for i := 0; i < relayCount; i++ { + relayKeys[i] = tu.RandomUint16() + } + blobCert := v2.BlobCertificate{ + RelayKeys: relayKeys, + } + + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(nil, fmt.Errorf("offline relay")) + + blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + assert.Nil(t, blob) + assert.NotNil(t, err) + + tester.assertExpectations(t) +} + +// TestNoRelaysInCert tests that having no relay keys in the cert is handled gracefully +func TestNoRelaysInCert(t *testing.T) { + tester := buildClientV2Tester(t) + + blobKey := v2.BlobKey(tu.RandomBytes(32)) + + // cert has no listed relay keys + blobCert := v2.BlobCertificate{ + RelayKeys: []v2.RelayKey{}, + } + + blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + assert.Nil(t, blob) + assert.NotNil(t, err) + + tester.assertExpectations(t) +} + +// TestGetBlobReturns0Len verifies that a 0 length blob returned from a relay is handled gracefully, and that the client retries after such a failure +func TestGetBlobReturns0Len(t *testing.T) { + tester := buildClientV2Tester(t) + + blobKey := v2.BlobKey(tu.RandomBytes(32)) + + relayCount := 10 + relayKeys := make([]v2.RelayKey, relayCount) + for i := 0; i < relayCount; i++ { + relayKeys[i] = tu.RandomUint16() + } + blobCert := v2.BlobCertificate{ + RelayKeys: relayKeys, + } + + // the first GetBlob will return a 0 len blob + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return([]byte{}, nil).Once() + // the second call will return random bytes + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tu.RandomBytes(100), nil).Once() + + tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tu.RandomBytes(50), nil) + + // the call to the first relay will fail with a 0 len blob returned. the call to the second relay will succeed + blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + assert.NotNil(t, blob) + assert.Nil(t, err) + + tester.assertExpectations(t) +} + +// TestFailedDecoding verifies that a failed blob decode is handled gracefully, and that the client retries after such a failure +func TestFailedDecoding(t *testing.T) { + tester := buildClientV2Tester(t) + + blobKey := v2.BlobKey(tu.RandomBytes(32)) + + relayCount := 10 + relayKeys := make([]v2.RelayKey, relayCount) + for i := 0; i < relayCount; i++ { + relayKeys[i] = tu.RandomUint16() + } + blobCert := v2.BlobCertificate{ + RelayKeys: relayKeys, + } + + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tu.RandomBytes(100), nil) + + tester.MockCodec.On("DecodeBlob", mock.Anything).Return(nil, fmt.Errorf("decode failed")).Once() + tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tu.RandomBytes(50), nil).Once() + + // decoding will fail the first time, but succeed the second time + blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + assert.NotNil(t, blob) + assert.Nil(t, err) + + tester.assertExpectations(t) +} + +// TestErrorFreeClose tests the happy case, where none of the internal closes yield an error +func TestErrorFreeClose(t *testing.T) { + tester := buildClientV2Tester(t) + + tester.MockRelayClient.On("Close").Return(nil).Once() + + err := tester.ClientV2.Close() + assert.Nil(t, err) + + tester.assertExpectations(t) +} + +// TestErrorClose tests what happens when subcomponents throw errors when being closed +func TestErrorClose(t *testing.T) { + tester := buildClientV2Tester(t) + + tester.MockRelayClient.On("Close").Return(fmt.Errorf("close failed")).Once() + + err := tester.ClientV2.Close() + assert.NotNil(t, err) + + tester.assertExpectations(t) +} + +// TestGetCodec checks that the codec used in construction is returned by GetCodec +func TestGetCodec(t *testing.T) { + tester := buildClientV2Tester(t) + + assert.Equal(t, tester.MockCodec, tester.ClientV2.GetCodec()) + + tester.assertExpectations(t) +} + +// TestBuilder tests that the method that builds the client from config doesn't throw any obvious errors +func TestBuilder(t *testing.T) { + clientConfig := clients.EigenDAClientConfig{ + StatusQueryTimeout: 10 * time.Minute, + StatusQueryRetryInterval: 50 * time.Millisecond, + ResponseTimeout: 10 * time.Second, + ConfirmationTimeout: 5 * time.Second, + CustomQuorumIDs: []uint{}, + SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", + DisableTLS: false, + PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + DisablePointVerificationMode: false, + WaitForFinalization: true, + RPC: "http://localhost:8080", + EthRpcUrl: "http://localhost:8545", + SvcManagerAddr: "0x1234567890123456789012345678901234567890", + } + + relayClientConfig := clients.RelayClientConfig{ + Sockets: make(map[v2.RelayKey]string), + UseSecureGrpcFlag: true, + } + + clientV2, err := clients.BuildEigenDAClientV2( + logging.NewNoopLogger(), + clientConfig, + relayClientConfig) + + assert.NotNil(t, clientV2) + assert.Nil(t, err) + + assert.NotNil(t, clientV2.GetCodec()) +} diff --git a/api/clients/mock/relay_client.go b/api/clients/mock/relay_client.go index e97e1e540..63d5750f4 100644 --- a/api/clients/mock/relay_client.go +++ b/api/clients/mock/relay_client.go @@ -19,7 +19,7 @@ func NewRelayClient() *MockRelayClient { } func (c *MockRelayClient) GetBlob(ctx context.Context, relayKey corev2.RelayKey, blobKey corev2.BlobKey) ([]byte, error) { - args := c.Called(blobKey) + args := c.Called(ctx, relayKey, blobKey) if args.Get(0) == nil { return nil, args.Error(1) } diff --git a/common/testutils/test_utils.go b/common/testutils/test_utils.go index e08180b65..586a1db64 100644 --- a/common/testutils/test_utils.go +++ b/common/testutils/test_utils.go @@ -108,3 +108,8 @@ func RandomString(length int) string { } return string(b) } + +// RandomUint16 generates a random uint16 +func RandomUint16() uint16 { + return uint16(rand.Uint32()) +} From 885c131e4fd62a962affb8a06952523214c14f03 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:13:11 -0500 Subject: [PATCH 02/18] Respond to PR comments Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/eigenda_client_v2.go | 124 ++++++++++++++------------ api/clients/eigenda_client_v2_test.go | 24 ++--- common/testutils/test_utils.go | 5 -- 3 files changed, 81 insertions(+), 72 deletions(-) diff --git a/api/clients/eigenda_client_v2.go b/api/clients/eigenda_client_v2.go index 2d92f1654..e754be65f 100644 --- a/api/clients/eigenda_client_v2.go +++ b/api/clients/eigenda_client_v2.go @@ -6,66 +6,58 @@ import ( "github.com/Layr-Labs/eigenda/api/clients/codecs" corev2 "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigensdk-go/logging" - "github.com/hashicorp/go-multierror" + "github.com/cockroachdb/errors/join" "math/rand" ) // EigenDAClientV2 provides the ability to get blobs from the relay subsystem, and to send new blobs to the disperser. -type EigenDAClientV2 interface { - // GetBlob iteratively attempts to retrieve a given blob with key blobKey from the relays listed in the blobCertificate. - GetBlob(ctx context.Context, blobKey corev2.BlobKey, blobCertificate corev2.BlobCertificate) ([]byte, error) - GetCodec() codecs.BlobCodec - Close() error +type EigenDAClientV2 struct { + log logging.Logger + // doesn't need to be cryptographically secure, as it's only used to distribute load across relays + random *rand.Rand + config *EigenDAClientConfig + codec codecs.BlobCodec + relayClient RelayClient } -type eigenDAClientV2 struct { - clientConfig EigenDAClientConfig - log logging.Logger - relayClient RelayClient - codec codecs.BlobCodec -} - -var _ EigenDAClientV2 = &eigenDAClientV2{} - // BuildEigenDAClientV2 builds an EigenDAClientV2 from config structs. func BuildEigenDAClientV2( log logging.Logger, - clientConfig EigenDAClientConfig, - relayClientConfig RelayClientConfig) (EigenDAClientV2, error) { + config *EigenDAClientConfig, + relayClientConfig *RelayClientConfig) (*EigenDAClientV2, error) { - err := clientConfig.CheckAndSetDefaults() + err := config.CheckAndSetDefaults() if err != nil { - return nil, err + return nil, fmt.Errorf("check and set client config defaults: %w", err) } - relayClient, err := NewRelayClient(&relayClientConfig, log) - - lowLevelCodec, err := codecs.BlobEncodingVersionToCodec(clientConfig.PutBlobEncodingVersion) + relayClient, err := NewRelayClient(relayClientConfig, log) if err != nil { - return nil, fmt.Errorf("create low level codec: %w", err) + return nil, fmt.Errorf("new relay client: %w", err) } - var codec codecs.BlobCodec - if clientConfig.DisablePointVerificationMode { - codec = codecs.NewNoIFFTCodec(lowLevelCodec) - } else { - codec = codecs.NewIFFTCodec(lowLevelCodec) + + codec, err := createCodec(config) + if err != nil { + return nil, err } - return NewEigenDAClientV2(log, clientConfig, relayClient, codec) + return NewEigenDAClientV2(log, rand.New(rand.NewSource(rand.Int63())), config, relayClient, codec) } // NewEigenDAClientV2 assembles an EigenDAClientV2 from subcomponents that have already been constructed and initialized. func NewEigenDAClientV2( log logging.Logger, - clientConfig EigenDAClientConfig, + random *rand.Rand, + config *EigenDAClientConfig, relayClient RelayClient, - codec codecs.BlobCodec) (EigenDAClientV2, error) { - - return &eigenDAClientV2{ - clientConfig: clientConfig, - log: log, - relayClient: relayClient, - codec: codec, + codec codecs.BlobCodec) (*EigenDAClientV2, error) { + + return &EigenDAClientV2{ + log: log, + random: random, + config: config, + codec: codec, + relayClient: relayClient, }, nil } @@ -74,15 +66,23 @@ func NewEigenDAClientV2( // The relays are attempted in random order. // // The returned blob is decoded. -func (c *eigenDAClientV2) GetBlob(ctx context.Context, blobKey corev2.BlobKey, blobCertificate corev2.BlobCertificate) ([]byte, error) { - // create a randomized array of indices, so that it isn't always the first relay in the list which gets hit - random := rand.New(rand.NewSource(rand.Int63())) +func (c *EigenDAClientV2) GetBlob( + ctx context.Context, + blobKey corev2.BlobKey, + blobCertificate corev2.BlobCertificate) ([]byte, error) { + relayKeyCount := len(blobCertificate.RelayKeys) + + if relayKeyCount == 0 { + return nil, fmt.Errorf("relay key count is zero") + } + var indices []int + // create a randomized array of indices, so that it isn't always the first relay in the list which gets hit for i := 0; i < relayKeyCount; i++ { indices = append(indices, i) } - random.Shuffle(len(indices), func(i int, j int) { + c.random.Shuffle(len(indices), func(i int, j int) { indices[i], indices[j] = indices[j], indices[i] }) @@ -96,22 +96,20 @@ func (c *eigenDAClientV2) GetBlob(ctx context.Context, blobKey corev2.BlobKey, b // if GetBlob returned an error, try calling a different relay if err != nil { - // TODO: should this log type be downgraded to debug to avoid log spam? I'm not sure how frequent retrieval - // from a relay will fail in practice (?) - c.log.Info("blob couldn't be retrieved from relay", "blobKey", blobKey, "relayKey", relayKey) + c.log.Warn("blob couldn't be retrieved from relay", "blobKey", blobKey, "relayKey", relayKey, "error", err) continue } // An honest relay should never send an empty blob if len(data) == 0 { - c.log.Warn("blob received from relay had length 0", "blobKey", blobKey, "relayKey", relayKey) + c.log.Warn("blob received from relay had length 0", "blobKey", blobKey, "relayKey", relayKey, "error", err) continue } // An honest relay should never send a blob which cannot be decoded decodedData, err := c.codec.DecodeBlob(data) if err != nil { - c.log.Warn("error decoding blob", "blobKey", blobKey, "relayKey", relayKey) + c.log.Warn("error decoding blob", "blobKey", blobKey, "relayKey", relayKey, "error", err) continue } @@ -121,22 +119,34 @@ func (c *eigenDAClientV2) GetBlob(ctx context.Context, blobKey corev2.BlobKey, b return nil, fmt.Errorf("unable to retrieve blob from any relay") } -func (c *eigenDAClientV2) GetCodec() codecs.BlobCodec { +// GetCodec returns the codec the client uses for encoding and decoding blobs +func (c *EigenDAClientV2) GetCodec() codecs.BlobCodec { return c.codec } -func (c *eigenDAClientV2) Close() error { - var errList *multierror.Error - - // TODO: this is using a multierror, since there will be more subcomponents requiring closing after adding PUT functionality +// Close is responsible for calling close on all internal clients. This method will do its best to close all internal +// clients, even if some closes fail. +// +// Any and all errors returned from closing internal clients will be joined and returned. +// +// This method should only be called once. +func (c *EigenDAClientV2) Close() error { relayClientErr := c.relayClient.Close() - if relayClientErr != nil { - errList = multierror.Append(errList, relayClientErr) - } - if errList != nil { - return errList.ErrorOrNil() + // TODO: this is using join, since there will be more subcomponents requiring closing after adding PUT functionality + return join.Join(relayClientErr) +} + +// createCodec creates the codec based on client config values +func createCodec(config *EigenDAClientConfig) (codecs.BlobCodec, error) { + lowLevelCodec, err := codecs.BlobEncodingVersionToCodec(config.PutBlobEncodingVersion) + if err != nil { + return nil, fmt.Errorf("create low level codec: %w", err) } - return nil + if config.DisablePointVerificationMode { + return codecs.NewNoIFFTCodec(lowLevelCodec), nil + } else { + return codecs.NewIFFTCodec(lowLevelCodec), nil + } } diff --git a/api/clients/eigenda_client_v2_test.go b/api/clients/eigenda_client_v2_test.go index ae0c1c432..eea7f2341 100644 --- a/api/clients/eigenda_client_v2_test.go +++ b/api/clients/eigenda_client_v2_test.go @@ -12,13 +12,13 @@ import ( "github.com/Layr-Labs/eigensdk-go/logging" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "golang.org/x/exp/rand" + "math/rand" "testing" "time" ) type ClientV2Tester struct { - ClientV2 clients.EigenDAClientV2 + ClientV2 *clients.EigenDAClientV2 MockRelayClient *clientsmock.MockRelayClient MockCodec *codecsmock.BlobCodec } @@ -32,13 +32,17 @@ func (c *ClientV2Tester) assertExpectations(t *testing.T) { func buildClientV2Tester(t *testing.T) ClientV2Tester { tu.InitializeRandom() logger := logging.NewNoopLogger() - clientConfig := clients.EigenDAClientConfig{} + clientConfig := &clients.EigenDAClientConfig{} mockRelayClient := clientsmock.MockRelayClient{} mockCodec := codecsmock.BlobCodec{} + // TODO (litt3): use TestRandom once the PR merges https://github.com/Layr-Labs/eigenda/pull/976 + random := rand.New(rand.NewSource(rand.Int63())) + client, err := clients.NewEigenDAClientV2( logger, + random, clientConfig, &mockRelayClient, &mockCodec) @@ -61,7 +65,7 @@ func TestGetBlobSuccess(t *testing.T) { blobBytes := tu.RandomBytes(100) relayKeys := make([]v2.RelayKey, 1) - relayKeys[0] = tu.RandomUint16() + relayKeys[0] = rand.Uint32() blobCert := v2.BlobCertificate{ RelayKeys: relayKeys, } @@ -88,7 +92,7 @@ func TestRandomRelayRetries(t *testing.T) { relayCount := 100 relayKeys := make([]v2.RelayKey, relayCount) for i := 0; i < relayCount; i++ { - relayKeys[i] = tu.RandomUint16() + relayKeys[i] = rand.Uint32() } blobCert := v2.BlobCertificate{ RelayKeys: relayKeys, @@ -135,7 +139,7 @@ func TestNoRelayResponse(t *testing.T) { relayCount := 10 relayKeys := make([]v2.RelayKey, relayCount) for i := 0; i < relayCount; i++ { - relayKeys[i] = tu.RandomUint16() + relayKeys[i] = rand.Uint32() } blobCert := v2.BlobCertificate{ RelayKeys: relayKeys, @@ -177,7 +181,7 @@ func TestGetBlobReturns0Len(t *testing.T) { relayCount := 10 relayKeys := make([]v2.RelayKey, relayCount) for i := 0; i < relayCount; i++ { - relayKeys[i] = tu.RandomUint16() + relayKeys[i] = rand.Uint32() } blobCert := v2.BlobCertificate{ RelayKeys: relayKeys, @@ -207,7 +211,7 @@ func TestFailedDecoding(t *testing.T) { relayCount := 10 relayKeys := make([]v2.RelayKey, relayCount) for i := 0; i < relayCount; i++ { - relayKeys[i] = tu.RandomUint16() + relayKeys[i] = rand.Uint32() } blobCert := v2.BlobCertificate{ RelayKeys: relayKeys, @@ -261,7 +265,7 @@ func TestGetCodec(t *testing.T) { // TestBuilder tests that the method that builds the client from config doesn't throw any obvious errors func TestBuilder(t *testing.T) { - clientConfig := clients.EigenDAClientConfig{ + clientConfig := &clients.EigenDAClientConfig{ StatusQueryTimeout: 10 * time.Minute, StatusQueryRetryInterval: 50 * time.Millisecond, ResponseTimeout: 10 * time.Second, @@ -277,7 +281,7 @@ func TestBuilder(t *testing.T) { SvcManagerAddr: "0x1234567890123456789012345678901234567890", } - relayClientConfig := clients.RelayClientConfig{ + relayClientConfig := &clients.RelayClientConfig{ Sockets: make(map[v2.RelayKey]string), UseSecureGrpcFlag: true, } diff --git a/common/testutils/test_utils.go b/common/testutils/test_utils.go index 586a1db64..e08180b65 100644 --- a/common/testutils/test_utils.go +++ b/common/testutils/test_utils.go @@ -108,8 +108,3 @@ func RandomString(length int) string { } return string(b) } - -// RandomUint16 generates a random uint16 -func RandomUint16() uint16 { - return uint16(rand.Uint32()) -} From 684866345456cc7bc8fdab20ba62b1e5b05e0382 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:39:24 -0500 Subject: [PATCH 03/18] Create new V2 client config Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/config_v2.go | 17 +++++++++++++++++ api/clients/eigenda_client_v2.go | 14 +++++--------- api/clients/eigenda_client_v2_test.go | 16 ++-------------- 3 files changed, 24 insertions(+), 23 deletions(-) create mode 100644 api/clients/config_v2.go diff --git a/api/clients/config_v2.go b/api/clients/config_v2.go new file mode 100644 index 000000000..667318be7 --- /dev/null +++ b/api/clients/config_v2.go @@ -0,0 +1,17 @@ +package clients + +import ( + "github.com/Layr-Labs/eigenda/api/clients/codecs" +) + +// EigenDAClientConfigV2 contains configuration values for EigenDAClientV2 +type EigenDAClientConfigV2 struct { + // The blob encoding version to use when writing blobs from the high level interface. + PutBlobEncodingVersion codecs.BlobEncodingVersion + + // Point verification mode does an IFFT on data before it is written, and does an FFT on data after it is read. + // This makes it possible to open points on the KZG commitment to prove that the field elements correspond to + // the commitment. With this mode disabled, you will need to supply the entire blob to perform a verification + // that any part of the data matches the KZG commitment. + DisablePointVerificationMode bool +} diff --git a/api/clients/eigenda_client_v2.go b/api/clients/eigenda_client_v2.go index e754be65f..81b03e7ec 100644 --- a/api/clients/eigenda_client_v2.go +++ b/api/clients/eigenda_client_v2.go @@ -15,7 +15,7 @@ type EigenDAClientV2 struct { log logging.Logger // doesn't need to be cryptographically secure, as it's only used to distribute load across relays random *rand.Rand - config *EigenDAClientConfig + config *EigenDAClientConfigV2 codec codecs.BlobCodec relayClient RelayClient } @@ -23,14 +23,9 @@ type EigenDAClientV2 struct { // BuildEigenDAClientV2 builds an EigenDAClientV2 from config structs. func BuildEigenDAClientV2( log logging.Logger, - config *EigenDAClientConfig, + config *EigenDAClientConfigV2, relayClientConfig *RelayClientConfig) (*EigenDAClientV2, error) { - err := config.CheckAndSetDefaults() - if err != nil { - return nil, fmt.Errorf("check and set client config defaults: %w", err) - } - relayClient, err := NewRelayClient(relayClientConfig, log) if err != nil { return nil, fmt.Errorf("new relay client: %w", err) @@ -48,7 +43,7 @@ func BuildEigenDAClientV2( func NewEigenDAClientV2( log logging.Logger, random *rand.Rand, - config *EigenDAClientConfig, + config *EigenDAClientConfigV2, relayClient RelayClient, codec codecs.BlobCodec) (*EigenDAClientV2, error) { @@ -92,6 +87,7 @@ func (c *EigenDAClientV2) GetBlob( for _, val := range indices { relayKey := blobCertificate.RelayKeys[val] + // TODO: does this need a timeout? data, err := c.relayClient.GetBlob(ctx, relayKey, blobKey) // if GetBlob returned an error, try calling a different relay @@ -138,7 +134,7 @@ func (c *EigenDAClientV2) Close() error { } // createCodec creates the codec based on client config values -func createCodec(config *EigenDAClientConfig) (codecs.BlobCodec, error) { +func createCodec(config *EigenDAClientConfigV2) (codecs.BlobCodec, error) { lowLevelCodec, err := codecs.BlobEncodingVersionToCodec(config.PutBlobEncodingVersion) if err != nil { return nil, fmt.Errorf("create low level codec: %w", err) diff --git a/api/clients/eigenda_client_v2_test.go b/api/clients/eigenda_client_v2_test.go index eea7f2341..02cf5334d 100644 --- a/api/clients/eigenda_client_v2_test.go +++ b/api/clients/eigenda_client_v2_test.go @@ -14,7 +14,6 @@ import ( "github.com/stretchr/testify/mock" "math/rand" "testing" - "time" ) type ClientV2Tester struct { @@ -32,7 +31,7 @@ func (c *ClientV2Tester) assertExpectations(t *testing.T) { func buildClientV2Tester(t *testing.T) ClientV2Tester { tu.InitializeRandom() logger := logging.NewNoopLogger() - clientConfig := &clients.EigenDAClientConfig{} + clientConfig := &clients.EigenDAClientConfigV2{} mockRelayClient := clientsmock.MockRelayClient{} mockCodec := codecsmock.BlobCodec{} @@ -265,20 +264,9 @@ func TestGetCodec(t *testing.T) { // TestBuilder tests that the method that builds the client from config doesn't throw any obvious errors func TestBuilder(t *testing.T) { - clientConfig := &clients.EigenDAClientConfig{ - StatusQueryTimeout: 10 * time.Minute, - StatusQueryRetryInterval: 50 * time.Millisecond, - ResponseTimeout: 10 * time.Second, - ConfirmationTimeout: 5 * time.Second, - CustomQuorumIDs: []uint{}, - SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", - DisableTLS: false, + clientConfig := &clients.EigenDAClientConfigV2{ PutBlobEncodingVersion: codecs.DefaultBlobEncoding, DisablePointVerificationMode: false, - WaitForFinalization: true, - RPC: "http://localhost:8080", - EthRpcUrl: "http://localhost:8545", - SvcManagerAddr: "0x1234567890123456789012345678901234567890", } relayClientConfig := &clients.RelayClientConfig{ From a48afb152a4bf90cc901c2f1ba6bfcd23cf90cca Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:07:32 -0500 Subject: [PATCH 04/18] Respond to more PR comments Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- .../mock/{BlobCodec.go => blob_codec.go} | 0 api/clients/config_v2.go | 24 ++++++++++++---- api/clients/eigenda_client_v2.go | 28 +++++++++---------- api/clients/eigenda_client_v2_test.go | 4 +-- 4 files changed, 34 insertions(+), 22 deletions(-) rename api/clients/codecs/mock/{BlobCodec.go => blob_codec.go} (100%) diff --git a/api/clients/codecs/mock/BlobCodec.go b/api/clients/codecs/mock/blob_codec.go similarity index 100% rename from api/clients/codecs/mock/BlobCodec.go rename to api/clients/codecs/mock/blob_codec.go diff --git a/api/clients/config_v2.go b/api/clients/config_v2.go index 667318be7..7189ad05f 100644 --- a/api/clients/config_v2.go +++ b/api/clients/config_v2.go @@ -4,14 +4,26 @@ import ( "github.com/Layr-Labs/eigenda/api/clients/codecs" ) +// VerificationMode is an enum that represents the different ways that a blob may be encoded/decoded between +// the client and the disperser. +type VerificationMode uint + +const ( + // TODO: write good docs here for IFFT and NoIFFT (I need to update my understanding to be able to write this) + IFFT VerificationMode = iota + NoIFFT +) + // EigenDAClientConfigV2 contains configuration values for EigenDAClientV2 type EigenDAClientConfigV2 struct { - // The blob encoding version to use when writing blobs from the high level interface. - PutBlobEncodingVersion codecs.BlobEncodingVersion + // The blob encoding version to use when writing and reading blobs + BlobEncodingVersion codecs.BlobEncodingVersion - // Point verification mode does an IFFT on data before it is written, and does an FFT on data after it is read. - // This makes it possible to open points on the KZG commitment to prove that the field elements correspond to - // the commitment. With this mode disabled, you will need to supply the entire blob to perform a verification + // If PointVerificationMode is IFFT, then the client codec will do an IFFT on blobs before they are dispersed, and + // will do an FFT on blobs after receiving them. This makes it possible to open points on the KZG commitment to prove + // that the field elements correspond to the commitment. + // + // If PointVerificationMode is NoIFFT, the blob must be supplied in its entirety, to perform a verification // that any part of the data matches the KZG commitment. - DisablePointVerificationMode bool + PointVerificationMode VerificationMode } diff --git a/api/clients/eigenda_client_v2.go b/api/clients/eigenda_client_v2.go index 81b03e7ec..2c6a2aeda 100644 --- a/api/clients/eigenda_client_v2.go +++ b/api/clients/eigenda_client_v2.go @@ -2,6 +2,7 @@ package clients import ( "context" + "errors" "fmt" "github.com/Layr-Labs/eigenda/api/clients/codecs" corev2 "github.com/Layr-Labs/eigenda/core/v2" @@ -11,6 +12,8 @@ import ( ) // EigenDAClientV2 provides the ability to get blobs from the relay subsystem, and to send new blobs to the disperser. +// +// This struct is not threadsafe. type EigenDAClientV2 struct { log logging.Logger // doesn't need to be cryptographically secure, as it's only used to distribute load across relays @@ -69,17 +72,11 @@ func (c *EigenDAClientV2) GetBlob( relayKeyCount := len(blobCertificate.RelayKeys) if relayKeyCount == 0 { - return nil, fmt.Errorf("relay key count is zero") + return nil, errors.New("relay key count is zero") } - var indices []int // create a randomized array of indices, so that it isn't always the first relay in the list which gets hit - for i := 0; i < relayKeyCount; i++ { - indices = append(indices, i) - } - c.random.Shuffle(len(indices), func(i int, j int) { - indices[i], indices[j] = indices[j], indices[i] - }) + indices := c.random.Perm(relayKeyCount) // TODO (litt3): consider creating a utility which can deprioritize relays that fail to respond (or respond maliciously) @@ -98,21 +95,21 @@ func (c *EigenDAClientV2) GetBlob( // An honest relay should never send an empty blob if len(data) == 0 { - c.log.Warn("blob received from relay had length 0", "blobKey", blobKey, "relayKey", relayKey, "error", err) + c.log.Warn("blob received from relay had length 0", "blobKey", blobKey, "relayKey", relayKey) continue } // An honest relay should never send a blob which cannot be decoded decodedData, err := c.codec.DecodeBlob(data) if err != nil { - c.log.Warn("error decoding blob", "blobKey", blobKey, "relayKey", relayKey, "error", err) + c.log.Warn("error decoding blob from relay", "blobKey", blobKey, "relayKey", relayKey, "error", err) continue } return decodedData, nil } - return nil, fmt.Errorf("unable to retrieve blob from any relay") + return nil, fmt.Errorf("unable to retrieve blob from any relay. relay count: %d", relayKeyCount) } // GetCodec returns the codec the client uses for encoding and decoding blobs @@ -135,14 +132,17 @@ func (c *EigenDAClientV2) Close() error { // createCodec creates the codec based on client config values func createCodec(config *EigenDAClientConfigV2) (codecs.BlobCodec, error) { - lowLevelCodec, err := codecs.BlobEncodingVersionToCodec(config.PutBlobEncodingVersion) + lowLevelCodec, err := codecs.BlobEncodingVersionToCodec(config.BlobEncodingVersion) if err != nil { return nil, fmt.Errorf("create low level codec: %w", err) } - if config.DisablePointVerificationMode { + switch config.PointVerificationMode { + case NoIFFT: return codecs.NewNoIFFTCodec(lowLevelCodec), nil - } else { + case IFFT: return codecs.NewIFFTCodec(lowLevelCodec), nil + default: + return nil, fmt.Errorf("unsupported point verification mode: %d", config.PointVerificationMode) } } diff --git a/api/clients/eigenda_client_v2_test.go b/api/clients/eigenda_client_v2_test.go index 02cf5334d..3a0c59479 100644 --- a/api/clients/eigenda_client_v2_test.go +++ b/api/clients/eigenda_client_v2_test.go @@ -265,8 +265,8 @@ func TestGetCodec(t *testing.T) { // TestBuilder tests that the method that builds the client from config doesn't throw any obvious errors func TestBuilder(t *testing.T) { clientConfig := &clients.EigenDAClientConfigV2{ - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, - DisablePointVerificationMode: false, + BlobEncodingVersion: codecs.DefaultBlobEncoding, + PointVerificationMode: clients.IFFT, } relayClientConfig := &clients.RelayClientConfig{ From 225f2a330561f6e6c52225893449acb657d60763 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:20:33 -0500 Subject: [PATCH 05/18] Fix failing unit test Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/eigenda_client_v2_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/clients/eigenda_client_v2_test.go b/api/clients/eigenda_client_v2_test.go index 3a0c59479..53c68090e 100644 --- a/api/clients/eigenda_client_v2_test.go +++ b/api/clients/eigenda_client_v2_test.go @@ -269,8 +269,11 @@ func TestBuilder(t *testing.T) { PointVerificationMode: clients.IFFT, } + sockets := make(map[v2.RelayKey]string) + sockets[v2.RelayKey(44)] = "socketVal" + relayClientConfig := &clients.RelayClientConfig{ - Sockets: make(map[v2.RelayKey]string), + Sockets: sockets, UseSecureGrpcFlag: true, } From e9d91c56f51cbf786ce5c8304c8adf45dc21215f Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:08:32 -0500 Subject: [PATCH 06/18] Adopt new package structure Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/{config_v2.go => v2/config.go} | 6 +- .../eigenda_client.go} | 49 ++++---- .../eigenda_client_test.go} | 112 +++++++++--------- 3 files changed, 84 insertions(+), 83 deletions(-) rename api/clients/{config_v2.go => v2/config.go} (88%) rename api/clients/{eigenda_client_v2.go => v2/eigenda_client.go} (74%) rename api/clients/{eigenda_client_v2_test.go => v2/eigenda_client_test.go} (72%) diff --git a/api/clients/config_v2.go b/api/clients/v2/config.go similarity index 88% rename from api/clients/config_v2.go rename to api/clients/v2/config.go index 7189ad05f..5b53eec90 100644 --- a/api/clients/config_v2.go +++ b/api/clients/v2/config.go @@ -1,4 +1,4 @@ -package clients +package v2 import ( "github.com/Layr-Labs/eigenda/api/clients/codecs" @@ -14,8 +14,8 @@ const ( NoIFFT ) -// EigenDAClientConfigV2 contains configuration values for EigenDAClientV2 -type EigenDAClientConfigV2 struct { +// EigenDAClientConfig contains configuration values for EigenDAClient +type EigenDAClientConfig struct { // The blob encoding version to use when writing and reading blobs BlobEncodingVersion codecs.BlobEncodingVersion diff --git a/api/clients/eigenda_client_v2.go b/api/clients/v2/eigenda_client.go similarity index 74% rename from api/clients/eigenda_client_v2.go rename to api/clients/v2/eigenda_client.go index 2c6a2aeda..9edac7a33 100644 --- a/api/clients/eigenda_client_v2.go +++ b/api/clients/v2/eigenda_client.go @@ -1,35 +1,36 @@ -package clients +package v2 import ( "context" "errors" "fmt" + "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/api/clients/codecs" - corev2 "github.com/Layr-Labs/eigenda/core/v2" + core "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/cockroachdb/errors/join" "math/rand" ) -// EigenDAClientV2 provides the ability to get blobs from the relay subsystem, and to send new blobs to the disperser. +// EigenDAClient provides the ability to get blobs from the relay subsystem, and to send new blobs to the disperser. // // This struct is not threadsafe. -type EigenDAClientV2 struct { +type EigenDAClient struct { log logging.Logger // doesn't need to be cryptographically secure, as it's only used to distribute load across relays random *rand.Rand - config *EigenDAClientConfigV2 + config *EigenDAClientConfig codec codecs.BlobCodec - relayClient RelayClient + relayClient clients.RelayClient } -// BuildEigenDAClientV2 builds an EigenDAClientV2 from config structs. -func BuildEigenDAClientV2( +// BuildEigenDAClient builds an EigenDAClient from config structs. +func BuildEigenDAClient( log logging.Logger, - config *EigenDAClientConfigV2, - relayClientConfig *RelayClientConfig) (*EigenDAClientV2, error) { + config *EigenDAClientConfig, + relayClientConfig *clients.RelayClientConfig) (*EigenDAClient, error) { - relayClient, err := NewRelayClient(relayClientConfig, log) + relayClient, err := clients.NewRelayClient(relayClientConfig, log) if err != nil { return nil, fmt.Errorf("new relay client: %w", err) } @@ -39,18 +40,18 @@ func BuildEigenDAClientV2( return nil, err } - return NewEigenDAClientV2(log, rand.New(rand.NewSource(rand.Int63())), config, relayClient, codec) + return NewEigenDAClient(log, rand.New(rand.NewSource(rand.Int63())), config, relayClient, codec) } -// NewEigenDAClientV2 assembles an EigenDAClientV2 from subcomponents that have already been constructed and initialized. -func NewEigenDAClientV2( +// NewEigenDAClient assembles an EigenDAClient from subcomponents that have already been constructed and initialized. +func NewEigenDAClient( log logging.Logger, random *rand.Rand, - config *EigenDAClientConfigV2, - relayClient RelayClient, - codec codecs.BlobCodec) (*EigenDAClientV2, error) { + config *EigenDAClientConfig, + relayClient clients.RelayClient, + codec codecs.BlobCodec) (*EigenDAClient, error) { - return &EigenDAClientV2{ + return &EigenDAClient{ log: log, random: random, config: config, @@ -64,10 +65,10 @@ func NewEigenDAClientV2( // The relays are attempted in random order. // // The returned blob is decoded. -func (c *EigenDAClientV2) GetBlob( +func (c *EigenDAClient) GetBlob( ctx context.Context, - blobKey corev2.BlobKey, - blobCertificate corev2.BlobCertificate) ([]byte, error) { + blobKey core.BlobKey, + blobCertificate core.BlobCertificate) ([]byte, error) { relayKeyCount := len(blobCertificate.RelayKeys) @@ -113,7 +114,7 @@ func (c *EigenDAClientV2) GetBlob( } // GetCodec returns the codec the client uses for encoding and decoding blobs -func (c *EigenDAClientV2) GetCodec() codecs.BlobCodec { +func (c *EigenDAClient) GetCodec() codecs.BlobCodec { return c.codec } @@ -123,7 +124,7 @@ func (c *EigenDAClientV2) GetCodec() codecs.BlobCodec { // Any and all errors returned from closing internal clients will be joined and returned. // // This method should only be called once. -func (c *EigenDAClientV2) Close() error { +func (c *EigenDAClient) Close() error { relayClientErr := c.relayClient.Close() // TODO: this is using join, since there will be more subcomponents requiring closing after adding PUT functionality @@ -131,7 +132,7 @@ func (c *EigenDAClientV2) Close() error { } // createCodec creates the codec based on client config values -func createCodec(config *EigenDAClientConfigV2) (codecs.BlobCodec, error) { +func createCodec(config *EigenDAClientConfig) (codecs.BlobCodec, error) { lowLevelCodec, err := codecs.BlobEncodingVersionToCodec(config.BlobEncodingVersion) if err != nil { return nil, fmt.Errorf("create low level codec: %w", err) diff --git a/api/clients/eigenda_client_v2_test.go b/api/clients/v2/eigenda_client_test.go similarity index 72% rename from api/clients/eigenda_client_v2_test.go rename to api/clients/v2/eigenda_client_test.go index 53c68090e..31936ed7f 100644 --- a/api/clients/eigenda_client_v2_test.go +++ b/api/clients/v2/eigenda_client_test.go @@ -1,4 +1,4 @@ -package clients_test +package v2 import ( "context" @@ -8,7 +8,7 @@ import ( codecsmock "github.com/Layr-Labs/eigenda/api/clients/codecs/mock" clientsmock "github.com/Layr-Labs/eigenda/api/clients/mock" tu "github.com/Layr-Labs/eigenda/common/testutils" - v2 "github.com/Layr-Labs/eigenda/core/v2" + core "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -16,22 +16,22 @@ import ( "testing" ) -type ClientV2Tester struct { - ClientV2 *clients.EigenDAClientV2 +type ClientTester struct { + Client *EigenDAClient MockRelayClient *clientsmock.MockRelayClient MockCodec *codecsmock.BlobCodec } -func (c *ClientV2Tester) assertExpectations(t *testing.T) { +func (c *ClientTester) assertExpectations(t *testing.T) { c.MockRelayClient.AssertExpectations(t) c.MockCodec.AssertExpectations(t) } -// buildClientV2Tester sets up a V2 client, with mocks necessary for testing -func buildClientV2Tester(t *testing.T) ClientV2Tester { +// buildClientTester sets up a client with mocks necessary for testing +func buildClientTester(t *testing.T) ClientTester { tu.InitializeRandom() logger := logging.NewNoopLogger() - clientConfig := &clients.EigenDAClientConfigV2{} + clientConfig := &EigenDAClientConfig{} mockRelayClient := clientsmock.MockRelayClient{} mockCodec := codecsmock.BlobCodec{} @@ -39,7 +39,7 @@ func buildClientV2Tester(t *testing.T) ClientV2Tester { // TODO (litt3): use TestRandom once the PR merges https://github.com/Layr-Labs/eigenda/pull/976 random := rand.New(rand.NewSource(rand.Int63())) - client, err := clients.NewEigenDAClientV2( + client, err := NewEigenDAClient( logger, random, clientConfig, @@ -49,8 +49,8 @@ func buildClientV2Tester(t *testing.T) ClientV2Tester { assert.NotNil(t, client) assert.Nil(t, err) - return ClientV2Tester{ - ClientV2: client, + return ClientTester{ + Client: client, MockRelayClient: &mockRelayClient, MockCodec: &mockCodec, } @@ -58,21 +58,21 @@ func buildClientV2Tester(t *testing.T) ClientV2Tester { // TestGetBlobSuccess tests that a blob is received without error in the happy case func TestGetBlobSuccess(t *testing.T) { - tester := buildClientV2Tester(t) + tester := buildClientTester(t) - blobKey := v2.BlobKey(tu.RandomBytes(32)) + blobKey := core.BlobKey(tu.RandomBytes(32)) blobBytes := tu.RandomBytes(100) - relayKeys := make([]v2.RelayKey, 1) + relayKeys := make([]core.RelayKey, 1) relayKeys[0] = rand.Uint32() - blobCert := v2.BlobCertificate{ + blobCert := core.BlobCertificate{ RelayKeys: relayKeys, } tester.MockRelayClient.On("GetBlob", mock.Anything, relayKeys[0], blobKey).Return(blobBytes, nil).Once() tester.MockCodec.On("DecodeBlob", blobBytes).Return(tu.RandomBytes(50), nil).Once() - blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) assert.NotNil(t, blob) assert.Nil(t, err) @@ -83,17 +83,17 @@ func TestGetBlobSuccess(t *testing.T) { // TestRandomRelayRetries verifies correct behavior when some relays from the certificate do not respond with the blob, // requiring the client to retry with other relays. func TestRandomRelayRetries(t *testing.T) { - tester := buildClientV2Tester(t) + tester := buildClientTester(t) - blobKey := v2.BlobKey(tu.RandomBytes(32)) + blobKey := core.BlobKey(tu.RandomBytes(32)) blobBytes := tu.RandomBytes(100) relayCount := 100 - relayKeys := make([]v2.RelayKey, relayCount) + relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { relayKeys[i] = rand.Uint32() } - blobCert := v2.BlobCertificate{ + blobCert := core.BlobCertificate{ RelayKeys: relayKeys, } @@ -101,8 +101,8 @@ func TestRandomRelayRetries(t *testing.T) { // we will be asserting that it takes a different amount of retries to dial this relay, since the array of relay keys to try is randomized onlineRelayKey := relayKeys[rand.Intn(len(relayKeys))] - offlineKeyMatcher := func(relayKey v2.RelayKey) bool { return relayKey != onlineRelayKey } - onlineKeyMatcher := func(relayKey v2.RelayKey) bool { return relayKey == onlineRelayKey } + offlineKeyMatcher := func(relayKey core.RelayKey) bool { return relayKey != onlineRelayKey } + onlineKeyMatcher := func(relayKey core.RelayKey) bool { return relayKey == onlineRelayKey } var failedCallCount int tester.MockRelayClient.On("GetBlob", mock.Anything, mock.MatchedBy(offlineKeyMatcher), blobKey).Return(nil, fmt.Errorf("offline relay")).Run(func(args mock.Arguments) { failedCallCount++ @@ -116,7 +116,7 @@ func TestRandomRelayRetries(t *testing.T) { for i := 0; i < relayCount; i++ { failedCallCount = 0 - blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) assert.NotNil(t, blob) assert.Nil(t, err) @@ -131,22 +131,22 @@ func TestRandomRelayRetries(t *testing.T) { // TestNoRelayResponse tests functionality when none of the relays listed in the blob certificate respond func TestNoRelayResponse(t *testing.T) { - tester := buildClientV2Tester(t) + tester := buildClientTester(t) - blobKey := v2.BlobKey(tu.RandomBytes(32)) + blobKey := core.BlobKey(tu.RandomBytes(32)) relayCount := 10 - relayKeys := make([]v2.RelayKey, relayCount) + relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { relayKeys[i] = rand.Uint32() } - blobCert := v2.BlobCertificate{ + blobCert := core.BlobCertificate{ RelayKeys: relayKeys, } tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(nil, fmt.Errorf("offline relay")) - blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) assert.Nil(t, blob) assert.NotNil(t, err) @@ -155,16 +155,16 @@ func TestNoRelayResponse(t *testing.T) { // TestNoRelaysInCert tests that having no relay keys in the cert is handled gracefully func TestNoRelaysInCert(t *testing.T) { - tester := buildClientV2Tester(t) + tester := buildClientTester(t) - blobKey := v2.BlobKey(tu.RandomBytes(32)) + blobKey := core.BlobKey(tu.RandomBytes(32)) // cert has no listed relay keys - blobCert := v2.BlobCertificate{ - RelayKeys: []v2.RelayKey{}, + blobCert := core.BlobCertificate{ + RelayKeys: []core.RelayKey{}, } - blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) assert.Nil(t, blob) assert.NotNil(t, err) @@ -173,16 +173,16 @@ func TestNoRelaysInCert(t *testing.T) { // TestGetBlobReturns0Len verifies that a 0 length blob returned from a relay is handled gracefully, and that the client retries after such a failure func TestGetBlobReturns0Len(t *testing.T) { - tester := buildClientV2Tester(t) + tester := buildClientTester(t) - blobKey := v2.BlobKey(tu.RandomBytes(32)) + blobKey := core.BlobKey(tu.RandomBytes(32)) relayCount := 10 - relayKeys := make([]v2.RelayKey, relayCount) + relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { relayKeys[i] = rand.Uint32() } - blobCert := v2.BlobCertificate{ + blobCert := core.BlobCertificate{ RelayKeys: relayKeys, } @@ -194,7 +194,7 @@ func TestGetBlobReturns0Len(t *testing.T) { tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tu.RandomBytes(50), nil) // the call to the first relay will fail with a 0 len blob returned. the call to the second relay will succeed - blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) assert.NotNil(t, blob) assert.Nil(t, err) @@ -203,16 +203,16 @@ func TestGetBlobReturns0Len(t *testing.T) { // TestFailedDecoding verifies that a failed blob decode is handled gracefully, and that the client retries after such a failure func TestFailedDecoding(t *testing.T) { - tester := buildClientV2Tester(t) + tester := buildClientTester(t) - blobKey := v2.BlobKey(tu.RandomBytes(32)) + blobKey := core.BlobKey(tu.RandomBytes(32)) relayCount := 10 - relayKeys := make([]v2.RelayKey, relayCount) + relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { relayKeys[i] = rand.Uint32() } - blobCert := v2.BlobCertificate{ + blobCert := core.BlobCertificate{ RelayKeys: relayKeys, } @@ -222,7 +222,7 @@ func TestFailedDecoding(t *testing.T) { tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tu.RandomBytes(50), nil).Once() // decoding will fail the first time, but succeed the second time - blob, err := tester.ClientV2.GetBlob(context.Background(), blobKey, blobCert) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) assert.NotNil(t, blob) assert.Nil(t, err) @@ -231,11 +231,11 @@ func TestFailedDecoding(t *testing.T) { // TestErrorFreeClose tests the happy case, where none of the internal closes yield an error func TestErrorFreeClose(t *testing.T) { - tester := buildClientV2Tester(t) + tester := buildClientTester(t) tester.MockRelayClient.On("Close").Return(nil).Once() - err := tester.ClientV2.Close() + err := tester.Client.Close() assert.Nil(t, err) tester.assertExpectations(t) @@ -243,11 +243,11 @@ func TestErrorFreeClose(t *testing.T) { // TestErrorClose tests what happens when subcomponents throw errors when being closed func TestErrorClose(t *testing.T) { - tester := buildClientV2Tester(t) + tester := buildClientTester(t) tester.MockRelayClient.On("Close").Return(fmt.Errorf("close failed")).Once() - err := tester.ClientV2.Close() + err := tester.Client.Close() assert.NotNil(t, err) tester.assertExpectations(t) @@ -255,35 +255,35 @@ func TestErrorClose(t *testing.T) { // TestGetCodec checks that the codec used in construction is returned by GetCodec func TestGetCodec(t *testing.T) { - tester := buildClientV2Tester(t) + tester := buildClientTester(t) - assert.Equal(t, tester.MockCodec, tester.ClientV2.GetCodec()) + assert.Equal(t, tester.MockCodec, tester.Client.GetCodec()) tester.assertExpectations(t) } // TestBuilder tests that the method that builds the client from config doesn't throw any obvious errors func TestBuilder(t *testing.T) { - clientConfig := &clients.EigenDAClientConfigV2{ + clientConfig := &EigenDAClientConfig{ BlobEncodingVersion: codecs.DefaultBlobEncoding, - PointVerificationMode: clients.IFFT, + PointVerificationMode: IFFT, } - sockets := make(map[v2.RelayKey]string) - sockets[v2.RelayKey(44)] = "socketVal" + sockets := make(map[core.RelayKey]string) + sockets[core.RelayKey(44)] = "socketVal" relayClientConfig := &clients.RelayClientConfig{ Sockets: sockets, UseSecureGrpcFlag: true, } - clientV2, err := clients.BuildEigenDAClientV2( + client, err := BuildEigenDAClient( logging.NewNoopLogger(), clientConfig, relayClientConfig) - assert.NotNil(t, clientV2) + assert.NotNil(t, client) assert.Nil(t, err) - assert.NotNil(t, clientV2.GetCodec()) + assert.NotNil(t, client.GetCodec()) } From dd3c262aae4c23714e9e0966680036f804035bb9 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:15:33 -0500 Subject: [PATCH 07/18] Use new test random util Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/eigenda_client_test.go | 51 +++++++++++++-------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/api/clients/v2/eigenda_client_test.go b/api/clients/v2/eigenda_client_test.go index 31936ed7f..2f7007724 100644 --- a/api/clients/v2/eigenda_client_test.go +++ b/api/clients/v2/eigenda_client_test.go @@ -7,16 +7,16 @@ import ( "github.com/Layr-Labs/eigenda/api/clients/codecs" codecsmock "github.com/Layr-Labs/eigenda/api/clients/codecs/mock" clientsmock "github.com/Layr-Labs/eigenda/api/clients/mock" - tu "github.com/Layr-Labs/eigenda/common/testutils" + testrandom "github.com/Layr-Labs/eigenda/common/testutils/random" core "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "math/rand" "testing" ) type ClientTester struct { + Random *testrandom.TestRandom Client *EigenDAClient MockRelayClient *clientsmock.MockRelayClient MockCodec *codecsmock.BlobCodec @@ -29,19 +29,17 @@ func (c *ClientTester) assertExpectations(t *testing.T) { // buildClientTester sets up a client with mocks necessary for testing func buildClientTester(t *testing.T) ClientTester { - tu.InitializeRandom() logger := logging.NewNoopLogger() clientConfig := &EigenDAClientConfig{} mockRelayClient := clientsmock.MockRelayClient{} mockCodec := codecsmock.BlobCodec{} - // TODO (litt3): use TestRandom once the PR merges https://github.com/Layr-Labs/eigenda/pull/976 - random := rand.New(rand.NewSource(rand.Int63())) + random := testrandom.NewTestRandom() client, err := NewEigenDAClient( logger, - random, + random.Rand, clientConfig, &mockRelayClient, &mockCodec) @@ -50,6 +48,7 @@ func buildClientTester(t *testing.T) ClientTester { assert.Nil(t, err) return ClientTester{ + Random: random, Client: client, MockRelayClient: &mockRelayClient, MockCodec: &mockCodec, @@ -60,17 +59,17 @@ func buildClientTester(t *testing.T) ClientTester { func TestGetBlobSuccess(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tu.RandomBytes(32)) - blobBytes := tu.RandomBytes(100) + blobKey := core.BlobKey(tester.Random.RandomBytes(32)) + blobBytes := tester.Random.RandomBytes(100) relayKeys := make([]core.RelayKey, 1) - relayKeys[0] = rand.Uint32() + relayKeys[0] = tester.Random.Uint32() blobCert := core.BlobCertificate{ RelayKeys: relayKeys, } tester.MockRelayClient.On("GetBlob", mock.Anything, relayKeys[0], blobKey).Return(blobBytes, nil).Once() - tester.MockCodec.On("DecodeBlob", blobBytes).Return(tu.RandomBytes(50), nil).Once() + tester.MockCodec.On("DecodeBlob", blobBytes).Return(tester.Random.RandomBytes(50), nil).Once() blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) @@ -85,13 +84,13 @@ func TestGetBlobSuccess(t *testing.T) { func TestRandomRelayRetries(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tu.RandomBytes(32)) - blobBytes := tu.RandomBytes(100) + blobKey := core.BlobKey(tester.Random.RandomBytes(32)) + blobBytes := tester.Random.RandomBytes(100) relayCount := 100 relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { - relayKeys[i] = rand.Uint32() + relayKeys[i] = tester.Random.Uint32() } blobCert := core.BlobCertificate{ RelayKeys: relayKeys, @@ -99,7 +98,7 @@ func TestRandomRelayRetries(t *testing.T) { // for this test, only a single relay is online // we will be asserting that it takes a different amount of retries to dial this relay, since the array of relay keys to try is randomized - onlineRelayKey := relayKeys[rand.Intn(len(relayKeys))] + onlineRelayKey := relayKeys[tester.Random.Intn(len(relayKeys))] offlineKeyMatcher := func(relayKey core.RelayKey) bool { return relayKey != onlineRelayKey } onlineKeyMatcher := func(relayKey core.RelayKey) bool { return relayKey == onlineRelayKey } @@ -108,7 +107,7 @@ func TestRandomRelayRetries(t *testing.T) { failedCallCount++ }) tester.MockRelayClient.On("GetBlob", mock.Anything, mock.MatchedBy(onlineKeyMatcher), blobKey).Return(blobBytes, nil) - tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tu.RandomBytes(50), nil) + tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.RandomBytes(50), nil) // keep track of how many tries various blob retrievals require // this allows us to assert that there is variability, i.e. that relay call order is actually random @@ -133,12 +132,12 @@ func TestRandomRelayRetries(t *testing.T) { func TestNoRelayResponse(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tu.RandomBytes(32)) + blobKey := core.BlobKey(tester.Random.RandomBytes(32)) relayCount := 10 relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { - relayKeys[i] = rand.Uint32() + relayKeys[i] = tester.Random.Uint32() } blobCert := core.BlobCertificate{ RelayKeys: relayKeys, @@ -157,7 +156,7 @@ func TestNoRelayResponse(t *testing.T) { func TestNoRelaysInCert(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tu.RandomBytes(32)) + blobKey := core.BlobKey(tester.Random.RandomBytes(32)) // cert has no listed relay keys blobCert := core.BlobCertificate{ @@ -175,12 +174,12 @@ func TestNoRelaysInCert(t *testing.T) { func TestGetBlobReturns0Len(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tu.RandomBytes(32)) + blobKey := core.BlobKey(tester.Random.RandomBytes(32)) relayCount := 10 relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { - relayKeys[i] = rand.Uint32() + relayKeys[i] = tester.Random.Uint32() } blobCert := core.BlobCertificate{ RelayKeys: relayKeys, @@ -189,9 +188,9 @@ func TestGetBlobReturns0Len(t *testing.T) { // the first GetBlob will return a 0 len blob tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return([]byte{}, nil).Once() // the second call will return random bytes - tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tu.RandomBytes(100), nil).Once() + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tester.Random.RandomBytes(100), nil).Once() - tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tu.RandomBytes(50), nil) + tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.RandomBytes(50), nil) // the call to the first relay will fail with a 0 len blob returned. the call to the second relay will succeed blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) @@ -205,21 +204,21 @@ func TestGetBlobReturns0Len(t *testing.T) { func TestFailedDecoding(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tu.RandomBytes(32)) + blobKey := core.BlobKey(tester.Random.RandomBytes(32)) relayCount := 10 relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { - relayKeys[i] = rand.Uint32() + relayKeys[i] = tester.Random.Uint32() } blobCert := core.BlobCertificate{ RelayKeys: relayKeys, } - tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tu.RandomBytes(100), nil) + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tester.Random.RandomBytes(100), nil) tester.MockCodec.On("DecodeBlob", mock.Anything).Return(nil, fmt.Errorf("decode failed")).Once() - tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tu.RandomBytes(50), nil).Once() + tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.RandomBytes(50), nil).Once() // decoding will fail the first time, but succeed the second time blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) From 88df865ae1824bd52d0cb4185407bedbb7283351 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:16:47 -0500 Subject: [PATCH 08/18] Implement relay call timeout Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 4 ++ api/clients/v2/eigenda_client.go | 15 ++++++- api/clients/v2/eigenda_client_test.go | 58 ++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index 5b53eec90..10318508b 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -2,6 +2,7 @@ package v2 import ( "github.com/Layr-Labs/eigenda/api/clients/codecs" + "time" ) // VerificationMode is an enum that represents the different ways that a blob may be encoded/decoded between @@ -26,4 +27,7 @@ type EigenDAClientConfig struct { // If PointVerificationMode is NoIFFT, the blob must be supplied in its entirety, to perform a verification // that any part of the data matches the KZG commitment. PointVerificationMode VerificationMode + + // The timeout duration for relay calls + RelayTimeout time.Duration } diff --git a/api/clients/v2/eigenda_client.go b/api/clients/v2/eigenda_client.go index 9edac7a33..d2a4ffb27 100644 --- a/api/clients/v2/eigenda_client.go +++ b/api/clients/v2/eigenda_client.go @@ -85,8 +85,7 @@ func (c *EigenDAClient) GetBlob( for _, val := range indices { relayKey := blobCertificate.RelayKeys[val] - // TODO: does this need a timeout? - data, err := c.relayClient.GetBlob(ctx, relayKey, blobKey) + data, err := c.getBlobWithTimeout(ctx, relayKey, blobKey) // if GetBlob returned an error, try calling a different relay if err != nil { @@ -113,6 +112,18 @@ func (c *EigenDAClient) GetBlob( return nil, fmt.Errorf("unable to retrieve blob from any relay. relay count: %d", relayKeyCount) } +// getBlobWithTimeout attempts to get a blob from a given relay, and times out based on config.RelayTimeout +func (c *EigenDAClient) getBlobWithTimeout( + ctx context.Context, + relayKey core.RelayKey, + blobKey core.BlobKey) ([]byte, error) { + + timeoutCtx, cancel := context.WithTimeout(ctx, c.config.RelayTimeout) + defer cancel() + + return c.relayClient.GetBlob(timeoutCtx, relayKey, blobKey) +} + // GetCodec returns the codec the client uses for encoding and decoding blobs func (c *EigenDAClient) GetCodec() codecs.BlobCodec { return c.codec diff --git a/api/clients/v2/eigenda_client_test.go b/api/clients/v2/eigenda_client_test.go index 2f7007724..f3b9d62b4 100644 --- a/api/clients/v2/eigenda_client_test.go +++ b/api/clients/v2/eigenda_client_test.go @@ -2,6 +2,7 @@ package v2 import ( "context" + "errors" "fmt" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/api/clients/codecs" @@ -13,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "testing" + "time" ) type ClientTester struct { @@ -30,7 +32,9 @@ func (c *ClientTester) assertExpectations(t *testing.T) { // buildClientTester sets up a client with mocks necessary for testing func buildClientTester(t *testing.T) ClientTester { logger := logging.NewNoopLogger() - clientConfig := &EigenDAClientConfig{} + clientConfig := &EigenDAClientConfig{ + RelayTimeout: 50 * time.Millisecond, + } mockRelayClient := clientsmock.MockRelayClient{} mockCodec := codecsmock.BlobCodec{} @@ -79,6 +83,57 @@ func TestGetBlobSuccess(t *testing.T) { tester.assertExpectations(t) } +// TestRelayCallTimeout verifies that calls to the relay timeout after the expected duration +func TestRelayCallTimeout(t *testing.T) { + tester := buildClientTester(t) + + blobKey := core.BlobKey(tester.Random.RandomBytes(32)) + + relayKeys := make([]core.RelayKey, 1) + relayKeys[0] = tester.Random.Uint32() + blobCert := core.BlobCertificate{ + RelayKeys: relayKeys, + } + + // the timeout should occur before the panic has a chance to be triggered + tester.MockRelayClient.On("GetBlob", mock.Anything, relayKeys[0], blobKey).Return( + nil, errors.New("timeout")).Once().Run( + func(args mock.Arguments) { + ctx := args.Get(0).(context.Context) + select { + case <-ctx.Done(): + // this is the expected case + return + case <-time.After(time.Second): + panic("call should have timed out first") + } + }) + + // the panic should be triggered, since it happens faster than the configured timout + tester.MockRelayClient.On("GetBlob", mock.Anything, relayKeys[0], blobKey).Return( + nil, errors.New("timeout")).Once().Run( + func(args mock.Arguments) { + ctx := args.Get(0).(context.Context) + select { + case <-ctx.Done(): + return + case <-time.After(time.Millisecond): + // this is the expected case + panic("call should not have timed out") + } + }) + + assert.NotPanics(t, func() { + _, _ = tester.Client.GetBlob(context.Background(), blobKey, blobCert) + }) + + assert.Panics(t, func() { + _, _ = tester.Client.GetBlob(context.Background(), blobKey, blobCert) + }) + + tester.assertExpectations(t) +} + // TestRandomRelayRetries verifies correct behavior when some relays from the certificate do not respond with the blob, // requiring the client to retry with other relays. func TestRandomRelayRetries(t *testing.T) { @@ -266,6 +321,7 @@ func TestBuilder(t *testing.T) { clientConfig := &EigenDAClientConfig{ BlobEncodingVersion: codecs.DefaultBlobEncoding, PointVerificationMode: IFFT, + RelayTimeout: 500 * time.Millisecond, } sockets := make(map[core.RelayKey]string) From 505a1f040108cad5f4309000c2c949967c4bfbf4 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:57:48 -0500 Subject: [PATCH 09/18] Use correct error join method Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/eigenda_client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/clients/v2/eigenda_client.go b/api/clients/v2/eigenda_client.go index d2a4ffb27..b256f509f 100644 --- a/api/clients/v2/eigenda_client.go +++ b/api/clients/v2/eigenda_client.go @@ -8,7 +8,6 @@ import ( "github.com/Layr-Labs/eigenda/api/clients/codecs" core "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigensdk-go/logging" - "github.com/cockroachdb/errors/join" "math/rand" ) @@ -139,7 +138,7 @@ func (c *EigenDAClient) Close() error { relayClientErr := c.relayClient.Close() // TODO: this is using join, since there will be more subcomponents requiring closing after adding PUT functionality - return join.Join(relayClientErr) + return errors.Join(relayClientErr) } // createCodec creates the codec based on client config values From cf1cd80a302775b31e63b5bc061125648d8f0933 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:23:55 -0500 Subject: [PATCH 10/18] Make updates required by upstream changes Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 5 +- api/clients/v2/eigenda_client.go | 14 ++--- .../v2/{ => test}/eigenda_client_test.go | 61 ++++++++++--------- 3 files changed, 42 insertions(+), 38 deletions(-) rename api/clients/v2/{ => test}/eigenda_client_test.go (89%) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index 10318508b..725fd9d13 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -1,8 +1,9 @@ -package v2 +package clients import ( - "github.com/Layr-Labs/eigenda/api/clients/codecs" "time" + + "github.com/Layr-Labs/eigenda/api/clients/codecs" ) // VerificationMode is an enum that represents the different ways that a blob may be encoded/decoded between diff --git a/api/clients/v2/eigenda_client.go b/api/clients/v2/eigenda_client.go index b256f509f..ddb0f56f0 100644 --- a/api/clients/v2/eigenda_client.go +++ b/api/clients/v2/eigenda_client.go @@ -1,14 +1,14 @@ -package v2 +package clients import ( "context" "errors" "fmt" - "github.com/Layr-Labs/eigenda/api/clients" + "math/rand" + "github.com/Layr-Labs/eigenda/api/clients/codecs" core "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigensdk-go/logging" - "math/rand" ) // EigenDAClient provides the ability to get blobs from the relay subsystem, and to send new blobs to the disperser. @@ -20,16 +20,16 @@ type EigenDAClient struct { random *rand.Rand config *EigenDAClientConfig codec codecs.BlobCodec - relayClient clients.RelayClient + relayClient RelayClient } // BuildEigenDAClient builds an EigenDAClient from config structs. func BuildEigenDAClient( log logging.Logger, config *EigenDAClientConfig, - relayClientConfig *clients.RelayClientConfig) (*EigenDAClient, error) { + relayClientConfig *RelayClientConfig) (*EigenDAClient, error) { - relayClient, err := clients.NewRelayClient(relayClientConfig, log) + relayClient, err := NewRelayClient(relayClientConfig, log) if err != nil { return nil, fmt.Errorf("new relay client: %w", err) } @@ -47,7 +47,7 @@ func NewEigenDAClient( log logging.Logger, random *rand.Rand, config *EigenDAClientConfig, - relayClient clients.RelayClient, + relayClient RelayClient, codec codecs.BlobCodec) (*EigenDAClient, error) { return &EigenDAClient{ diff --git a/api/clients/v2/eigenda_client_test.go b/api/clients/v2/test/eigenda_client_test.go similarity index 89% rename from api/clients/v2/eigenda_client_test.go rename to api/clients/v2/test/eigenda_client_test.go index f3b9d62b4..d70c3dd64 100644 --- a/api/clients/v2/eigenda_client_test.go +++ b/api/clients/v2/test/eigenda_client_test.go @@ -1,27 +1,28 @@ -package v2 +package test import ( "context" "errors" "fmt" - "github.com/Layr-Labs/eigenda/api/clients" + "testing" + "time" + "github.com/Layr-Labs/eigenda/api/clients/codecs" codecsmock "github.com/Layr-Labs/eigenda/api/clients/codecs/mock" - clientsmock "github.com/Layr-Labs/eigenda/api/clients/mock" + "github.com/Layr-Labs/eigenda/api/clients/v2" + clientsmock "github.com/Layr-Labs/eigenda/api/clients/v2/mock" testrandom "github.com/Layr-Labs/eigenda/common/testutils/random" core "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "testing" - "time" ) type ClientTester struct { - Random *testrandom.TestRandom - Client *EigenDAClient + Random *testrandom.TestRandom + Client *clients.EigenDAClient MockRelayClient *clientsmock.MockRelayClient - MockCodec *codecsmock.BlobCodec + MockCodec *codecsmock.BlobCodec } func (c *ClientTester) assertExpectations(t *testing.T) { @@ -32,16 +33,16 @@ func (c *ClientTester) assertExpectations(t *testing.T) { // buildClientTester sets up a client with mocks necessary for testing func buildClientTester(t *testing.T) ClientTester { logger := logging.NewNoopLogger() - clientConfig := &EigenDAClientConfig{ + clientConfig := &clients.EigenDAClientConfig{ RelayTimeout: 50 * time.Millisecond, } mockRelayClient := clientsmock.MockRelayClient{} mockCodec := codecsmock.BlobCodec{} - random := testrandom.NewTestRandom() + random := testrandom.NewTestRandom(t) - client, err := NewEigenDAClient( + client, err := clients.NewEigenDAClient( logger, random.Rand, clientConfig, @@ -63,8 +64,8 @@ func buildClientTester(t *testing.T) ClientTester { func TestGetBlobSuccess(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tester.Random.RandomBytes(32)) - blobBytes := tester.Random.RandomBytes(100) + blobKey := core.BlobKey(tester.Random.Bytes(32)) + blobBytes := tester.Random.Bytes(100) relayKeys := make([]core.RelayKey, 1) relayKeys[0] = tester.Random.Uint32() @@ -73,7 +74,7 @@ func TestGetBlobSuccess(t *testing.T) { } tester.MockRelayClient.On("GetBlob", mock.Anything, relayKeys[0], blobKey).Return(blobBytes, nil).Once() - tester.MockCodec.On("DecodeBlob", blobBytes).Return(tester.Random.RandomBytes(50), nil).Once() + tester.MockCodec.On("DecodeBlob", blobBytes).Return(tester.Random.Bytes(50), nil).Once() blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) @@ -87,7 +88,7 @@ func TestGetBlobSuccess(t *testing.T) { func TestRelayCallTimeout(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tester.Random.RandomBytes(32)) + blobKey := core.BlobKey(tester.Random.Bytes(32)) relayKeys := make([]core.RelayKey, 1) relayKeys[0] = tester.Random.Uint32() @@ -139,8 +140,8 @@ func TestRelayCallTimeout(t *testing.T) { func TestRandomRelayRetries(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tester.Random.RandomBytes(32)) - blobBytes := tester.Random.RandomBytes(100) + blobKey := core.BlobKey(tester.Random.Bytes(32)) + blobBytes := tester.Random.Bytes(100) relayCount := 100 relayKeys := make([]core.RelayKey, relayCount) @@ -162,7 +163,7 @@ func TestRandomRelayRetries(t *testing.T) { failedCallCount++ }) tester.MockRelayClient.On("GetBlob", mock.Anything, mock.MatchedBy(onlineKeyMatcher), blobKey).Return(blobBytes, nil) - tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.RandomBytes(50), nil) + tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.Bytes(50), nil) // keep track of how many tries various blob retrievals require // this allows us to assert that there is variability, i.e. that relay call order is actually random @@ -187,7 +188,7 @@ func TestRandomRelayRetries(t *testing.T) { func TestNoRelayResponse(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tester.Random.RandomBytes(32)) + blobKey := core.BlobKey(tester.Random.Bytes(32)) relayCount := 10 relayKeys := make([]core.RelayKey, relayCount) @@ -211,7 +212,7 @@ func TestNoRelayResponse(t *testing.T) { func TestNoRelaysInCert(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tester.Random.RandomBytes(32)) + blobKey := core.BlobKey(tester.Random.Bytes(32)) // cert has no listed relay keys blobCert := core.BlobCertificate{ @@ -229,7 +230,7 @@ func TestNoRelaysInCert(t *testing.T) { func TestGetBlobReturns0Len(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tester.Random.RandomBytes(32)) + blobKey := core.BlobKey(tester.Random.Bytes(32)) relayCount := 10 relayKeys := make([]core.RelayKey, relayCount) @@ -243,9 +244,11 @@ func TestGetBlobReturns0Len(t *testing.T) { // the first GetBlob will return a 0 len blob tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return([]byte{}, nil).Once() // the second call will return random bytes - tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tester.Random.RandomBytes(100), nil).Once() + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return( + tester.Random.Bytes(100), + nil).Once() - tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.RandomBytes(50), nil) + tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.Bytes(50), nil) // the call to the first relay will fail with a 0 len blob returned. the call to the second relay will succeed blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) @@ -259,7 +262,7 @@ func TestGetBlobReturns0Len(t *testing.T) { func TestFailedDecoding(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tester.Random.RandomBytes(32)) + blobKey := core.BlobKey(tester.Random.Bytes(32)) relayCount := 10 relayKeys := make([]core.RelayKey, relayCount) @@ -270,10 +273,10 @@ func TestFailedDecoding(t *testing.T) { RelayKeys: relayKeys, } - tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tester.Random.RandomBytes(100), nil) + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tester.Random.Bytes(100), nil) tester.MockCodec.On("DecodeBlob", mock.Anything).Return(nil, fmt.Errorf("decode failed")).Once() - tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.RandomBytes(50), nil).Once() + tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.Bytes(50), nil).Once() // decoding will fail the first time, but succeed the second time blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) @@ -318,9 +321,9 @@ func TestGetCodec(t *testing.T) { // TestBuilder tests that the method that builds the client from config doesn't throw any obvious errors func TestBuilder(t *testing.T) { - clientConfig := &EigenDAClientConfig{ + clientConfig := &clients.EigenDAClientConfig{ BlobEncodingVersion: codecs.DefaultBlobEncoding, - PointVerificationMode: IFFT, + PointVerificationMode: clients.IFFT, RelayTimeout: 500 * time.Millisecond, } @@ -332,7 +335,7 @@ func TestBuilder(t *testing.T) { UseSecureGrpcFlag: true, } - client, err := BuildEigenDAClient( + client, err := clients.BuildEigenDAClient( logging.NewNoopLogger(), clientConfig, relayClientConfig) From 53893d8dce430f0931ec672ba6ee035beb20c13c Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 13 Jan 2025 11:49:22 -0500 Subject: [PATCH 11/18] Update how FFT and IFFT are referred to Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/polynomial_form.go | 12 ++ api/clients/v2/config.go | 24 +-- api/clients/v2/eigenda_client.go | 27 ++- api/clients/v2/test/eigenda_client_test.go | 156 ++++++++---------- .../v2/verification/commitment_utils.go | 14 +- .../v2/verification/commitment_utils_test.go | 15 +- 6 files changed, 124 insertions(+), 124 deletions(-) create mode 100644 api/clients/codecs/polynomial_form.go diff --git a/api/clients/codecs/polynomial_form.go b/api/clients/codecs/polynomial_form.go new file mode 100644 index 000000000..8234ba6ee --- /dev/null +++ b/api/clients/codecs/polynomial_form.go @@ -0,0 +1,12 @@ +package codecs + +// PolynomialForm is an enum that represents the different ways that a blob polynomial may be represented +type PolynomialForm uint + +const ( + // Eval is short for "evaluation form". The field elements represent the evaluation at the polynomial's expanded + // roots of unity + Eval PolynomialForm = iota + // Coeff is short for "coefficient form". The field elements represent the coefficients of the polynomial + Coeff +) \ No newline at end of file diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index 725fd9d13..a36d08aa5 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -6,28 +6,20 @@ import ( "github.com/Layr-Labs/eigenda/api/clients/codecs" ) -// VerificationMode is an enum that represents the different ways that a blob may be encoded/decoded between -// the client and the disperser. -type VerificationMode uint - -const ( - // TODO: write good docs here for IFFT and NoIFFT (I need to update my understanding to be able to write this) - IFFT VerificationMode = iota - NoIFFT -) - // EigenDAClientConfig contains configuration values for EigenDAClient type EigenDAClientConfig struct { // The blob encoding version to use when writing and reading blobs BlobEncodingVersion codecs.BlobEncodingVersion - // If PointVerificationMode is IFFT, then the client codec will do an IFFT on blobs before they are dispersed, and - // will do an FFT on blobs after receiving them. This makes it possible to open points on the KZG commitment to prove - // that the field elements correspond to the commitment. + // BlobPolynomialForm is the form that the blob polynomial is commited to and dispersed in, as well as the form the + // blob polynomial will be received in from the relay. // - // If PointVerificationMode is NoIFFT, the blob must be supplied in its entirety, to perform a verification - // that any part of the data matches the KZG commitment. - PointVerificationMode VerificationMode + // The chosen form dictates how the KZG commitment made to the blob can be used. If the polynomial is in Coeff form + // when committed to, then it will be possible to open points on the KZG commitment to prove that the field elements + // correspond to the commitment. If the polynomial is in Eval form when committed to, then it will not be possible + // to create a commitment opening: the blob will need to be supplied in its entirety to perform a verification that + // any part of the data matches the KZG commitment. + BlobPolynomialForm codecs.PolynomialForm // The timeout duration for relay calls RelayTimeout time.Duration diff --git a/api/clients/v2/eigenda_client.go b/api/clients/v2/eigenda_client.go index ddb0f56f0..5a8dbc51c 100644 --- a/api/clients/v2/eigenda_client.go +++ b/api/clients/v2/eigenda_client.go @@ -9,6 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/api/clients/codecs" core "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigensdk-go/logging" + "github.com/consensys/gnark-crypto/ecc/bn254" ) // EigenDAClient provides the ability to get blobs from the relay subsystem, and to send new blobs to the disperser. @@ -21,6 +22,7 @@ type EigenDAClient struct { config *EigenDAClientConfig codec codecs.BlobCodec relayClient RelayClient + g1Srs []bn254.G1Affine } // BuildEigenDAClient builds an EigenDAClient from config structs. @@ -59,17 +61,17 @@ func NewEigenDAClient( }, nil } -// GetBlob iteratively attempts to retrieve a given blob with key blobKey from the relays listed in the blobCertificate. +// GetBlob iteratively attempts to retrieve a given blob with key blobKey from supplied relays. // // The relays are attempted in random order. // -// The returned blob is decoded. +// IMPORTANT: The returned blob is NOT decoded and NOT verified func (c *EigenDAClient) GetBlob( ctx context.Context, blobKey core.BlobKey, - blobCertificate core.BlobCertificate) ([]byte, error) { + relayKeys []core.RelayKey) ([]byte, error) { - relayKeyCount := len(blobCertificate.RelayKeys) + relayKeyCount := len(relayKeys) if relayKeyCount == 0 { return nil, errors.New("relay key count is zero") @@ -82,7 +84,7 @@ func (c *EigenDAClient) GetBlob( // iterate over relays in random order, until we are able to get the blob from someone for _, val := range indices { - relayKey := blobCertificate.RelayKeys[val] + relayKey := relayKeys[val] data, err := c.getBlobWithTimeout(ctx, relayKey, blobKey) @@ -111,6 +113,8 @@ func (c *EigenDAClient) GetBlob( return nil, fmt.Errorf("unable to retrieve blob from any relay. relay count: %d", relayKeyCount) } +func (c *EigenDAClient) VerifyBlobCommitment(blob []byte) + // getBlobWithTimeout attempts to get a blob from a given relay, and times out based on config.RelayTimeout func (c *EigenDAClient) getBlobWithTimeout( ctx context.Context, @@ -148,12 +152,17 @@ func createCodec(config *EigenDAClientConfig) (codecs.BlobCodec, error) { return nil, fmt.Errorf("create low level codec: %w", err) } - switch config.PointVerificationMode { - case NoIFFT: + switch config.BlobPolynomialForm { + case codecs.Eval: + // a blob polynomial is already in Eval form after being encoded. Therefore, we use the NoIFFTCodec, which + // doesn't do any further conversion. return codecs.NewNoIFFTCodec(lowLevelCodec), nil - case IFFT: + case codecs.Coeff: + // a blob polynomial starts in Eval form after being encoded. Therefore, we use the IFFT codec to transform + // the blob into Coeff form after initial encoding. This codec also transforms the Coeff form received from the + // relay back into Eval form when decoding. return codecs.NewIFFTCodec(lowLevelCodec), nil default: - return nil, fmt.Errorf("unsupported point verification mode: %d", config.PointVerificationMode) + return nil, fmt.Errorf("unsupported polynomial form: %d", config.BlobPolynomialForm) } } diff --git a/api/clients/v2/test/eigenda_client_test.go b/api/clients/v2/test/eigenda_client_test.go index d70c3dd64..f65012c0e 100644 --- a/api/clients/v2/test/eigenda_client_test.go +++ b/api/clients/v2/test/eigenda_client_test.go @@ -14,18 +14,18 @@ import ( testrandom "github.com/Layr-Labs/eigenda/common/testutils/random" core "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigensdk-go/logging" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) type ClientTester struct { - Random *testrandom.TestRandom - Client *clients.EigenDAClient + Random *testrandom.TestRandom + Client *clients.EigenDAClient MockRelayClient *clientsmock.MockRelayClient - MockCodec *codecsmock.BlobCodec + MockCodec *codecsmock.BlobCodec } -func (c *ClientTester) assertExpectations(t *testing.T) { +func (c *ClientTester) requireExpectations(t *testing.T) { c.MockRelayClient.AssertExpectations(t) c.MockCodec.AssertExpectations(t) } @@ -49,8 +49,8 @@ func buildClientTester(t *testing.T) ClientTester { &mockRelayClient, &mockCodec) - assert.NotNil(t, client) - assert.Nil(t, err) + require.NotNil(t, client) + require.NoError(t, err) return ClientTester{ Random: random, @@ -69,19 +69,16 @@ func TestGetBlobSuccess(t *testing.T) { relayKeys := make([]core.RelayKey, 1) relayKeys[0] = tester.Random.Uint32() - blobCert := core.BlobCertificate{ - RelayKeys: relayKeys, - } tester.MockRelayClient.On("GetBlob", mock.Anything, relayKeys[0], blobKey).Return(blobBytes, nil).Once() tester.MockCodec.On("DecodeBlob", blobBytes).Return(tester.Random.Bytes(50), nil).Once() - blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, relayKeys) - assert.NotNil(t, blob) - assert.Nil(t, err) + require.NotNil(t, blob) + require.NoError(t, err) - tester.assertExpectations(t) + tester.requireExpectations(t) } // TestRelayCallTimeout verifies that calls to the relay timeout after the expected duration @@ -92,9 +89,6 @@ func TestRelayCallTimeout(t *testing.T) { relayKeys := make([]core.RelayKey, 1) relayKeys[0] = tester.Random.Uint32() - blobCert := core.BlobCertificate{ - RelayKeys: relayKeys, - } // the timeout should occur before the panic has a chance to be triggered tester.MockRelayClient.On("GetBlob", mock.Anything, relayKeys[0], blobKey).Return( @@ -123,19 +117,21 @@ func TestRelayCallTimeout(t *testing.T) { panic("call should not have timed out") } }) - - assert.NotPanics(t, func() { - _, _ = tester.Client.GetBlob(context.Background(), blobKey, blobCert) - }) - assert.Panics(t, func() { - _, _ = tester.Client.GetBlob(context.Background(), blobKey, blobCert) - }) + require.NotPanics( + t, func() { + _, _ = tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + }) - tester.assertExpectations(t) + require.Panics( + t, func() { + _, _ = tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + }) + + tester.requireExpectations(t) } -// TestRandomRelayRetries verifies correct behavior when some relays from the certificate do not respond with the blob, +// TestRandomRelayRetries verifies correct behavior when some relays do not respond with the blob, // requiring the client to retry with other relays. func TestRandomRelayRetries(t *testing.T) { tester := buildClientTester(t) @@ -148,43 +144,45 @@ func TestRandomRelayRetries(t *testing.T) { for i := 0; i < relayCount; i++ { relayKeys[i] = tester.Random.Uint32() } - blobCert := core.BlobCertificate{ - RelayKeys: relayKeys, - } // for this test, only a single relay is online - // we will be asserting that it takes a different amount of retries to dial this relay, since the array of relay keys to try is randomized + // we will be requireing that it takes a different amount of retries to dial this relay, since the array of relay keys to try is randomized onlineRelayKey := relayKeys[tester.Random.Intn(len(relayKeys))] offlineKeyMatcher := func(relayKey core.RelayKey) bool { return relayKey != onlineRelayKey } onlineKeyMatcher := func(relayKey core.RelayKey) bool { return relayKey == onlineRelayKey } var failedCallCount int - tester.MockRelayClient.On("GetBlob", mock.Anything, mock.MatchedBy(offlineKeyMatcher), blobKey).Return(nil, fmt.Errorf("offline relay")).Run(func(args mock.Arguments) { - failedCallCount++ - }) - tester.MockRelayClient.On("GetBlob", mock.Anything, mock.MatchedBy(onlineKeyMatcher), blobKey).Return(blobBytes, nil) + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.MatchedBy(offlineKeyMatcher), blobKey).Return( + nil, + fmt.Errorf("offline relay")).Run( + func(args mock.Arguments) { + failedCallCount++ + }) + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.MatchedBy(onlineKeyMatcher), blobKey).Return( + blobBytes, + nil) tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.Bytes(50), nil) // keep track of how many tries various blob retrievals require - // this allows us to assert that there is variability, i.e. that relay call order is actually random + // this allows us to require that there is variability, i.e. that relay call order is actually random requiredTries := map[int]bool{} for i := 0; i < relayCount; i++ { failedCallCount = 0 - blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) - assert.NotNil(t, blob) - assert.Nil(t, err) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + require.NotNil(t, blob) + require.NoError(t, err) requiredTries[failedCallCount] = true } - // with 100 random tries, with possible values between 1 and 100, we can very confidently assert that there are at least 10 unique values - assert.Greater(t, len(requiredTries), 10) + // with 100 random tries, with possible values between 1 and 100, we can very confidently require that there are at least 10 unique values + require.Greater(t, len(requiredTries), 10) - tester.assertExpectations(t) + tester.requireExpectations(t) } -// TestNoRelayResponse tests functionality when none of the relays listed in the blob certificate respond +// TestNoRelayResponse tests functionality when none of the relays respond func TestNoRelayResponse(t *testing.T) { tester := buildClientTester(t) @@ -195,35 +193,27 @@ func TestNoRelayResponse(t *testing.T) { for i := 0; i < relayCount; i++ { relayKeys[i] = tester.Random.Uint32() } - blobCert := core.BlobCertificate{ - RelayKeys: relayKeys, - } tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(nil, fmt.Errorf("offline relay")) - blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) - assert.Nil(t, blob) - assert.NotNil(t, err) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + require.Nil(t, blob) + require.NotNil(t, err) - tester.assertExpectations(t) + tester.requireExpectations(t) } -// TestNoRelaysInCert tests that having no relay keys in the cert is handled gracefully -func TestNoRelaysInCert(t *testing.T) { +// TestNoRelays tests that having no relay keys is handled gracefully +func TestNoRelays(t *testing.T) { tester := buildClientTester(t) blobKey := core.BlobKey(tester.Random.Bytes(32)) - // cert has no listed relay keys - blobCert := core.BlobCertificate{ - RelayKeys: []core.RelayKey{}, - } - - blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) - assert.Nil(t, blob) - assert.NotNil(t, err) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, []core.RelayKey{}) + require.Nil(t, blob) + require.NotNil(t, err) - tester.assertExpectations(t) + tester.requireExpectations(t) } // TestGetBlobReturns0Len verifies that a 0 length blob returned from a relay is handled gracefully, and that the client retries after such a failure @@ -237,9 +227,6 @@ func TestGetBlobReturns0Len(t *testing.T) { for i := 0; i < relayCount; i++ { relayKeys[i] = tester.Random.Uint32() } - blobCert := core.BlobCertificate{ - RelayKeys: relayKeys, - } // the first GetBlob will return a 0 len blob tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return([]byte{}, nil).Once() @@ -251,11 +238,11 @@ func TestGetBlobReturns0Len(t *testing.T) { tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.Bytes(50), nil) // the call to the first relay will fail with a 0 len blob returned. the call to the second relay will succeed - blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) - assert.NotNil(t, blob) - assert.Nil(t, err) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + require.NotNil(t, blob) + require.NoError(t, err) - tester.assertExpectations(t) + tester.requireExpectations(t) } // TestFailedDecoding verifies that a failed blob decode is handled gracefully, and that the client retries after such a failure @@ -269,9 +256,6 @@ func TestFailedDecoding(t *testing.T) { for i := 0; i < relayCount; i++ { relayKeys[i] = tester.Random.Uint32() } - blobCert := core.BlobCertificate{ - RelayKeys: relayKeys, - } tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tester.Random.Bytes(100), nil) @@ -279,11 +263,11 @@ func TestFailedDecoding(t *testing.T) { tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.Bytes(50), nil).Once() // decoding will fail the first time, but succeed the second time - blob, err := tester.Client.GetBlob(context.Background(), blobKey, blobCert) - assert.NotNil(t, blob) - assert.Nil(t, err) + blob, err := tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + require.NotNil(t, blob) + require.NoError(t, err) - tester.assertExpectations(t) + tester.requireExpectations(t) } // TestErrorFreeClose tests the happy case, where none of the internal closes yield an error @@ -293,9 +277,9 @@ func TestErrorFreeClose(t *testing.T) { tester.MockRelayClient.On("Close").Return(nil).Once() err := tester.Client.Close() - assert.Nil(t, err) + require.NoError(t, err) - tester.assertExpectations(t) + tester.requireExpectations(t) } // TestErrorClose tests what happens when subcomponents throw errors when being closed @@ -305,26 +289,26 @@ func TestErrorClose(t *testing.T) { tester.MockRelayClient.On("Close").Return(fmt.Errorf("close failed")).Once() err := tester.Client.Close() - assert.NotNil(t, err) + require.NotNil(t, err) - tester.assertExpectations(t) + tester.requireExpectations(t) } // TestGetCodec checks that the codec used in construction is returned by GetCodec func TestGetCodec(t *testing.T) { tester := buildClientTester(t) - assert.Equal(t, tester.MockCodec, tester.Client.GetCodec()) + require.Equal(t, tester.MockCodec, tester.Client.GetCodec()) - tester.assertExpectations(t) + tester.requireExpectations(t) } // TestBuilder tests that the method that builds the client from config doesn't throw any obvious errors func TestBuilder(t *testing.T) { clientConfig := &clients.EigenDAClientConfig{ - BlobEncodingVersion: codecs.DefaultBlobEncoding, - PointVerificationMode: clients.IFFT, - RelayTimeout: 500 * time.Millisecond, + BlobEncodingVersion: codecs.DefaultBlobEncoding, + BlobPolynomialForm: codecs.Coeff, + RelayTimeout: 500 * time.Millisecond, } sockets := make(map[core.RelayKey]string) @@ -340,8 +324,8 @@ func TestBuilder(t *testing.T) { clientConfig, relayClientConfig) - assert.NotNil(t, client) - assert.Nil(t, err) + require.NotNil(t, client) + require.NoError(t, err) - assert.NotNil(t, client.GetCodec()) + require.NotNil(t, client.GetCodec()) } diff --git a/api/clients/v2/verification/commitment_utils.go b/api/clients/v2/verification/commitment_utils.go index 3a488082b..ce79262dd 100644 --- a/api/clients/v2/verification/commitment_utils.go +++ b/api/clients/v2/verification/commitment_utils.go @@ -2,6 +2,7 @@ package verification import ( "fmt" + "github.com/Layr-Labs/eigenda/encoding" "github.com/consensys/gnark-crypto/ecc" "github.com/consensys/gnark-crypto/ecc/bn254" @@ -36,23 +37,22 @@ func GenerateBlobCommitment( } // GenerateAndCompareBlobCommitment generates the kzg-bn254 commitment of the blob, and compares it with a claimed -// commitment. An error is returned if there is a problem generating the commitment, or if the comparison fails. +// commitment. An error is returned if there is a problem generating the commitment. True is returned if the commitment +// is successfully generated, and is equal to the claimed commitment, otherwise false. func GenerateAndCompareBlobCommitment( g1Srs []bn254.G1Affine, blobBytes []byte, - claimedCommitment *encoding.G1Commitment) error { + claimedCommitment *encoding.G1Commitment) (bool, error) { computedCommitment, err := GenerateBlobCommitment(g1Srs, blobBytes) if err != nil { - return fmt.Errorf("compute commitment: %w", err) + return false, fmt.Errorf("compute commitment: %w", err) } if claimedCommitment.X.Equal(&computedCommitment.X) && claimedCommitment.Y.Equal(&computedCommitment.Y) { - return nil + return true, nil } - return fmt.Errorf( - "commitment field elements do not match. computed commitment: (x: %x, y: %x), claimed commitment (x: %x, y: %x)", - computedCommitment.X, computedCommitment.Y, claimedCommitment.X, claimedCommitment.Y) + return false, nil } diff --git a/api/clients/v2/verification/commitment_utils_test.go b/api/clients/v2/verification/commitment_utils_test.go index 8e9927e0c..2c5fe499a 100644 --- a/api/clients/v2/verification/commitment_utils_test.go +++ b/api/clients/v2/verification/commitment_utils_test.go @@ -1,13 +1,14 @@ package verification import ( + "math" + "runtime" + "testing" + "github.com/Layr-Labs/eigenda/common/testutils/random" "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/stretchr/testify/require" - "math" - "runtime" - "testing" ) const g1Path = "../../../../inabox/resources/kzg/g1.point" @@ -42,10 +43,11 @@ func TestComputeAndCompareKzgCommitmentSuccess(t *testing.T) { require.NoError(t, err) // make sure the commitment verifies correctly - err = GenerateAndCompareBlobCommitment( + result, err := GenerateAndCompareBlobCommitment( g1Srs, randomBytes, commitment) + require.True(t, result) require.NoError(t, err) } @@ -65,11 +67,12 @@ func TestComputeAndCompareKzgCommitmentFailure(t *testing.T) { // randomly modify the bytes, and make sure the commitment verification fails randomlyModifyBytes(testRandom, randomBytes) - err = GenerateAndCompareBlobCommitment( + result, err := GenerateAndCompareBlobCommitment( g1Srs, randomBytes, commitment) - require.NotNil(t, err) + require.False(t, result) + require.NoError(t, err) } func TestGenerateBlobCommitmentEquality(t *testing.T) { From 0373dd75324b7079c745dbf7178dd676e3becf0c Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:39:08 -0500 Subject: [PATCH 12/18] Implement GetPayload Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/eigenda_client.go | 52 +++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/api/clients/v2/eigenda_client.go b/api/clients/v2/eigenda_client.go index 5a8dbc51c..120410afa 100644 --- a/api/clients/v2/eigenda_client.go +++ b/api/clients/v2/eigenda_client.go @@ -7,6 +7,7 @@ import ( "math/rand" "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/Layr-Labs/eigenda/api/clients/v2/verification" core "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/consensys/gnark-crypto/ecc/bn254" @@ -113,7 +114,56 @@ func (c *EigenDAClient) GetBlob( return nil, fmt.Errorf("unable to retrieve blob from any relay. relay count: %d", relayKeyCount) } -func (c *EigenDAClient) VerifyBlobCommitment(blob []byte) +// GetPayload iteratively attempts to fetch a given blob with key blobKey from relays that have it, as claimed by the +// blob certificate. The relays are attempted in random order. +// +// If the blob is successfully retrieved, then the blob is verified. If the verification succeeds, the blob is decoded +// to yield the payload (the original user data), and the payload is returned. +func (c *EigenDAClient) GetPayload( + ctx context.Context, + blobKey core.BlobKey, + blobCertificate *core.BlobCertificate) ([]byte, error) { + + blob, err := c.GetBlob(ctx, blobKey, blobCertificate.RelayKeys) + if err != nil { + return nil, fmt.Errorf("get blob %v: %w", blobKey, err) + } + + blobCommitments := blobCertificate.BlobHeader.BlobCommitments + + // TODO: in the future, this will be optimized to use fiat shamir transformation for verification, rather than + // regenerating the commitment: https://github.com/Layr-Labs/eigenda/issues/1037 + valid, err := verification.GenerateAndCompareBlobCommitment( + c.g1Srs, + blob, + blobCommitments.Commitment) + if err != nil { + return nil, fmt.Errorf("generate and compare blob commitment for blob %v: %w", blobKey, err) + } + + if !valid { + return nil, fmt.Errorf("blob commitment for blob %v is invalid", blobKey) + } + + // checking that the length returned by the relay matches the claimed length is sufficient here: it isn't necessary + // to verify the length proof itself, since this will have been done by DA nodes prior to signing for availability. + if uint(len(blob)) != blobCommitments.Length { + return nil, fmt.Errorf( + "blob %v length (%d) doesn't match length claimed in blob commitments (%d)", + blobKey, + len(blob), + blobCommitments.Length) + } + + // TODO: call verifyBlobV2 + + payload, err := c.codec.DecodeBlob(blob) + if err != nil { + return nil, fmt.Errorf("decode blob %v: %w", blobKey, err) + } + + return payload, nil +} // getBlobWithTimeout attempts to get a blob from a given relay, and times out based on config.RelayTimeout func (c *EigenDAClient) getBlobWithTimeout( From 826a026101f96ec68ccc01462cfd06dad12912ce Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:35:11 -0500 Subject: [PATCH 13/18] Remove GetBlob, leaving only GetPayload Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/eigenda_client.go | 93 +++++++++------ api/clients/v2/test/eigenda_client_test.go | 132 ++++++++++++++------- 2 files changed, 142 insertions(+), 83 deletions(-) diff --git a/api/clients/v2/eigenda_client.go b/api/clients/v2/eigenda_client.go index 120410afa..ef87e18f4 100644 --- a/api/clients/v2/eigenda_client.go +++ b/api/clients/v2/eigenda_client.go @@ -30,7 +30,8 @@ type EigenDAClient struct { func BuildEigenDAClient( log logging.Logger, config *EigenDAClientConfig, - relayClientConfig *RelayClientConfig) (*EigenDAClient, error) { + relayClientConfig *RelayClientConfig, + g1Srs []bn254.G1Affine) (*EigenDAClient, error) { relayClient, err := NewRelayClient(relayClientConfig, log) if err != nil { @@ -42,7 +43,7 @@ func BuildEigenDAClient( return nil, err } - return NewEigenDAClient(log, rand.New(rand.NewSource(rand.Int63())), config, relayClient, codec) + return NewEigenDAClient(log, rand.New(rand.NewSource(rand.Int63())), config, relayClient, codec, g1Srs) } // NewEigenDAClient assembles an EigenDAClient from subcomponents that have already been constructed and initialized. @@ -51,7 +52,8 @@ func NewEigenDAClient( random *rand.Rand, config *EigenDAClientConfig, relayClient RelayClient, - codec codecs.BlobCodec) (*EigenDAClient, error) { + codec codecs.BlobCodec, + g1Srs []bn254.G1Affine) (*EigenDAClient, error) { return &EigenDAClient{ log: log, @@ -59,19 +61,21 @@ func NewEigenDAClient( config: config, codec: codec, relayClient: relayClient, + g1Srs: g1Srs, }, nil } -// GetBlob iteratively attempts to retrieve a given blob with key blobKey from supplied relays. -// -// The relays are attempted in random order. +// GetPayload iteratively attempts to fetch a given blob with key blobKey from relays that have it, as claimed by the +// blob certificate. The relays are attempted in random order. // -// IMPORTANT: The returned blob is NOT decoded and NOT verified -func (c *EigenDAClient) GetBlob( +// If the blob is successfully retrieved, then the blob is verified. If the verification succeeds, the blob is decoded +// to yield the payload (the original user data), and the payload is returned. +func (c *EigenDAClient) GetPayload( ctx context.Context, blobKey core.BlobKey, - relayKeys []core.RelayKey) ([]byte, error) { + blobCertificate *core.BlobCertificate) ([]byte, error) { + relayKeys := blobCertificate.RelayKeys relayKeyCount := len(relayKeys) if relayKeyCount == 0 { @@ -87,7 +91,7 @@ func (c *EigenDAClient) GetBlob( for _, val := range indices { relayKey := relayKeys[val] - data, err := c.getBlobWithTimeout(ctx, relayKey, blobKey) + blob, err := c.getBlobWithTimeout(ctx, relayKey, blobKey) // if GetBlob returned an error, try calling a different relay if err != nil { @@ -95,38 +99,38 @@ func (c *EigenDAClient) GetBlob( continue } - // An honest relay should never send an empty blob - if len(data) == 0 { - c.log.Warn("blob received from relay had length 0", "blobKey", blobKey, "relayKey", relayKey) + // An honest relay should never send a blob which doesn't verify + if !c.verifyBlobFromRelay(blobKey, relayKey, blob, blobCertificate) { + // specifics are logged in verifyBlobFromRelay continue } - // An honest relay should never send a blob which cannot be decoded - decodedData, err := c.codec.DecodeBlob(data) + // An honest relay should never send a blob which cannot be decoded into a payload + payload, err := c.codec.DecodeBlob(blob) if err != nil { c.log.Warn("error decoding blob from relay", "blobKey", blobKey, "relayKey", relayKey, "error", err) continue } - return decodedData, nil + return payload, nil } - return nil, fmt.Errorf("unable to retrieve blob from any relay. relay count: %d", relayKeyCount) + return nil, fmt.Errorf("unable to retrieve blob %v from any relay. relay count: %d", blobKey, relayKeyCount) } -// GetPayload iteratively attempts to fetch a given blob with key blobKey from relays that have it, as claimed by the -// blob certificate. The relays are attempted in random order. +// verifyBlobFromRelay performs the necessary verification after having retrieved a blob from a relay. // -// If the blob is successfully retrieved, then the blob is verified. If the verification succeeds, the blob is decoded -// to yield the payload (the original user data), and the payload is returned. -func (c *EigenDAClient) GetPayload( - ctx context.Context, +// If all verifications succeed, the method returns true. Otherwise, it logs a warning and returns false. +func (c *EigenDAClient) verifyBlobFromRelay( blobKey core.BlobKey, - blobCertificate *core.BlobCertificate) ([]byte, error) { + relayKey core.RelayKey, + blob []byte, + blobCertificate *core.BlobCertificate) bool { - blob, err := c.GetBlob(ctx, blobKey, blobCertificate.RelayKeys) - if err != nil { - return nil, fmt.Errorf("get blob %v: %w", blobKey, err) + // An honest relay should never send an empty blob + if len(blob) == 0 { + c.log.Warn("blob received from relay had length 0", "blobKey", blobKey, "relayKey", relayKey) + return false } blobCommitments := blobCertificate.BlobHeader.BlobCommitments @@ -138,31 +142,46 @@ func (c *EigenDAClient) GetPayload( blob, blobCommitments.Commitment) if err != nil { - return nil, fmt.Errorf("generate and compare blob commitment for blob %v: %w", blobKey, err) + c.log.Warn( + "error generating commitment from received blob", + "blobKey", + blobKey, + "relayKey", + relayKey, + "error", + err) + return false } if !valid { - return nil, fmt.Errorf("blob commitment for blob %v is invalid", blobKey) + c.log.Warn( + "blob commitment is invalid for received bytes", + "blobKey", + blobKey, + "relayKey", + relayKey) + return false } // checking that the length returned by the relay matches the claimed length is sufficient here: it isn't necessary // to verify the length proof itself, since this will have been done by DA nodes prior to signing for availability. if uint(len(blob)) != blobCommitments.Length { - return nil, fmt.Errorf( - "blob %v length (%d) doesn't match length claimed in blob commitments (%d)", + c.log.Warn( + "blob length doesn't match length claimed in blob commitments", + "blobKey", blobKey, + "relayKey", + relayKey, + "blobLength", len(blob), + "blobCommitments.Length", blobCommitments.Length) + return false } // TODO: call verifyBlobV2 - payload, err := c.codec.DecodeBlob(blob) - if err != nil { - return nil, fmt.Errorf("decode blob %v: %w", blobKey, err) - } - - return payload, nil + return true } // getBlobWithTimeout attempts to get a blob from a given relay, and times out based on config.RelayTimeout diff --git a/api/clients/v2/test/eigenda_client_test.go b/api/clients/v2/test/eigenda_client_test.go index f65012c0e..61f911657 100644 --- a/api/clients/v2/test/eigenda_client_test.go +++ b/api/clients/v2/test/eigenda_client_test.go @@ -4,30 +4,38 @@ import ( "context" "errors" "fmt" + "runtime" "testing" "time" "github.com/Layr-Labs/eigenda/api/clients/codecs" - codecsmock "github.com/Layr-Labs/eigenda/api/clients/codecs/mock" "github.com/Layr-Labs/eigenda/api/clients/v2" clientsmock "github.com/Layr-Labs/eigenda/api/clients/v2/mock" + "github.com/Layr-Labs/eigenda/api/clients/v2/verification" testrandom "github.com/Layr-Labs/eigenda/common/testutils/random" core "github.com/Layr-Labs/eigenda/core/v2" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigensdk-go/logging" + "github.com/consensys/gnark-crypto/ecc/bn254" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) +const g1Path = "../../../../inabox/resources/kzg/g1.point" +const payloadLength = 100 + type ClientTester struct { Random *testrandom.TestRandom Client *clients.EigenDAClient MockRelayClient *clientsmock.MockRelayClient - MockCodec *codecsmock.BlobCodec + Codec *codecs.DefaultBlobCodec + G1Srs []bn254.G1Affine } func (c *ClientTester) requireExpectations(t *testing.T) { c.MockRelayClient.AssertExpectations(t) - c.MockCodec.AssertExpectations(t) + // TODO: get rid of this } // buildClientTester sets up a client with mocks necessary for testing @@ -38,16 +46,21 @@ func buildClientTester(t *testing.T) ClientTester { } mockRelayClient := clientsmock.MockRelayClient{} - mockCodec := codecsmock.BlobCodec{} + codec := codecs.NewDefaultBlobCodec() random := testrandom.NewTestRandom(t) + g1Srs, err := kzg.ReadG1Points(g1Path, uint64(payloadLength), uint64(runtime.GOMAXPROCS(0))) + require.NotNil(t, g1Srs) + require.NoError(t, err) + client, err := clients.NewEigenDAClient( logger, random.Rand, clientConfig, &mockRelayClient, - &mockCodec) + &codec, + g1Srs) require.NotNil(t, client) require.NoError(t, err) @@ -56,26 +69,56 @@ func buildClientTester(t *testing.T) ClientTester { Random: random, Client: client, MockRelayClient: &mockRelayClient, - MockCodec: &mockCodec, + Codec: &codec, + G1Srs: g1Srs, } } -// TestGetBlobSuccess tests that a blob is received without error in the happy case -func TestGetBlobSuccess(t *testing.T) { - tester := buildClientTester(t) +func buildBlobAndCert( + t *testing.T, + tester ClientTester, + relayKeys []core.RelayKey) (core.BlobKey, []byte, *core.BlobCertificate) { blobKey := core.BlobKey(tester.Random.Bytes(32)) - blobBytes := tester.Random.Bytes(100) + payloadBytes := tester.Random.Bytes(payloadLength) + blobBytes, err := tester.Codec.EncodeBlob(payloadBytes) + require.NoError(t, err) + require.NotNil(t, blobBytes) + kzgCommitment, err := verification.GenerateBlobCommitment(tester.G1Srs, blobBytes) + require.NoError(t, err) + require.NotNil(t, kzgCommitment) + + commitments := encoding.BlobCommitments{ + Commitment: kzgCommitment, + Length: uint(len(blobBytes)), + } + + blobHeader := &core.BlobHeader{ + BlobCommitments: commitments, + } + + return blobKey, blobBytes, &core.BlobCertificate{ + RelayKeys: relayKeys, + BlobHeader: blobHeader, + } +} + +// TestGetPayloadSuccess tests that a blob is received without error in the happy case +func TestGetPayloadSuccess(t *testing.T) { + tester := buildClientTester(t) relayKeys := make([]core.RelayKey, 1) relayKeys[0] = tester.Random.Uint32() + blobKey, blobBytes, blobCert := buildBlobAndCert(t, tester, relayKeys) tester.MockRelayClient.On("GetBlob", mock.Anything, relayKeys[0], blobKey).Return(blobBytes, nil).Once() - tester.MockCodec.On("DecodeBlob", blobBytes).Return(tester.Random.Bytes(50), nil).Once() - blob, err := tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + payload, err := tester.Client.GetPayload( + context.Background(), + blobKey, + blobCert) - require.NotNil(t, blob) + require.NotNil(t, payload) require.NoError(t, err) tester.requireExpectations(t) @@ -84,11 +127,9 @@ func TestGetBlobSuccess(t *testing.T) { // TestRelayCallTimeout verifies that calls to the relay timeout after the expected duration func TestRelayCallTimeout(t *testing.T) { tester := buildClientTester(t) - - blobKey := core.BlobKey(tester.Random.Bytes(32)) - relayKeys := make([]core.RelayKey, 1) relayKeys[0] = tester.Random.Uint32() + blobKey, _, blobCert := buildBlobAndCert(t, tester, relayKeys) // the timeout should occur before the panic has a chance to be triggered tester.MockRelayClient.On("GetBlob", mock.Anything, relayKeys[0], blobKey).Return( @@ -120,12 +161,12 @@ func TestRelayCallTimeout(t *testing.T) { require.NotPanics( t, func() { - _, _ = tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + _, _ = tester.Client.GetPayload(context.Background(), blobKey, blobCert) }) require.Panics( t, func() { - _, _ = tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + _, _ = tester.Client.GetPayload(context.Background(), blobKey, blobCert) }) tester.requireExpectations(t) @@ -136,17 +177,15 @@ func TestRelayCallTimeout(t *testing.T) { func TestRandomRelayRetries(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tester.Random.Bytes(32)) - blobBytes := tester.Random.Bytes(100) - relayCount := 100 relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { relayKeys[i] = tester.Random.Uint32() } + blobKey, blobBytes, blobCert := buildBlobAndCert(t, tester, relayKeys) // for this test, only a single relay is online - // we will be requireing that it takes a different amount of retries to dial this relay, since the array of relay keys to try is randomized + // we will be requiring that it takes a different amount of retries to dial this relay, since the array of relay keys to try is randomized onlineRelayKey := relayKeys[tester.Random.Intn(len(relayKeys))] offlineKeyMatcher := func(relayKey core.RelayKey) bool { return relayKey != onlineRelayKey } @@ -161,7 +200,6 @@ func TestRandomRelayRetries(t *testing.T) { tester.MockRelayClient.On("GetBlob", mock.Anything, mock.MatchedBy(onlineKeyMatcher), blobKey).Return( blobBytes, nil) - tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.Bytes(50), nil) // keep track of how many tries various blob retrievals require // this allows us to require that there is variability, i.e. that relay call order is actually random @@ -169,7 +207,7 @@ func TestRandomRelayRetries(t *testing.T) { for i := 0; i < relayCount; i++ { failedCallCount = 0 - blob, err := tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + blob, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) require.NotNil(t, blob) require.NoError(t, err) @@ -186,17 +224,19 @@ func TestRandomRelayRetries(t *testing.T) { func TestNoRelayResponse(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tester.Random.Bytes(32)) - relayCount := 10 relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { relayKeys[i] = tester.Random.Uint32() } + blobKey, _, blobCert := buildBlobAndCert(t, tester, relayKeys) tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(nil, fmt.Errorf("offline relay")) - blob, err := tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + blob, err := tester.Client.GetPayload( + context.Background(), + blobKey, + blobCert) require.Nil(t, blob) require.NotNil(t, err) @@ -206,10 +246,9 @@ func TestNoRelayResponse(t *testing.T) { // TestNoRelays tests that having no relay keys is handled gracefully func TestNoRelays(t *testing.T) { tester := buildClientTester(t) + blobKey, _, blobCert := buildBlobAndCert(t, tester, []core.RelayKey{}) - blobKey := core.BlobKey(tester.Random.Bytes(32)) - - blob, err := tester.Client.GetBlob(context.Background(), blobKey, []core.RelayKey{}) + blob, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) require.Nil(t, blob) require.NotNil(t, err) @@ -220,25 +259,22 @@ func TestNoRelays(t *testing.T) { func TestGetBlobReturns0Len(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tester.Random.Bytes(32)) - relayCount := 10 relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { relayKeys[i] = tester.Random.Uint32() } + blobKey, blobBytes, blobCert := buildBlobAndCert(t, tester, relayKeys) // the first GetBlob will return a 0 len blob tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return([]byte{}, nil).Once() - // the second call will return random bytes + // the second call will return blob bytes tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return( - tester.Random.Bytes(100), + blobBytes, nil).Once() - tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.Bytes(50), nil) - // the call to the first relay will fail with a 0 len blob returned. the call to the second relay will succeed - blob, err := tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + blob, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) require.NotNil(t, blob) require.NoError(t, err) @@ -249,21 +285,24 @@ func TestGetBlobReturns0Len(t *testing.T) { func TestFailedDecoding(t *testing.T) { tester := buildClientTester(t) - blobKey := core.BlobKey(tester.Random.Bytes(32)) - relayCount := 10 relayKeys := make([]core.RelayKey, relayCount) for i := 0; i < relayCount; i++ { relayKeys[i] = tester.Random.Uint32() } + blobKey, blobBytes, blobCert := buildBlobAndCert(t, tester, relayKeys) - tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tester.Random.Bytes(100), nil) - - tester.MockCodec.On("DecodeBlob", mock.Anything).Return(nil, fmt.Errorf("decode failed")).Once() - tester.MockCodec.On("DecodeBlob", mock.Anything).Return(tester.Random.Bytes(50), nil).Once() + // payloadLength random bytes are guaranteed to be an invalid blob + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return( + tester.Random.Bytes(payloadLength), + nil).Once() + // the second call will return blob bytes + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return( + blobBytes, + nil).Once() // decoding will fail the first time, but succeed the second time - blob, err := tester.Client.GetBlob(context.Background(), blobKey, relayKeys) + blob, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) require.NotNil(t, blob) require.NoError(t, err) @@ -298,7 +337,7 @@ func TestErrorClose(t *testing.T) { func TestGetCodec(t *testing.T) { tester := buildClientTester(t) - require.Equal(t, tester.MockCodec, tester.Client.GetCodec()) + require.Equal(t, tester.Codec, tester.Client.GetCodec()) tester.requireExpectations(t) } @@ -322,7 +361,8 @@ func TestBuilder(t *testing.T) { client, err := clients.BuildEigenDAClient( logging.NewNoopLogger(), clientConfig, - relayClientConfig) + relayClientConfig, + []bn254.G1Affine{}) require.NotNil(t, client) require.NoError(t, err) From 975b6e5c1996495cbddee9535634abef2f24764e Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:37:15 -0500 Subject: [PATCH 14/18] Remove unnecessary codec mock Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/mock/blob_codec.go | 68 --------------------------- 1 file changed, 68 deletions(-) delete mode 100644 api/clients/codecs/mock/blob_codec.go diff --git a/api/clients/codecs/mock/blob_codec.go b/api/clients/codecs/mock/blob_codec.go deleted file mode 100644 index 973669bff..000000000 --- a/api/clients/codecs/mock/blob_codec.go +++ /dev/null @@ -1,68 +0,0 @@ -package mock - -import mock "github.com/stretchr/testify/mock" - -// BlobCodec is an autogenerated mock type for the BlobCodec type -type BlobCodec struct { - mock.Mock -} - -// DecodeBlob provides a mock function with given fields: encodedData -func (_m *BlobCodec) DecodeBlob(encodedData []byte) ([]byte, error) { - ret := _m.Called(encodedData) - - if len(ret) == 0 { - panic("no return value specified for DecodeBlob") - } - - var r0 []byte - var r1 error - if rf, ok := ret.Get(0).(func([]byte) ([]byte, error)); ok { - return rf(encodedData) - } - if rf, ok := ret.Get(0).(func([]byte) []byte); ok { - r0 = rf(encodedData) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - } - - if rf, ok := ret.Get(1).(func([]byte) error); ok { - r1 = rf(encodedData) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// EncodeBlob provides a mock function with given fields: rawData -func (_m *BlobCodec) EncodeBlob(rawData []byte) ([]byte, error) { - ret := _m.Called(rawData) - - if len(ret) == 0 { - panic("no return value specified for EncodeBlob") - } - - var r0 []byte - var r1 error - if rf, ok := ret.Get(0).(func([]byte) ([]byte, error)); ok { - return rf(rawData) - } - if rf, ok := ret.Get(0).(func([]byte) []byte); ok { - r0 = rf(rawData) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - } - - if rf, ok := ret.Get(1).(func([]byte) error); ok { - r1 = rf(rawData) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} From 0666d240099ed9c21a1726f4f4c6d155e36569d1 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:42:51 -0500 Subject: [PATCH 15/18] Use more reasonable line breaks for logs Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/eigenda_client.go | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/api/clients/v2/eigenda_client.go b/api/clients/v2/eigenda_client.go index ef87e18f4..5db2e3c41 100644 --- a/api/clients/v2/eigenda_client.go +++ b/api/clients/v2/eigenda_client.go @@ -13,7 +13,7 @@ import ( "github.com/consensys/gnark-crypto/ecc/bn254" ) -// EigenDAClient provides the ability to get blobs from the relay subsystem, and to send new blobs to the disperser. +// EigenDAClient provides the ability to get payloads from the relay subsystem, and to send new payloads to the disperser. // // This struct is not threadsafe. type EigenDAClient struct { @@ -137,29 +137,18 @@ func (c *EigenDAClient) verifyBlobFromRelay( // TODO: in the future, this will be optimized to use fiat shamir transformation for verification, rather than // regenerating the commitment: https://github.com/Layr-Labs/eigenda/issues/1037 - valid, err := verification.GenerateAndCompareBlobCommitment( - c.g1Srs, - blob, - blobCommitments.Commitment) + valid, err := verification.GenerateAndCompareBlobCommitment(c.g1Srs, blob, blobCommitments.Commitment) if err != nil { c.log.Warn( "error generating commitment from received blob", - "blobKey", - blobKey, - "relayKey", - relayKey, - "error", - err) + "blobKey", blobKey, "relayKey", relayKey, "error", err) return false } if !valid { c.log.Warn( "blob commitment is invalid for received bytes", - "blobKey", - blobKey, - "relayKey", - relayKey) + "blobKey", blobKey, "relayKey", relayKey) return false } @@ -168,14 +157,8 @@ func (c *EigenDAClient) verifyBlobFromRelay( if uint(len(blob)) != blobCommitments.Length { c.log.Warn( "blob length doesn't match length claimed in blob commitments", - "blobKey", - blobKey, - "relayKey", - relayKey, - "blobLength", - len(blob), - "blobCommitments.Length", - blobCommitments.Length) + "blobKey", blobKey, "relayKey", relayKey, "blobLength", len(blob), + "blobCommitments.Length", blobCommitments.Length) return false } From 0a49aa58ee7a3075a912c7b4f6ccbeebdeb472f0 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 13 Jan 2025 17:09:58 -0500 Subject: [PATCH 16/18] Test malicious cert Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/eigenda_client.go | 11 ++++-- api/clients/v2/test/eigenda_client_test.go | 45 +++++++++++----------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/api/clients/v2/eigenda_client.go b/api/clients/v2/eigenda_client.go index 5db2e3c41..4db366310 100644 --- a/api/clients/v2/eigenda_client.go +++ b/api/clients/v2/eigenda_client.go @@ -105,11 +105,16 @@ func (c *EigenDAClient) GetPayload( continue } - // An honest relay should never send a blob which cannot be decoded into a payload payload, err := c.codec.DecodeBlob(blob) if err != nil { - c.log.Warn("error decoding blob from relay", "blobKey", blobKey, "relayKey", relayKey, "error", err) - continue + c.log.Error( + `Blob verification was successful, but decode blob failed! + This is likely a problem with the local blob codec configuration, + but could potentially indicate a maliciously generated blob certificate. + It should not be possible for an honestly generated certificate to verify + for an invalid blob!`, + "blobKey", blobKey, "relayKey", relayKey, "blobCertificate", blobCertificate, "error", err) + return nil, fmt.Errorf("decode blob: %w", err) } return payload, nil diff --git a/api/clients/v2/test/eigenda_client_test.go b/api/clients/v2/test/eigenda_client_test.go index 61f911657..dffefdac5 100644 --- a/api/clients/v2/test/eigenda_client_test.go +++ b/api/clients/v2/test/eigenda_client_test.go @@ -2,6 +2,7 @@ package test import ( "context" + "encoding/binary" "errors" "fmt" "runtime" @@ -33,11 +34,6 @@ type ClientTester struct { G1Srs []bn254.G1Affine } -func (c *ClientTester) requireExpectations(t *testing.T) { - c.MockRelayClient.AssertExpectations(t) - // TODO: get rid of this -} - // buildClientTester sets up a client with mocks necessary for testing func buildClientTester(t *testing.T) ClientTester { logger := logging.NewNoopLogger() @@ -121,7 +117,7 @@ func TestGetPayloadSuccess(t *testing.T) { require.NotNil(t, payload) require.NoError(t, err) - tester.requireExpectations(t) + tester.MockRelayClient.AssertExpectations(t) } // TestRelayCallTimeout verifies that calls to the relay timeout after the expected duration @@ -169,7 +165,7 @@ func TestRelayCallTimeout(t *testing.T) { _, _ = tester.Client.GetPayload(context.Background(), blobKey, blobCert) }) - tester.requireExpectations(t) + tester.MockRelayClient.AssertExpectations(t) } // TestRandomRelayRetries verifies correct behavior when some relays do not respond with the blob, @@ -217,7 +213,7 @@ func TestRandomRelayRetries(t *testing.T) { // with 100 random tries, with possible values between 1 and 100, we can very confidently require that there are at least 10 unique values require.Greater(t, len(requiredTries), 10) - tester.requireExpectations(t) + tester.MockRelayClient.AssertExpectations(t) } // TestNoRelayResponse tests functionality when none of the relays respond @@ -240,7 +236,7 @@ func TestNoRelayResponse(t *testing.T) { require.Nil(t, blob) require.NotNil(t, err) - tester.requireExpectations(t) + tester.MockRelayClient.AssertExpectations(t) } // TestNoRelays tests that having no relay keys is handled gracefully @@ -252,7 +248,7 @@ func TestNoRelays(t *testing.T) { require.Nil(t, blob) require.NotNil(t, err) - tester.requireExpectations(t) + tester.MockRelayClient.AssertExpectations(t) } // TestGetBlobReturns0Len verifies that a 0 length blob returned from a relay is handled gracefully, and that the client retries after such a failure @@ -278,7 +274,7 @@ func TestGetBlobReturns0Len(t *testing.T) { require.NotNil(t, blob) require.NoError(t, err) - tester.requireExpectations(t) + tester.MockRelayClient.AssertExpectations(t) } // TestFailedDecoding verifies that a failed blob decode is handled gracefully, and that the client retries after such a failure @@ -292,21 +288,26 @@ func TestFailedDecoding(t *testing.T) { } blobKey, blobBytes, blobCert := buildBlobAndCert(t, tester, relayKeys) - // payloadLength random bytes are guaranteed to be an invalid blob - tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return( - tester.Random.Bytes(payloadLength), - nil).Once() - // the second call will return blob bytes + // intentionally cause the claimed length to differ from the actual length + binary.BigEndian.PutUint32(blobBytes[2:6], uint32(len(blobBytes)-1)) + + // generate a malicious cert, which will verify for the invalid blob + maliciousCommitment, err := verification.GenerateBlobCommitment(tester.G1Srs, blobBytes) + require.NoError(t, err) + require.NotNil(t, maliciousCommitment) + + blobCert.BlobHeader.BlobCommitments.Commitment = maliciousCommitment + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return( blobBytes, nil).Once() // decoding will fail the first time, but succeed the second time blob, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) - require.NotNil(t, blob) - require.NoError(t, err) + require.Error(t, err) + require.Nil(t, blob) - tester.requireExpectations(t) + tester.MockRelayClient.AssertExpectations(t) } // TestErrorFreeClose tests the happy case, where none of the internal closes yield an error @@ -318,7 +319,7 @@ func TestErrorFreeClose(t *testing.T) { err := tester.Client.Close() require.NoError(t, err) - tester.requireExpectations(t) + tester.MockRelayClient.AssertExpectations(t) } // TestErrorClose tests what happens when subcomponents throw errors when being closed @@ -330,7 +331,7 @@ func TestErrorClose(t *testing.T) { err := tester.Client.Close() require.NotNil(t, err) - tester.requireExpectations(t) + tester.MockRelayClient.AssertExpectations(t) } // TestGetCodec checks that the codec used in construction is returned by GetCodec @@ -339,7 +340,7 @@ func TestGetCodec(t *testing.T) { require.Equal(t, tester.Codec, tester.Client.GetCodec()) - tester.requireExpectations(t) + tester.MockRelayClient.AssertExpectations(t) } // TestBuilder tests that the method that builds the client from config doesn't throw any obvious errors From 2d392ff03dd3e5511ea975aa402058a8b07e9bdb Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:40:38 -0500 Subject: [PATCH 17/18] Finish test coverage Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/test/eigenda_client_test.go | 109 ++++++++++++++++++--- 1 file changed, 95 insertions(+), 14 deletions(-) diff --git a/api/clients/v2/test/eigenda_client_test.go b/api/clients/v2/test/eigenda_client_test.go index dffefdac5..d4081128e 100644 --- a/api/clients/v2/test/eigenda_client_test.go +++ b/api/clients/v2/test/eigenda_client_test.go @@ -46,7 +46,7 @@ func buildClientTester(t *testing.T) ClientTester { random := testrandom.NewTestRandom(t) - g1Srs, err := kzg.ReadG1Points(g1Path, uint64(payloadLength), uint64(runtime.GOMAXPROCS(0))) + g1Srs, err := kzg.ReadG1Points(g1Path, 5, uint64(runtime.GOMAXPROCS(0))) require.NotNil(t, g1Srs) require.NoError(t, err) @@ -70,6 +70,7 @@ func buildClientTester(t *testing.T) ClientTester { } } +// Builds a random blob key, blob bytes, and valid certificate func buildBlobAndCert( t *testing.T, tester ClientTester, @@ -203,8 +204,8 @@ func TestRandomRelayRetries(t *testing.T) { for i := 0; i < relayCount; i++ { failedCallCount = 0 - blob, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) - require.NotNil(t, blob) + payload, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) + require.NotNil(t, payload) require.NoError(t, err) requiredTries[failedCallCount] = true @@ -229,11 +230,11 @@ func TestNoRelayResponse(t *testing.T) { tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(nil, fmt.Errorf("offline relay")) - blob, err := tester.Client.GetPayload( + payload, err := tester.Client.GetPayload( context.Background(), blobKey, blobCert) - require.Nil(t, blob) + require.Nil(t, payload) require.NotNil(t, err) tester.MockRelayClient.AssertExpectations(t) @@ -244,8 +245,8 @@ func TestNoRelays(t *testing.T) { tester := buildClientTester(t) blobKey, _, blobCert := buildBlobAndCert(t, tester, []core.RelayKey{}) - blob, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) - require.Nil(t, blob) + payload, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) + require.Nil(t, payload) require.NotNil(t, err) tester.MockRelayClient.AssertExpectations(t) @@ -270,14 +271,95 @@ func TestGetBlobReturns0Len(t *testing.T) { nil).Once() // the call to the first relay will fail with a 0 len blob returned. the call to the second relay will succeed - blob, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) - require.NotNil(t, blob) + payload, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) + require.NotNil(t, payload) + require.NoError(t, err) + + tester.MockRelayClient.AssertExpectations(t) +} + +// TestGetBlobReturnsDifferentBlob tests what happens when one relay returns a blob that doesn't match the commitment. +// It also tests that the client retries to get the correct blob from a different relay +func TestGetBlobReturnsDifferentBlob(t *testing.T) { + tester := buildClientTester(t) + relayCount := 10 + relayKeys := make([]core.RelayKey, relayCount) + for i := 0; i < relayCount; i++ { + relayKeys[i] = tester.Random.Uint32() + } + blobKey1, blobBytes1, blobCert1 := buildBlobAndCert(t, tester, relayKeys) + _, blobBytes2, _ := buildBlobAndCert(t, tester, relayKeys) + + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey1).Return(blobBytes2, nil).Once() + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey1).Return(blobBytes1, nil).Once() + + payload, err := tester.Client.GetPayload( + context.Background(), + blobKey1, + blobCert1) + require.NotNil(t, payload) + require.NoError(t, err) + + tester.MockRelayClient.AssertExpectations(t) +} + +// TestGetBlobReturnsInvalidBlob tests what happens if a relay returns a blob which causes commitment verification to +// throw an error. It verifies that the client tries again with a different relay after such a failure. +func TestGetBlobReturnsInvalidBlob(t *testing.T) { + tester := buildClientTester(t) + relayCount := 10 + relayKeys := make([]core.RelayKey, relayCount) + for i := 0; i < relayCount; i++ { + relayKeys[i] = tester.Random.Uint32() + } + blobKey, blobBytes, blobCert := buildBlobAndCert(t, tester, relayKeys) + + tooLongBytes := make([]byte, len(blobBytes)+100) + copy(tooLongBytes[:], blobBytes) + + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(tooLongBytes, nil).Once() + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(blobBytes, nil).Once() + + // this will fail the first time, since there isn't enough srs loaded to compute the commitment of the returned bytes + // it will succeed when the second relay gives the correct bytes + payload, err := tester.Client.GetPayload( + context.Background(), + blobKey, + blobCert) + + require.NotNil(t, payload) require.NoError(t, err) tester.MockRelayClient.AssertExpectations(t) } -// TestFailedDecoding verifies that a failed blob decode is handled gracefully, and that the client retries after such a failure +// TestGetBlobReturnsBlobWithInvalidLen check what happens if the blob length doesn't match the length that exists in +// the BlobCommitment +func TestGetBlobReturnsBlobWithInvalidLen(t *testing.T) { + tester := buildClientTester(t) + + relayKeys := make([]core.RelayKey, 1) + relayKeys[0] = tester.Random.Uint32() + + blobKey, blobBytes, blobCert := buildBlobAndCert(t, tester, relayKeys) + + blobCert.BlobHeader.BlobCommitments.Length-- + + tester.MockRelayClient.On("GetBlob", mock.Anything, mock.Anything, blobKey).Return(blobBytes, nil).Once() + + // this will fail, since the length in the BlobCommitment doesn't match the actual blob length + payload, err := tester.Client.GetPayload( + context.Background(), + blobKey, + blobCert) + + require.Nil(t, payload) + require.Error(t, err) + + tester.MockRelayClient.AssertExpectations(t) +} + +// TestFailedDecoding verifies that a failed blob decode is handled gracefully func TestFailedDecoding(t *testing.T) { tester := buildClientTester(t) @@ -288,7 +370,7 @@ func TestFailedDecoding(t *testing.T) { } blobKey, blobBytes, blobCert := buildBlobAndCert(t, tester, relayKeys) - // intentionally cause the claimed length to differ from the actual length + // intentionally cause the payload header claimed length to differ from the actual length binary.BigEndian.PutUint32(blobBytes[2:6], uint32(len(blobBytes)-1)) // generate a malicious cert, which will verify for the invalid blob @@ -302,10 +384,9 @@ func TestFailedDecoding(t *testing.T) { blobBytes, nil).Once() - // decoding will fail the first time, but succeed the second time - blob, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) + payload, err := tester.Client.GetPayload(context.Background(), blobKey, blobCert) require.Error(t, err) - require.Nil(t, blob) + require.Nil(t, payload) tester.MockRelayClient.AssertExpectations(t) } From db512914ddb00d25271c82a81b735faf5fff0643 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:47:15 -0500 Subject: [PATCH 18/18] Fix commitment length check Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/eigenda_client.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/api/clients/v2/eigenda_client.go b/api/clients/v2/eigenda_client.go index 4db366310..fefc99bf9 100644 --- a/api/clients/v2/eigenda_client.go +++ b/api/clients/v2/eigenda_client.go @@ -157,11 +157,15 @@ func (c *EigenDAClient) verifyBlobFromRelay( return false } - // checking that the length returned by the relay matches the claimed length is sufficient here: it isn't necessary - // to verify the length proof itself, since this will have been done by DA nodes prior to signing for availability. - if uint(len(blob)) != blobCommitments.Length { + // Checking that the length returned by the relay is <= the length claimed in the BlobCommitments is sufficient + // here: it isn't necessary to verify the length proof itself, since this will have been done by DA nodes prior to + // signing for availability. + // + // Note that the length in the commitment is the length of the blob, padded to a power of 2. For this reason, we + // assert <=, as opposed to asserting equality + if uint(len(blob)) > blobCommitments.Length { c.log.Warn( - "blob length doesn't match length claimed in blob commitments", + "blob length is greater than length claimed in blob commitments", "blobKey", blobKey, "relayKey", relayKey, "blobLength", len(blob), "blobCommitments.Length", blobCommitments.Length) return false