test(integration): fix flaky integration tests with proper health checks

integration_test.go

@@ -2,12 +2,18 @@
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
 	"math/big"
+	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/golang-jwt/jwt/v5"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -26,29 +32,44 @@
 const (
 	TEST_ETH_URL    = "http://localhost:8545"
 	TEST_ENGINE_URL = "http://localhost:8551"
-	JWT_SECRET      = "09a23c010d96caaebb21c193b85d30bbb62a9bac5bd0a684e9e91c77c811ca65" //nolint:gosec
 
 	CHAIN_ID          = "1234"
 	GENESIS_HASH      = "0x8bf225d50da44f60dee1c4ee6f810fe5b44723c76ac765654b6692d50459f216"
 	GENESIS_STATEROOT = "0x362b7d8a31e7671b0f357756221ac385790c25a27ab222dc8cbdd08944f5aea4"
 	TEST_PRIVATE_KEY  = "cece4f25ac74deb1468965160c7185e07dff413f23fcadb611b05ca37ab0a52e"
 	TEST_TO_ADDRESS   = "0x944fDcD1c868E3cC566C78023CcB38A32cDA836E"
 
-	DOCKER_CHAIN_PATH      = "./docker/chain"     // path relative to the test file
-	DOCKER_JWTSECRET_PATH  = "./docker/jwttoken/" //nolint:gosec // path relative to the test file
-	DOCKER_JWT_SECRET_FILE = "testsecret.hex"
+	DOCKER_PATH  = "./docker"
+	JWT_FILENAME = "testsecret.hex"
 )
 
-func setupTestRethEngine(t *testing.T) {
+func generateJWTSecret() (string, error) {
+	jwtSecret := make([]byte, 32)
+	_, err := rand.Read(jwtSecret)
+	if err != nil {
+		return "", fmt.Errorf("failed to generate random bytes: %w", err)
+	}
+
+	return hex.EncodeToString(jwtSecret), nil
+}
+
+func setupTestRethEngine(t *testing.T) string {
 	t.Helper()
 
-	chainPath, err := filepath.Abs(DOCKER_CHAIN_PATH)
+	chainPath, err := filepath.Abs(filepath.Join(DOCKER_PATH, "chain"))
+	require.NoError(t, err)
+
+	jwtPath, err := filepath.Abs(filepath.Join(DOCKER_PATH, "jwttoken"))
+	require.NoError(t, err)
+
+	err = os.MkdirAll(jwtPath, 0755)
 	require.NoError(t, err)
 
-	jwtSecretPath, err := filepath.Abs(DOCKER_JWTSECRET_PATH)
+	jwtSecret, err := generateJWTSecret()
 	require.NoError(t, err)
 
-	err = os.WriteFile(DOCKER_JWTSECRET_PATH+DOCKER_JWT_SECRET_FILE, []byte(JWT_SECRET), 0600)
+	jwtFile := filepath.Join(jwtPath, JWT_FILENAME)
+	err = os.WriteFile(jwtFile, []byte(jwtSecret), 0600)
 	require.NoError(t, err)
 
 	cli, err := client.NewClientWithOpts()
@@ -83,7 +104,7 @@
 		&container.HostConfig{
 			Binds: []string{
 				chainPath + ":/root/chain:ro",
-				jwtSecretPath + ":/root/jwt:ro",
+				jwtPath + ":/root/jwt:ro",
 			},
 			PortBindings: nat.PortMap{
 				nat.Port("8545/tcp"): []nat.PortBinding{
@@ -106,21 +127,82 @@
 	err = cli.ContainerStart(context.Background(), rethContainer.ID, container.StartOptions{})
 	require.NoError(t, err)
 
-	// TODO: check container status instead of sleeping
-	time.Sleep(5 * time.Second)
+	err = waitForRethContainer(t, jwtSecret)
+	require.NoError(t, err)
 
 	t.Cleanup(func() {
 		err = cli.ContainerStop(context.Background(), rethContainer.ID, container.StopOptions{})
 		require.NoError(t, err)
 		err = cli.ContainerRemove(context.Background(), rethContainer.ID, container.RemoveOptions{})
 		require.NoError(t, err)
-		err = os.Remove(DOCKER_JWTSECRET_PATH + DOCKER_JWT_SECRET_FILE)
+		err = os.Remove(jwtFile)
 		require.NoError(t, err)
 	})
+
+	return jwtSecret
+}
+
+// waitForRethContainer polls the reth endpoints until they're ready or timeout occurs
+func waitForRethContainer(t *testing.T, jwtSecret string) error {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	client := &http.Client{
+		Timeout: 1 * time.Second,
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("timeout waiting for reth container to be ready")
+		default:
+			// check :8545 is ready
+			rpcReq := strings.NewReader(`{"jsonrpc":"2.0","method":"net_version","params":[],"id":1}`)
+			resp, err := client.Post(TEST_ETH_URL, "application/json", rpcReq)
+			if err == nil {
+				resp.Body.Close()
+				if resp.StatusCode == http.StatusOK {
+					// check :8551 is ready
+					req, err := http.NewRequest("POST", TEST_ENGINE_URL, strings.NewReader(`{"jsonrpc":"2.0","method":"engine_exchangeTransitionConfigurationV1","params":[],"id":1}`))
+					if err != nil {
+						return err
+					}
+					req.Header.Set("Content-Type", "application/json")
+
+					jwtSecretBytes, err := hex.DecodeString(strings.TrimPrefix(jwtSecret, "0x"))
+					if err != nil {
+						return fmt.Errorf("failed to decode JWT secret: %w", err)
+					}
+
+					token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
+						"iat": time.Now().Unix(),
+					})
+
+					signedToken, err := token.SignedString(jwtSecretBytes)
+					if err != nil {
+						return fmt.Errorf("failed to sign JWT token: %w", err)
+					}
+
+					req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", signedToken))
+
+					resp, err := client.Do(req)
+					if err == nil {
+						resp.Body.Close()
+						if resp.StatusCode == http.StatusOK {
+							return nil // Both endpoints are ready
+						}
+					}
+				}
+			}
+			time.Sleep(500 * time.Millisecond)
+		}
+	}
 }
 
 func TestExecutionClientLifecycle(t *testing.T) {
-	setupTestRethEngine(t)
+	jwtSecret := setupTestRethEngine(t)
 
 	initialHeight := uint64(0)
 	genesisHash := common.HexToHash(GENESIS_HASH)
@@ -135,7 +217,7 @@
 		&proxy_json_rpc.Config{},
 		TEST_ETH_URL,
 		TEST_ENGINE_URL,
-		JWT_SECRET,
+		jwtSecret,
 		genesisHash,
 		common.Address{},
 	)
@@ -206,7 +288,7 @@
 }
 
 func TestExecutionClient_InitChain_InvalidPayloadTimestamp(t *testing.T) {
-	setupTestRethEngine(t)
+	jwtSecret := setupTestRethEngine(t)
 
 	initialHeight := uint64(0)
 	genesisHash := common.HexToHash(GENESIS_HASH)
@@ -216,7 +298,7 @@
 		&proxy_json_rpc.Config{},
 		TEST_ETH_URL,
 		TEST_ENGINE_URL,
-		JWT_SECRET,
+		jwtSecret,
 		genesisHash,
 		common.Address{},
 	)