diff --git a/core/go/internal/engine/baseledgertx/dispatch_action.go b/core/go/internal/engine/baseledgertx/dispatch_action.go index 04968bbe3..f0fc4ae2a 100644 --- a/core/go/internal/engine/baseledgertx/dispatch_action.go +++ b/core/go/internal/engine/baseledgertx/dispatch_action.go @@ -83,13 +83,13 @@ func (baseLedgerEngine *baseLedgerTxEngine) dispatchAction(ctx context.Context, } } -func (te *orchestrator) dispatchAction(ctx context.Context, mtx *baseTypes.ManagedTX, action APIRequestType, response chan<- APIResponse) { +func (oc *orchestrator) dispatchAction(ctx context.Context, mtx *baseTypes.ManagedTX, action APIRequestType, response chan<- APIResponse) { switch action { case ActionSuspend, ActionResume: - te.InFlightTxsMux.Lock() - defer te.InFlightTxsMux.Unlock() + oc.InFlightTxsMux.Lock() + defer oc.InFlightTxsMux.Unlock() var pending *InFlightTransactionStageController - for _, inflight := range te.InFlightTxs { + for _, inflight := range oc.InFlightTxs { if inflight.stateManager.GetTxID() == mtx.ID { pending = inflight break @@ -101,21 +101,21 @@ func (te *orchestrator) dispatchAction(ctx context.Context, mtx *baseTypes.Manag } if pending == nil { // transaction not in flight yet, update the DB directly and tell the engine to not pick up the transaction until we completed - te.transactionIDsInStatusUpdate = append(te.transactionIDsInStatusUpdate, mtx.ID) + oc.transactionIDsInStatusUpdate = append(oc.transactionIDsInStatusUpdate, mtx.ID) go func() { defer func() { - te.InFlightTxsMux.Lock() - defer te.InFlightTxsMux.Unlock() - newTransactionIDsInStatusUpdate := make([]string, 0, len(te.transactionIDsInStatusUpdate)-1) - for _, txID := range te.transactionIDsInStatusUpdate { + oc.InFlightTxsMux.Lock() + defer oc.InFlightTxsMux.Unlock() + newTransactionIDsInStatusUpdate := make([]string, 0, len(oc.transactionIDsInStatusUpdate)-1) + for _, txID := range oc.transactionIDsInStatusUpdate { if txID != mtx.ID { newTransactionIDsInStatusUpdate = append(newTransactionIDsInStatusUpdate, txID) } } - te.transactionIDsInStatusUpdate = newTransactionIDsInStatusUpdate + oc.transactionIDsInStatusUpdate = newTransactionIDsInStatusUpdate }() log.L(ctx).Debugf("Setting status to '%s' for transaction %s", newStatus, mtx.ID) - err := te.txStore.UpdateTransaction(ctx, mtx.ID, &baseTypes.BaseTXUpdates{ + err := oc.txStore.UpdateTransaction(ctx, mtx.ID, &baseTypes.BaseTXUpdates{ Status: &newStatus, }) if err != nil { diff --git a/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller.go b/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller.go index 9b6d01584..ccac5b73b 100644 --- a/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller.go +++ b/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller.go @@ -51,6 +51,8 @@ type InFlightTransactionStageController struct { stateManager baseTypes.InFlightTransactionStateManager + confirmed bool + // pauseRequested bool // deleteRequested bool // figure out what's the reliable approach for deletion } @@ -64,9 +66,10 @@ type PointOfTime struct { type GenericStatus string const ( - GenericStatusSuccess GenericStatus = "success" - GenericStatusFail GenericStatus = "fail" - GenericStatusTimeOut GenericStatus = "timeout" + GenericStatusSuccess GenericStatus = "success" + GenericStatusFail GenericStatus = "fail" + GenericStatusConflict GenericStatus = "conflict" + GenericStatusTimeOut GenericStatus = "timeout" ) type InFlightTxOperation string @@ -95,12 +98,12 @@ type BasicActionError struct { func NewInFlightTransactionStageController( enth *baseLedgerTxEngine, - te *orchestrator, + oc *orchestrator, mtx *baseTypes.ManagedTX, ) *InFlightTransactionStageController { ift := &InFlightTransactionStageController{ - orchestrator: te, + orchestrator: oc, txInflightTime: time.Now(), txInDBTime: *mtx.Created.Time(), txTimeline: []PointOfTime{ @@ -113,7 +116,7 @@ func NewInFlightTransactionStageController( } ift.MarkTime("wait_in_inflight_queue") - ift.stateManager = NewInFlightTransactionStateManager(enth.thMetrics, enth.balanceManager, enth.txStore, enth.txConfirmationListener, ift, NewInMemoryTxStateMananger(enth.ctx, mtx), te.turnOffHistory, ift.testOnlyNoEventMode) + ift.stateManager = NewInFlightTransactionStateManager(enth.thMetrics, enth.balanceManager, enth.txStore, enth.bIndexer, ift, NewInMemoryTxStateMananger(enth.ctx, mtx), oc.retry, oc.turnOffHistory, ift.testOnlyNoEventMode) return ift } @@ -235,10 +238,6 @@ func (it *InFlightTransactionStageController) ProduceLatestInFlightStageContext( MaxFeePerGas: (*ethtypes.HexInteger)(gpo.MaxFeePerGas), MaxPriorityFeePerGas: (*ethtypes.HexInteger)(gpo.MaxPriorityFeePerGas), } - rsc.StageOutputsToBePersisted.PolicyInfo = &baseTypes.EnterprisePolicyInfo{ - LastWarnTime: rsc.InMemoryTx.GetPolicyInfo().LastWarnTime, - SubmittedTxHashes: rsc.InMemoryTx.GetPolicyInfo().SubmittedTxHashes, - } rsc.StageOutputsToBePersisted.AddSubStatusAction(baseTypes.BaseTxActionRetrieveGasPrice, fftypes.JSONAnyPtr(string(gpoJSON)), nil) } _ = it.TriggerPersistTxState(ctx) @@ -334,10 +333,7 @@ func (it *InFlightTransactionStageController) ProduceLatestInFlightStageContext( rsc.StageOutputsToBePersisted.AddSubStatusAction(baseTypes.BaseTxActionSubmitTransaction, fftypes.JSONAnyPtr(`{"reason":"`+string(rsIn.SubmitOutput.ErrorReason)+`"}`), fftypes.JSONAnyPtr(`{"error":"`+rsIn.SubmitOutput.Err.Error()+`"}`)) if rsc.InMemoryTx.GetTransactionHash() != "" { // did a re-submission, no matter the result, update the last warn time to avoid another retry - rsc.StageOutputsToBePersisted.PolicyInfo = &baseTypes.EnterprisePolicyInfo{ - LastWarnTime: fftypes.Now(), - SubmittedTxHashes: rsc.InMemoryTx.GetPolicyInfo().SubmittedTxHashes, - } + rsc.StageOutputsToBePersisted.TxUpdates.LastSubmit = fftypes.Now() } } else { if rsIn.SubmitOutput.SubmissionOutcome == baseTypes.SubmissionOutcomeSubmittedNew { @@ -372,16 +368,11 @@ func (it *InFlightTransactionStageController) ProduceLatestInFlightStageContext( rsc.StageOutputsToBePersisted.TxUpdates.TransactionHash = &rsc.StageOutput.SubmitOutput.TxHash rsc.StageOutputsToBePersisted.TxUpdates.LastSubmit = rsIn.SubmitOutput.SubmissionTime } - - rsc.StageOutputsToBePersisted.PolicyInfo = &baseTypes.EnterprisePolicyInfo{ - LastWarnTime: fftypes.Now(), - SubmittedTxHashes: rsc.InMemoryTx.GetPolicyInfo().SubmittedTxHashes, - } if rsc.StageOutput.SubmitOutput.TxHash != "" { // as long as we add the tx hash in to the submitted transaction array // the pre-tracking logic will figure out which transaction hash is the right one to set // on the main tx record based on receipt checking - existingHashes := rsc.InMemoryTx.GetPolicyInfo().SubmittedTxHashes + existingHashes := rsc.InMemoryTx.GetSubmittedHashes() if existingHashes == nil { existingHashes = make([]string, 0) } @@ -392,53 +383,17 @@ func (it *InFlightTransactionStageController) ProduceLatestInFlightStageContext( } } if !hashAlreadyTracked { + if rsc.StageOutputsToBePersisted.TxUpdates == nil { + rsc.StageOutputsToBePersisted.TxUpdates = &baseTypes.BaseTXUpdates{} + } existingHashes = append(existingHashes, rsIn.SubmitOutput.TxHash) - rsc.StageOutputsToBePersisted.PolicyInfo.SubmittedTxHashes = existingHashes + rsc.StageOutputsToBePersisted.TxUpdates.SubmittedHashes = existingHashes } } } _ = it.TriggerPersistTxState(ctx) } - case baseTypes.InFlightTxStageReceipting: - // first check whether we've already completed the action and just waiting for required persistence to go to the next stage - if rsIn.PersistenceOutput != nil { - // record persistence information so that if the current stage is not complete - // the logic outside the output check loop can use the information to determine the next step - if rsIn.PersistenceOutput.PersistenceError != nil { - if time.Since(rsIn.PersistenceOutput.Time) > it.persistenceRetryTimeout { - // retry persistence - _ = it.TriggerPersistTxState(ctx) - } else { - // wait for retry timeout - unprocessedStageOutputs = append(unprocessedStageOutputs, rsIn) - } - continue - } - if rsIn.PersistenceOutput.PersistenceError == nil { - // we've persisted successfully, it's safe to move to the next stage based on the latest state of the managed transaction - it.TriggerNewStageRun(ctx, baseTypes.InFlightTxStageConfirming, baseTypes.BaseTxSubStatusTracking, nil) // just do the stage switch, listener is already set up. - it.MarkInFlightTxStale() // notify tx is stale as confirmation events might already in the queue - } - } else if rsIn.ReceiptOutput == nil { - log.L(ctx).Errorf("receiptOutput should not be nil for transaction with ID: %s, in the stage output object: %+v.", rsc.InMemoryTx.GetTxID(), rsIn) - tOut.Error = i18n.NewError(ctx, msgs.MsgInvalidStageOutput, "receiptOutput", rsIn) - // unexpected error, reset the running stage context so that it gets retried - it.stateManager.ClearRunningStageContext(ctx) - } else { - if rsIn.ReceiptOutput.Err != nil { - // for now, we log the error and wait for the stale transaction timeout to re-trigger the signing - log.L(ctx).Errorf("Setting up receipt listener for transaction with ID: %s failed due to error: %+v", rsc.InMemoryTx.GetTxID(), rsIn.ReceiptOutput.Err) - rsc.StageErrored = true - } else { - // receipt received - log.L(ctx).Debugf("Receipt received for transaction %s at nonce %s / %d - hash: %s", rsc.InMemoryTx.GetTxID(), rsc.InMemoryTx.GetFrom(), rsc.InMemoryTx.GetNonce().Int64(), rsc.InMemoryTx.GetTransactionHash()) - rsc.SetNewPersistenceUpdateOutput() - rsc.StageOutputsToBePersisted.Receipt = rsIn.ReceiptOutput.Receipt - rsc.StageOutputsToBePersisted.AddSubStatusAction(baseTypes.BaseTxActionReceiveReceipt, fftypes.JSONAnyPtr(`{"protocolId":"`+rsIn.ReceiptOutput.Receipt.ProtocolID+`"}`), nil) - _ = it.TriggerPersistTxState(ctx) // we could have unnecessary persistence call here when transaction paused/crash during tracking state after the receipt is stored. - } - } case baseTypes.InFlightTxStageConfirming: // first check whether we've already completed the action and just waiting for required persistence to go to the next stage if rsIn.PersistenceOutput != nil { @@ -462,15 +417,13 @@ func (it *InFlightTransactionStageController) ProduceLatestInFlightStageContext( // dispatch an event to event handler and discard any handling errors if rsc.InMemoryTx.GetStatus() == baseTypes.BaseTxStatusSucceeded { _ = it.managedTXEventNotifier.Notify(ctx, baseTypes.ManagedTransactionEvent{ - Type: baseTypes.ManagedTXProcessSucceeded, - Tx: rsc.InMemoryTx.GetTx(), - Receipt: rsc.InMemoryTx.GetReceipt(), + Type: baseTypes.ManagedTXProcessSucceeded, + Tx: rsc.InMemoryTx.GetTx(), }) } else if rsc.InMemoryTx.GetStatus() == baseTypes.BaseTxStatusFailed { _ = it.managedTXEventNotifier.Notify(ctx, baseTypes.ManagedTransactionEvent{ - Type: baseTypes.ManagedTXProcessFailed, - Tx: rsc.InMemoryTx.GetTx(), - Receipt: rsc.InMemoryTx.GetReceipt(), + Type: baseTypes.ManagedTXProcessFailed, + Tx: rsc.InMemoryTx.GetTx(), }) } log.L(ctx).Infof("Complete: Transaction %s marked ready to be removed (status=%s) after %s (%s in-flight)", rsc.InMemoryTx.GetTxID(), rsc.InMemoryTx.GetStatus(), time.Since(it.txInDBTime), time.Since(it.txInflightTime)) @@ -478,7 +431,7 @@ func (it *InFlightTransactionStageController) ProduceLatestInFlightStageContext( it.stateManager.ClearRunningStageContext(ctx) it.MarkInFlightTxStale() } else { - // when there are still more confirmations, clear the stage output to be persisted to let new one go through + // the transaction didn't rsc.StageOutputsToBePersisted = nil } } @@ -488,16 +441,21 @@ func (it *InFlightTransactionStageController) ProduceLatestInFlightStageContext( // unexpected error, reset the running stage context so that it gets retried it.stateManager.ClearRunningStageContext(ctx) } else { - if rsc.StageOutputsToBePersisted != nil { - // there is already a confirmation wait for persistence, wait for that to finish first - unprocessedStageOutputs = append(unprocessedStageOutputs, rsIn) - } else { + if !it.confirmed { // new transaction confirmed received log.L(ctx).Debugf("Confirmed transaction %s at nonce %s / %d - hash: %s", rsc.InMemoryTx.GetTxID(), rsc.InMemoryTx.GetFrom(), rsc.InMemoryTx.GetNonce().Int64(), rsc.InMemoryTx.GetTransactionHash()) rsc.SetNewPersistenceUpdateOutput() - rsc.StageOutputsToBePersisted.Confirmations = rsIn.ConfirmationOutput.Confirmations + rsc.StageOutputsToBePersisted.IndexedTransaction = rsIn.ConfirmationOutput.IndexedTransaction + rsc.StageOutputsToBePersisted.MissedIndexedTransaction = rsIn.ConfirmationOutput.IndexedTransaction == nil // this is an edge case when block indexer has missed a transaction + if rsc.StageOutputsToBePersisted.IndexedTransaction != nil { + rsc.StageOutputsToBePersisted.AddSubStatusAction(baseTypes.BaseTxActionConfirmTransaction, fftypes.JSONAnyPtr(`{"confirmedHash":"`+rsIn.ConfirmationOutput.IndexedTransaction.Hash.String()+`"}`), nil) + } else { + rsc.StageOutputsToBePersisted.AddSubStatusAction(baseTypes.BaseTxActionConfirmTransaction, fftypes.JSONAnyPtr(`{"confirmedHash": null }`), nil) + } _ = it.TriggerPersistTxState(ctx) } + it.confirmed = true + } case baseTypes.InFlightTxStageStatusUpdate: // only requires persistence output for this stage @@ -539,14 +497,9 @@ func (it *InFlightTransactionStageController) ProduceLatestInFlightStageContext( return unprocessedStageOutputs }) - if (rsc.Stage == baseTypes.InFlightTxStageConfirming || rsc.Stage == baseTypes.InFlightTxStageReceipting) && time.Since(*rsc.InMemoryTx.GetPolicyInfo().LastWarnTime.Time()) > it.resubmitInterval { - // timed out waiting for receipt - log.L(ctx).Debugf("Cancelling confirmation manager tracking of hash %s for TX %s", rsc.InMemoryTx.GetTransactionHash(), rsc.InMemoryTx.GetTxID()) - if err := it.txConfirmationListener.Remove(ctx, rsc.InMemoryTx.GetTransactionHash()); err != nil { - // Unexpected error, ignore as fail to remove the listener is not a stop deal - log.L(ctx).Errorf("Error detected notifying confirmation manager to remove transaction hash: %s", err.Error()) - } - // trigger a resubmission when exceeded the resubmit interval + if rsc.Stage == baseTypes.InFlightTxStageConfirming && time.Since(*rsc.InMemoryTx.GetLastSubmitTime().Time()) > it.resubmitInterval { + // timed out waiting for confirmation + log.L(ctx).Debugf("Clearing stage context to trigger a resubmission TX %s (hash %s)", rsc.InMemoryTx.GetTxID(), rsc.InMemoryTx.GetTransactionHash()) it.stateManager.ClearRunningStageContext(ctx) } @@ -559,7 +512,7 @@ func (it *InFlightTransactionStageController) ProduceLatestInFlightStageContext( } if it.stateManager.GetGasPriceObject() != nil { - if it.stateManager.GetReceipt() != nil { + if it.stateManager.GetIndexedTransaction() != nil { // already has receipt so the cost to submit this transaction is zero tOut.Cost = big.NewInt(0) } else { @@ -573,8 +526,13 @@ func (it *InFlightTransactionStageController) ProduceLatestInFlightStageContext( if it.stateManager.GetRunningStageContext(ctx) == nil { // no running context in flight - // first apply any status update that's required - if it.newStatus != nil && !it.stateManager.IsComplete() && *it.newStatus != it.stateManager.GetStatus() { + // first check whether the current transaction is before the confirmed nonce + if tIn.CurrentConfirmedNonce != nil && it.stateManager.GetNonce().Cmp(tIn.CurrentConfirmedNonce) != 1 && !it.confirmed { + // check and track the existing transaction hash + log.L(ctx).Debugf("Transaction with ID %s has gone past confirmed nonce, entering confirmation stage", it.stateManager.GetTxID()) + it.stateManager.AddConfirmationsOutput(ctx, nil) + it.TriggerNewStageRun(ctx, baseTypes.InFlightTxStageConfirming, baseTypes.BaseTxSubStatusTracking, nil) + } else if it.newStatus != nil && !it.stateManager.IsComplete() && *it.newStatus != it.stateManager.GetStatus() { // first apply any status update that's required log.L(ctx).Debugf("Transaction with ID %s entering status update, current status: %s, target status: %s", it.stateManager.GetTxID(), it.stateManager.GetStatus(), *it.newStatus) it.TriggerNewStageRun(ctx, baseTypes.InFlightTxStageStatusUpdate, baseTypes.BaseTxSubStatusReceived, nil) } else if it.stateManager.IsComplete() || it.stateManager.IsSuspended() { @@ -606,15 +564,15 @@ func (it *InFlightTransactionStageController) ProduceLatestInFlightStageContext( } } else { // once we validated the transaction hash matched the transaction state - policyInfo := it.stateManager.GetPolicyInfo() - if policyInfo != nil && policyInfo.LastWarnTime != nil && time.Since(*policyInfo.LastWarnTime.Time()) > it.resubmitInterval { + lastSubmitTime := it.stateManager.GetLastSubmitTime() + if lastSubmitTime != nil && time.Since(*lastSubmitTime.Time()) > it.resubmitInterval { // do a resubmission when exceeded the resubmit interval log.L(ctx).Debugf("Transaction with ID %s entering retrieve gas price as exceeded resubmit interval of %s.", it.stateManager.GetTxID(), it.resubmitInterval.String()) it.TriggerNewStageRun(ctx, baseTypes.InFlightTxStageRetrieveGasPrice, baseTypes.BaseTxSubStatusStale, nil) } else { // check and track the existing transaction hash log.L(ctx).Debugf("Transaction with ID %s entering tracking stage", it.stateManager.GetTxID()) - it.TriggerNewStageRun(ctx, baseTypes.InFlightTxStageReceipting, baseTypes.BaseTxSubStatusTracking, nil) + it.TriggerNewStageRun(ctx, baseTypes.InFlightTxStageConfirming, baseTypes.BaseTxSubStatusTracking, nil) } } @@ -714,47 +672,6 @@ func (it *InFlightTransactionStageController) TriggerStatusUpdate(ctx context.Co }, ctx, it.stateManager.GetStage(ctx), false) return nil } -func (it *InFlightTransactionStageController) TriggerTracking(ctx context.Context) error { - it.executeAsync(func() { - transactionHashToBeTracked := it.stateManager.GetTransactionHash() - policyInfo := it.stateManager.GetPolicyInfo() - if len(policyInfo.SubmittedTxHashes) > 0 { - // check which transaction hash can be tracked - for _, hash := range policyInfo.SubmittedTxHashes { - if hash != "" && hash != it.stateManager.GetTransactionHash() { - _, err := it.ethClient.GetTransactionReceipt(ctx, hash) - if err == nil { - // found receipt for transaction hash use it for tracking - transactionHashToBeTracked = hash - break - } - } - } - } - - if transactionHashToBeTracked != it.stateManager.GetTransactionHash() { - rsc := it.stateManager.GetRunningStageContext(ctx) - rsc.SetNewPersistenceUpdateOutput() - rsc.StageOutputsToBePersisted.AddSubStatusAction(baseTypes.BaseTxActionStateTransition, fftypes.JSONAnyPtr(`{"txHash":"`+transactionHashToBeTracked+`"}`), nil) - rsc.StageOutputsToBePersisted.TxUpdates = &baseTypes.BaseTXUpdates{ - TransactionHash: &transactionHashToBeTracked, - } - _, _, err := it.stateManager.PersistTxState(ctx) - if err != nil { - // restart from scratch - it.stateManager.ClearRunningStageContext(ctx) - return - } - rsc.StageOutputsToBePersisted = nil - } - it.MarkTime("register_receipt_confirmation_handler") - log.L(ctx).Debugf("Passing transaction to confirmation manager %s hash=%s", it.stateManager.GetTxID(), it.stateManager.GetTransactionHash()) - if err := it.txConfirmationListener.Add(ctx, it.stateManager.GetTxID(), it.stateManager.GetTransactionHash(), it.CreateTransactionReceiptReceivedHandler(it.stateManager.GetTxID()), it.CreateTransactionConfirmationsHandler(it.stateManager.GetTxID())); err != nil { - it.stateManager.AddReceiptOutput(ctx, nil, err) - } - }, ctx, it.stateManager.GetStage(ctx), false) - return nil -} func (it *InFlightTransactionStageController) TriggerSignTx(ctx context.Context) error { it.executeAsync(func() { signedMessage, txHash, err := it.signTx(ctx, it.stateManager.GetTx()) diff --git a/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_basic_test.go b/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_basic_test.go index dbe03e937..74f76c131 100644 --- a/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_basic_test.go +++ b/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_basic_test.go @@ -31,7 +31,7 @@ import ( type testInFlightTransactionWithMocksAndConf struct { it *InFlightTransactionStageController - mCL *enginemocks.TransactionConfirmationListener + mBI *componentmocks.BlockIndexer mEC *componentmocks.EthClient mEN *enginemocks.ManagedTxEventNotifier mTS *enginemocks.TransactionStore @@ -46,21 +46,21 @@ func NewTestInFlightTransactionWithMocks(t *testing.T) *testInFlightTransactionW ble, conf := NewTestTransactionEngine(t) mockBalanceManager, mEC, _ := NewTestBalanceManager(context.Background(), t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mTS := enginemocks.NewTransactionStore(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.ctx = ctx ble.balanceManager = mockBalanceManager orchestratorConf := conf.SubSection(OrchestratorSection) - te := NewOrchestrator(ble, imtxs.GetFrom(), orchestratorConf) - it := NewInFlightTransactionStageController(ble, te, imtxs.GetTx()) + oc := NewOrchestrator(ble, imtxs.GetFrom(), orchestratorConf) + it := NewInFlightTransactionStageController(ble, oc, imtxs.GetTx()) it.timeLineLoggingEnabled = true it.testOnlyNoActionMode = true return &testInFlightTransactionWithMocksAndConf{ it: it, - mCL: mCL, + mBI: mBI, mEC: mEC, mEN: mEN, mTS: mTS, diff --git a/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_confirm_test.go b/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_confirm_test.go index e0b13cabe..bfb21c928 100644 --- a/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_confirm_test.go +++ b/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_confirm_test.go @@ -96,7 +96,7 @@ func TestProduceLatestInFlightStageContextConfirming(t *testing.T) { assert.True(t, tOut.TransactionSubmitted) rsc = it.stateManager.GetRunningStageContext(ctx) assert.NotNil(t, rsc.StageOutputsToBePersisted) - assert.Equal(t, testConfirmation, rsc.StageOutputsToBePersisted.Confirmations) + assert.Equal(t, testConfirmation, rsc.StageOutputsToBePersisted.IndexedTransaction) // persisting error waiting for persistence retry timeout rsc.StageErrored = false @@ -312,7 +312,7 @@ func TestProduceLatestInFlightStageContextSanityChecksForCompletedTransactions(t ProtocolID: "0000/0001", } - imtxs.Receipt = testReceipt + imtxs.IndexedTransaction = testReceipt tOut := it.ProduceLatestInFlightStageContext(ctx, &baseTypes.OrchestratorContext{ AvailableToSpend: nil, PreviousNonceCostUnknown: false, diff --git a/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_receipt_test.go b/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_receipt_test.go index 95f49681a..c5a574547 100644 --- a/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_receipt_test.go +++ b/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_receipt_test.go @@ -142,11 +142,11 @@ func TestProduceLatestInFlightStageContextReceiptingCheckExistingHashes(t *testi // set existing transaction hashes oldHash := mtx.TransactionHash imtxs.policyInfo = &baseTypes.EnterprisePolicyInfo{ - SubmittedTxHashes: []string{oldHash, "hash1"}, + SubmittedHashes: []string{oldHash, "hash1"}, } called := make(chan bool, 3) mEC := it.ethClient.(*componentmocks.EthClient) - mEC.On("GetTransactionReceipt", ctx, "hash1").Run(func(args mock.Arguments) { + mEC.On("GetIndexedTransaction", ctx, "hash1").Run(func(args mock.Arguments) { called <- true }).Return(nil, nil).Once() mTS.On("AddSubStatusAction", mock.Anything, mtx.ID, baseTypes.BaseTxSubStatusTracking, baseTypes.BaseTxActionStateTransition, fftypes.JSONAnyPtr(`{"txHash":"hash1"}`), (*fftypes.JSONAny)(nil), mock.Anything).Return(nil).Maybe() @@ -155,9 +155,9 @@ func TestProduceLatestInFlightStageContextReceiptingCheckExistingHashes(t *testi persistenceCalled <- true }).Return(nil).Maybe() - mCL := testInFlightTransactionStateManagerWithMocks.mCL - addMock := mCL.On("Add", mock.Anything, mtx.ID, "hash1", mock.Anything, mock.Anything) - mCL.On("Remove", mock.Anything, oldHash).Return(nil).Maybe() + mBI := testInFlightTransactionStateManagerWithMocks.mBI + addMock := mBI.On("Add", mock.Anything, mtx.ID, "hash1", mock.Anything, mock.Anything) + mBI.On("Remove", mock.Anything, oldHash).Return(nil).Maybe() eventHandlerCalled := make(chan bool, 3) addMock.Run(func(args mock.Arguments) { addMock.Return(nil) @@ -192,11 +192,11 @@ func TestProduceLatestInFlightStageContextReceiptingCheckExistingHashesPersisten mTS := testInFlightTransactionStateManagerWithMocks.mTS // set existing transaction hashes imtxs.policyInfo = &baseTypes.EnterprisePolicyInfo{ - SubmittedTxHashes: []string{mtx.TransactionHash, "hash1"}, + SubmittedHashes: []string{mtx.TransactionHash, "hash1"}, } called := make(chan bool, 3) mEC := testInFlightTransactionStateManagerWithMocks.mEC - mEC.On("GetTransactionReceipt", ctx, "hash1").Run(func(args mock.Arguments) { + mEC.On("GetIndexedTransaction", ctx, "hash1").Run(func(args mock.Arguments) { called <- true }).Return(nil, nil).Once() mTS.On("AddSubStatusAction", mock.Anything, mtx.ID, baseTypes.BaseTxSubStatusTracking, baseTypes.BaseTxActionStateTransition, fftypes.JSONAnyPtr(`{"txHash":"hash1"}`), (*fftypes.JSONAny)(nil), mock.Anything).Return(nil).Maybe() @@ -233,11 +233,11 @@ func TestProduceLatestInFlightStageContextReceiptingCheckExistingHashesTrackingF // set existing transaction hashes oldHash := mtx.TransactionHash imtxs.policyInfo = &baseTypes.EnterprisePolicyInfo{ - SubmittedTxHashes: []string{oldHash, "hash1"}, + SubmittedHashes: []string{oldHash, "hash1"}, } getReceiptCalled := make(chan bool, 3) mEC := testInFlightTransactionStateManagerWithMocks.mEC - mEC.On("GetTransactionReceipt", ctx, "hash1").Run(func(args mock.Arguments) { + mEC.On("GetIndexedTransaction", ctx, "hash1").Run(func(args mock.Arguments) { getReceiptCalled <- true }).Return(nil, nil).Once() mTS.On("AddSubStatusAction", mock.Anything, mtx.ID, baseTypes.BaseTxSubStatusTracking, baseTypes.BaseTxActionStateTransition, fftypes.JSONAnyPtr(`{"txHash":"hash1"}`), (*fftypes.JSONAny)(nil), mock.Anything).Return(nil).Maybe() @@ -246,9 +246,9 @@ func TestProduceLatestInFlightStageContextReceiptingCheckExistingHashesTrackingF persistenceCalled <- true }).Return(nil).Maybe() - mCL := testInFlightTransactionStateManagerWithMocks.mCL - addMock := mCL.On("Add", mock.Anything, mtx.ID, "hash1", mock.Anything, mock.Anything) - mCL.On("Remove", mock.Anything, oldHash).Return(nil).Maybe() + mBI := testInFlightTransactionStateManagerWithMocks.mBI + addMock := mBI.On("Add", mock.Anything, mtx.ID, "hash1", mock.Anything, mock.Anything) + mBI.On("Remove", mock.Anything, oldHash).Return(nil).Maybe() eventHandlerCalled := make(chan bool, 3) addMock.Run(func(args mock.Arguments) { addMock.Return(fmt.Errorf("failed to add")) @@ -286,10 +286,10 @@ func TestProduceLatestInFlightStageContextReceiptingExceededTimeout(t *testing.T assert.Equal(t, baseTypes.InFlightTxStageReceipting, inFlightStageMananger.stage) - mCL := testInFlightTransactionStateManagerWithMocks.mCL - mCL.On("Add", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() + mBI := testInFlightTransactionStateManagerWithMocks.mBI + mBI.On("Add", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() - removeMock := mCL.On("Remove", mock.Anything, mock.Anything) + removeMock := mBI.On("Remove", mock.Anything, mock.Anything) removeMock.Run(func(args mock.Arguments) { removeMock.Return(nil) }).Maybe() @@ -327,9 +327,9 @@ func TestProduceLatestInFlightStageContextReceiptingExceededTimeoutIgnoreRemoval assert.True(t, tOut.TransactionSubmitted) assert.Equal(t, baseTypes.InFlightTxStageReceipting, inFlightStageMananger.stage) - mCL := testInFlightTransactionStateManagerWithMocks.mCL - removeMock := mCL.On("Remove", mock.Anything, mock.Anything) - mCL.On("Add", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() + mBI := testInFlightTransactionStateManagerWithMocks.mBI + removeMock := mBI.On("Remove", mock.Anything, mock.Anything) + mBI.On("Add", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() removeCalled := make(chan struct{}) removeMock.Run(func(args mock.Arguments) { diff --git a/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_submit_test.go b/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_submit_test.go index 8b4893f1d..fb7fdf560 100644 --- a/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_submit_test.go +++ b/core/go/internal/engine/baseledgertx/in_flight_transaction_stage_controller_test_submit_test.go @@ -76,7 +76,7 @@ func TestProduceLatestInFlightStageContextSubmitComplete(t *testing.T) { mTS := testInFlightTransactionStateManagerWithMocks.mTS mtx.FirstSubmit = nil imtxs.policyInfo = &baseTypes.EnterprisePolicyInfo{ - SubmittedTxHashes: []string{}, + SubmittedHashes: []string{}, } submissionTime := fftypes.Now() @@ -93,7 +93,7 @@ func TestProduceLatestInFlightStageContextSubmitComplete(t *testing.T) { assert.Equal(t, 1, len(rsc.StageOutputsToBePersisted.HistoryUpdates)) mTS.On("AddSubStatusAction", mock.Anything, mtx.ID, baseTypes.BaseTxSubStatusReceived, baseTypes.BaseTxActionSubmitTransaction, fftypes.JSONAnyPtr(`{"txHash":"`+txHash+`"}`), (*fftypes.JSONAny)(nil), mock.Anything).Return(nil).Maybe() _ = rsc.StageOutputsToBePersisted.HistoryUpdates[0](mTS) - assert.Equal(t, []string{txHash}, rsc.StageOutputsToBePersisted.PolicyInfo.SubmittedTxHashes) + assert.Equal(t, []string{txHash}, rsc.StageOutputsToBePersisted.PolicyInfo.SubmittedHashes) assert.Equal(t, submissionTime, rsc.StageOutputsToBePersisted.TxUpdates.FirstSubmit) assert.Equal(t, submissionTime, rsc.StageOutputsToBePersisted.TxUpdates.LastSubmit) @@ -113,7 +113,7 @@ func TestProduceLatestInFlightStageContextSubmitComplete(t *testing.T) { assert.NotNil(t, rsc.StageOutputsToBePersisted) mTS.On("AddSubStatusAction", mock.Anything, mtx.ID, baseTypes.BaseTxSubStatusReceived, baseTypes.BaseTxActionSubmitTransaction, fftypes.JSONAnyPtr(`{"txHash":"`+txHash+`"}`), (*fftypes.JSONAny)(nil), mock.Anything).Return(nil).Maybe() _ = rsc.StageOutputsToBePersisted.HistoryUpdates[0](mTS) - assert.Equal(t, []string{txHash}, rsc.StageOutputsToBePersisted.PolicyInfo.SubmittedTxHashes) + assert.Equal(t, []string{txHash}, rsc.StageOutputsToBePersisted.PolicyInfo.SubmittedHashes) assert.Equal(t, submissionTime, rsc.StageOutputsToBePersisted.TxUpdates.LastSubmit) assert.Equal(t, txHash, *rsc.StageOutputsToBePersisted.TxUpdates.TransactionHash) } @@ -135,7 +135,7 @@ func TestProduceLatestInFlightStageContextCannotSubmit(t *testing.T) { mtx.TransactionHash = "" mtx.FirstSubmit = nil imtxs.policyInfo = &baseTypes.EnterprisePolicyInfo{ - SubmittedTxHashes: []string{}, + SubmittedHashes: []string{}, } tOut := it.ProduceLatestInFlightStageContext(ctx, &baseTypes.OrchestratorContext{ @@ -150,7 +150,7 @@ func TestProduceLatestInFlightStageContextCannotSubmit(t *testing.T) { inFlightStageMananger.bufferedStageOutputs = make([]*baseTypes.StageOutput, 0) mtx.TransactionHash = "test" imtxs.policyInfo = &baseTypes.EnterprisePolicyInfo{ - SubmittedTxHashes: []string{}, + SubmittedHashes: []string{}, } mtx.GasLimit = ethtypes.NewHexInteger64(-1) // invalid limit it.stateManager.SetValidatedTransactionHashMatchState(ctx, false) @@ -180,7 +180,7 @@ func TestProduceLatestInFlightStageContextSubmitCompleteAlreadyKnown(t *testing. mtx.TransactionHash = "" mtx.FirstSubmit = nil imtxs.policyInfo = &baseTypes.EnterprisePolicyInfo{ - SubmittedTxHashes: []string{}, + SubmittedHashes: []string{}, } submissionTime := fftypes.Now() @@ -190,7 +190,7 @@ func TestProduceLatestInFlightStageContextSubmitCompleteAlreadyKnown(t *testing. mtx.FirstSubmit = fftypes.Now() rsc = it.stateManager.GetRunningStageContext(ctx) imtxs.policyInfo = &baseTypes.EnterprisePolicyInfo{ - SubmittedTxHashes: []string{txHash}, + SubmittedHashes: []string{txHash}, } it.stateManager.AddSubmitOutput(ctx, txHash, submissionTime, baseTypes.SubmissionOutcomeAlreadyKnown, ethclient.ErrorReason(""), nil) tOut := it.ProduceLatestInFlightStageContext(ctx, &baseTypes.OrchestratorContext{ @@ -202,7 +202,7 @@ func TestProduceLatestInFlightStageContextSubmitCompleteAlreadyKnown(t *testing. assert.Equal(t, baseTypes.InFlightTxStageSubmitting, rsc.Stage) assert.NotNil(t, rsc.StageOutputsToBePersisted) assert.Empty(t, rsc.StageOutputsToBePersisted.HistoryUpdates) - assert.Equal(t, []string{txHash}, rsc.StageOutputsToBePersisted.PolicyInfo.SubmittedTxHashes) + assert.Equal(t, []string{txHash}, rsc.StageOutputsToBePersisted.PolicyInfo.SubmittedHashes) } func TestProduceLatestInFlightStageContextSubmitErrors(t *testing.T) { ctx := context.Background() @@ -222,7 +222,7 @@ func TestProduceLatestInFlightStageContextSubmitErrors(t *testing.T) { mtx.TransactionHash = "" mtx.FirstSubmit = nil imtxs.policyInfo = &baseTypes.EnterprisePolicyInfo{ - SubmittedTxHashes: []string{}, + SubmittedHashes: []string{}, } submissionTime := fftypes.Now() @@ -339,7 +339,7 @@ func TestProduceLatestInFlightStageContextSubmitRePrepare(t *testing.T) { mtx.TransactionHash = "" mtx.FirstSubmit = nil imtxs.policyInfo = &baseTypes.EnterprisePolicyInfo{ - SubmittedTxHashes: []string{}, + SubmittedHashes: []string{}, } // persisted stage error - require re-preparation @@ -374,7 +374,7 @@ func TestProduceLatestInFlightStageContextSubmitSuccess(t *testing.T) { imtxs := inFlightStageMananger.InMemoryTxStateManager.(*inMemoryTxState) imtxs.policyInfo = &baseTypes.EnterprisePolicyInfo{ - SubmittedTxHashes: []string{}, + SubmittedHashes: []string{}, } // persisted stage error - require re-preparation diff --git a/core/go/internal/engine/baseledgertx/in_flight_transaction_state_manager.go b/core/go/internal/engine/baseledgertx/in_flight_transaction_state_manager.go index 112078eba..eb301ff0c 100644 --- a/core/go/internal/engine/baseledgertx/in_flight_transaction_state_manager.go +++ b/core/go/internal/engine/baseledgertx/in_flight_transaction_state_manager.go @@ -17,31 +17,33 @@ package baseledgertx import ( "context" - "encoding/json" "math/big" "sync" "time" "github.com/hyperledger/firefly-common/pkg/fftypes" "github.com/hyperledger/firefly-common/pkg/i18n" + "github.com/hyperledger/firefly-common/pkg/retry" baseTypes "github.com/kaleido-io/paladin/core/internal/engine/enginespi" "github.com/kaleido-io/paladin/core/internal/msgs" + "github.com/kaleido-io/paladin/core/pkg/blockindexer" "github.com/kaleido-io/paladin/core/pkg/ethclient" "github.com/kaleido-io/paladin/toolkit/pkg/log" + "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" ) type inFlightTransactionState struct { testMode bool // Note: this flag can never be set in normal code path, exposed for testing only + retry *retry.Retry BaseLedgerTxEngineMetricsManager baseTypes.BalanceManager - txStore baseTypes.TransactionStore - txConfirmationListener baseTypes.TransactionConfirmationListener + txStore baseTypes.TransactionStore + bIndexer blockindexer.BlockIndexer // input that should be set once the stage is running *baseTypes.TransientPreviousStageOutputs orchestratorContext *baseTypes.OrchestratorContext - baseTypes.InFlightStageActionTriggers baseTypes.InMemoryTxStateManager @@ -112,8 +114,8 @@ func (iftxs *inFlightTransactionState) StartNewStageContext(ctx context.Context, } iftxs.stageTriggerError = iftxs.TriggerSubmitTx(ctx, signedMessage) case baseTypes.InFlightTxStageReceipting: - log.L(ctx).Tracef("Transaction with ID %s, triggering receipting", rsc.InMemoryTx.GetTxID()) - iftxs.stageTriggerError = iftxs.TriggerTracking(ctx) + log.L(ctx).Tracef("Transaction with ID %s, entered confirmation tracking, waiting for block indexer to index the transaction", rsc.InMemoryTx.GetTxID()) + // no action required, relies on block indexer to trigger events case baseTypes.InFlightTxStageStatusUpdate: log.L(ctx).Tracef("Transaction with ID %s, triggering status update", rsc.InMemoryTx.GetTxID()) iftxs.stageTriggerError = iftxs.TriggerStatusUpdate(ctx) @@ -183,18 +185,20 @@ func (iftxs *inFlightTransactionState) AddStageOutputs(ctx context.Context, stag func NewInFlightTransactionStateManager(thm BaseLedgerTxEngineMetricsManager, bm baseTypes.BalanceManager, txStore baseTypes.TransactionStore, - txConfirmationListener baseTypes.TransactionConfirmationListener, + bIndexer blockindexer.BlockIndexer, ifsat baseTypes.InFlightStageActionTriggers, imtxs baseTypes.InMemoryTxStateManager, + retry *retry.Retry, turnOffHistory bool, noEventMode bool, ) baseTypes.InFlightTransactionStateManager { return &inFlightTransactionState{ testMode: noEventMode, + retry: retry, BaseLedgerTxEngineMetricsManager: thm, BalanceManager: bm, txStore: txStore, - txConfirmationListener: txConfirmationListener, + bIndexer: bIndexer, InFlightStageActionTriggers: ifsat, bufferedStageOutputs: make([]*baseTypes.StageOutput, 0), txLevelStageStartTime: time.Now(), @@ -260,26 +264,12 @@ func (iftxs *inFlightTransactionState) AddGasPriceOutput(ctx context.Context, ga log.L(ctx).Debugf("%s AddGasPriceOutput took %s to write the result", iftxs.InMemoryTxStateManager.GetTxID(), time.Since(start)) } -func (iftxs *inFlightTransactionState) AddReceiptOutput(ctx context.Context, rpt *ethclient.TransactionReceiptResponse, err error) { - start := time.Now() - iftxs.AddStageOutputs(ctx, &baseTypes.StageOutput{ - Stage: baseTypes.InFlightTxStageReceipting, - ReceiptOutput: &baseTypes.ReceiptOutputs{ - Receipt: rpt, - ReceiptNotify: fftypes.Now(), - Err: err, - }, - }) - log.L(ctx).Debugf("%s AddReceiptOutput took %s to write the result", iftxs.InMemoryTxStateManager.GetTxID(), time.Since(start)) -} - -func (iftxs *inFlightTransactionState) AddConfirmationsOutput(ctx context.Context, cmfs *baseTypes.ConfirmationsNotification) { +func (iftxs *inFlightTransactionState) AddConfirmationsOutput(ctx context.Context, indexedTx *blockindexer.IndexedTransaction) { start := time.Now() iftxs.AddStageOutputs(ctx, &baseTypes.StageOutput{ Stage: baseTypes.InFlightTxStageConfirming, ConfirmationOutput: &baseTypes.ConfirmationOutputs{ - Confirmations: cmfs, - ConfirmNotify: fftypes.Now(), + IndexedTransaction: indexedTx, }, }) log.L(ctx).Debugf("%s AddConfirmationsOutput took %s to write the result", iftxs.InMemoryTxStateManager.GetTxID(), time.Since(start)) @@ -304,50 +294,32 @@ func (iftxs *inFlightTransactionState) PersistTxState(ctx context.Context) (stag } switch rsc.StageOutputsToBePersisted.UpdateType { case baseTypes.PersistenceUpdateUpdate: - if rsc.StageOutputsToBePersisted.PolicyInfo != nil { - if rsc.StageOutputsToBePersisted.TxUpdates == nil { - rsc.StageOutputsToBePersisted.TxUpdates = &baseTypes.BaseTXUpdates{} - } - infoBytes, _ := json.Marshal(rsc.StageOutputsToBePersisted.PolicyInfo) - rsc.StageOutputsToBePersisted.TxUpdates.PolicyInfo = fftypes.JSONAnyPtrBytes(infoBytes) - } - - if rsc.StageOutputsToBePersisted.Receipt != nil { + it := rsc.StageOutputsToBePersisted.IndexedTransaction + if it != nil { iftxs.NotifyAddressBalanceChanged(ctx, string(mtx.From)) if mtx.Value != nil && mtx.To != nil { iftxs.NotifyAddressBalanceChanged(ctx, mtx.To.String()) } - if err = iftxs.txStore.SetTransactionReceipt(ctx, mtx.ID, rsc.StageOutputsToBePersisted.Receipt); err != nil { - log.L(ctx).Errorf("Failed to persist receipt for transaction %s due to error: %+v, receipt: %+v", mtx.ID, err, rsc.StageOutputsToBePersisted.Receipt) + if err = iftxs.txStore.SetIndexedTransaction(ctx, mtx.ID, it); err != nil { + log.L(ctx).Errorf("Failed to persist receipt for transaction %s due to error: %+v, receipt: %+v", mtx.ID, err, it) return rsc.Stage, time.Now(), err } // update the in memory state - iftxs.SetReceipt(ctx, rsc.StageOutputsToBePersisted.Receipt) - } + iftxs.SetIndexedTransaction(ctx, it) - if rsc.StageOutputsToBePersisted.Confirmations != nil { - if err := iftxs.txStore.AddTransactionConfirmations(ctx, mtx.ID, rsc.StageOutputsToBePersisted.Confirmations.NewFork, rsc.StageOutputsToBePersisted.Confirmations.Confirmations...); err != nil { - log.L(ctx).Errorf("Failed to persist confirmations for transaction %s due to error: %+v, confirmations: %+v", mtx.ID, err, rsc.StageOutputsToBePersisted.Confirmations) - return rsc.Stage, time.Now(), err - } - if rsc.StageOutputsToBePersisted.Confirmations.Confirmed { - if rsc.StageOutputsToBePersisted.TxUpdates == nil { - rsc.StageOutputsToBePersisted.TxUpdates = &baseTypes.BaseTXUpdates{} - } - if rsc.SubStatus != baseTypes.BaseTxSubStatusConfirmed { - rsc.StageOutputsToBePersisted.AddSubStatusAction(baseTypes.BaseTxActionConfirmTransaction, nil, nil) - rsc.SetSubStatus(baseTypes.BaseTxSubStatusConfirmed) - } - if iftxs.GetReceipt() == nil { - receipt, err := iftxs.txStore.GetTransactionReceipt(ctx, mtx.ID) - if err != nil { - log.L(ctx).Errorf("Failed to retrieve receipt for confirmation check for transaction %s due to error: %+v", mtx.ID, err) - return rsc.Stage, time.Now(), err - } - iftxs.SetReceipt(ctx, receipt) + // check whether the hash of the indexed transaction matches one of the tracking hashes + trackedHashes := iftxs.GetSubmittedHashes() + + matchFound := false + for _, h := range trackedHashes { + if h == it.Hash.String() { + matchFound = true + break } - recpt := iftxs.GetReceipt() - if recpt != nil && recpt.Success { + } + + if matchFound { + if it.Result == blockindexer.TXResult_SUCCESS.Enum() { mtx.Status = baseTypes.BaseTxStatusSucceeded rsc.StageOutputsToBePersisted.TxUpdates.Status = &mtx.Status iftxs.RecordCompletedTransactionCountMetrics(ctx, string(GenericStatusSuccess)) @@ -356,6 +328,23 @@ func (iftxs *inFlightTransactionState) PersistTxState(ctx context.Context) (stag rsc.StageOutputsToBePersisted.TxUpdates.Status = &mtx.Status iftxs.RecordCompletedTransactionCountMetrics(ctx, string(GenericStatusFail)) } + } else { + mtx.Status = baseTypes.BaseTxStatusConflict + rsc.StageOutputsToBePersisted.TxUpdates.Status = &mtx.Status + iftxs.RecordCompletedTransactionCountMetrics(ctx, string(GenericStatusConflict)) + } + } else if rsc.StageOutputsToBePersisted.MissedIndexedTransaction { + err := iftxs.retry.Do(ctx, "get indexed transaction for "+mtx.ID, func(attempt int) (retry bool, err error) { + tx, err := iftxs.bIndexer.GetIndexedTransactionByNonce(ctx, *tktypes.MustEthAddress(string(mtx.From)), mtx.Nonce.Uint64()) + if tx == nil { + // panic("block indexer missed a nonce") + // the logic is in a confirmation loop until block indexer indexed the missing transaction + return false, i18n.NewError(ctx, msgs.MsgMissingIndexedTransaction, mtx.ID) + } + return true, err + }) + if err != nil { + return rsc.Stage, time.Now(), err } } if !iftxs.turnOffHistory { @@ -366,7 +355,6 @@ func (iftxs *inFlightTransactionState) PersistTxState(ctx context.Context) (stag } } } - oldTxHash := mtx.TransactionHash if rsc.StageOutputsToBePersisted.TxUpdates != nil { err := iftxs.txStore.UpdateTransaction(ctx, mtx.ID, rsc.StageOutputsToBePersisted.TxUpdates) if err != nil { @@ -376,14 +364,6 @@ func (iftxs *inFlightTransactionState) PersistTxState(ctx context.Context) (stag // update the in memory state iftxs.ApplyTxUpdates(ctx, rsc.StageOutputsToBePersisted.TxUpdates) } - if oldTxHash != "" && oldTxHash != mtx.TransactionHash { - // if had a previous transaction hash, emit an event to for transaction hash removal - log.L(ctx).Debugf("Cancelling confirmation manager tracking of stale hash for TX %s oldHash=%s newHash=%s", mtx.ID, oldTxHash, mtx.TransactionHash) - if err := iftxs.txConfirmationListener.Remove(ctx, oldTxHash); err != nil { - // Unexpected error, ignore as fail to remove the listener is not a stop deal - log.L(ctx).Errorf("Error detected notifying confirmation manager to remove old transaction hash: %s", err.Error()) - } - } } return rsc.Stage, time.Now(), nil } diff --git a/core/go/internal/engine/baseledgertx/in_flight_transaction_state_manager_test.go b/core/go/internal/engine/baseledgertx/in_flight_transaction_state_manager_test.go index 76328fa91..b2cdad1a5 100644 --- a/core/go/internal/engine/baseledgertx/in_flight_transaction_state_manager_test.go +++ b/core/go/internal/engine/baseledgertx/in_flight_transaction_state_manager_test.go @@ -33,11 +33,10 @@ import ( ) type testInFlightTransactionStateManagerWithMocks struct { - stateManager baseTypes.InFlightTransactionStateManager - mEC *componentmocks.EthClient - mTS *enginemocks.TransactionStore - mCL *enginemocks.TransactionConfirmationListener - + stateManager baseTypes.InFlightTransactionStateManager + mEC *componentmocks.EthClient + mTS *enginemocks.TransactionStore + mBI *componentmocks.BlockIndexer mBM baseTypes.BalanceManager mAT *enginemocks.InFlightStageActionTriggers inMemoryTxState baseTypes.InMemoryTxStateManager @@ -48,13 +47,13 @@ func newTestInFlightTransactionStateManager(t *testing.T) *testInFlightTransacti mockInMemoryState := NewTestInMemoryTxState(t) mockActionTriggers := enginemocks.NewInFlightStageActionTriggers(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) - iftxs := NewInFlightTransactionStateManager(&baseLedgerTxEngineMetrics{}, mBM, mTS, mCL, mockActionTriggers, mockInMemoryState, false, false) + mBI := componentmocks.NewBlockIndexer(t) + iftxs := NewInFlightTransactionStateManager(&baseLedgerTxEngineMetrics{}, mBM, mTS, mBI, mockActionTriggers, mockInMemoryState, false, false) return &testInFlightTransactionStateManagerWithMocks{ iftxs, mEC, mTS, - mCL, + mBI, mBM, mockActionTriggers, mockInMemoryState, @@ -331,8 +330,8 @@ func TestStateManagerStageOutputManagement(t *testing.T) { } else { actualNumberOfSignErrorOutput++ } - } else if stageOutput.ReceiptOutput != nil { - if stageOutput.ReceiptOutput.Err == nil { + } else if stageOutput.ConfirmationOutput != nil { + if stageOutput.ConfirmationOutput.Err == nil { actualNumberOfReceiptSuccessOutput++ } else { actualNumberOfReceiptErrorOutput++ @@ -409,8 +408,8 @@ func TestStateManagerTxPersistenceManagementTransactionConfirmed(t *testing.T) { newGas := ethtypes.NewHexInteger64(111) newGasPrice := ethtypes.NewHexInteger64(111) newTestPolicyInfo := &baseTypes.EnterprisePolicyInfo{ - LastWarnTime: newTime, - SubmittedTxHashes: []string{"0x00000", "0x00001", "0x00002", "0x00003"}, + LastWarnTime: newTime, + SubmittedHashes: []string{"0x00000", "0x00001", "0x00002", "0x00003"}, } testReceipt := ethclient.TransactionReceiptResponse{ BlockNumber: fftypes.NewFFBigInt(1233), @@ -428,7 +427,7 @@ func TestStateManagerTxPersistenceManagementTransactionConfirmed(t *testing.T) { } rsc.StageOutputsToBePersisted.PolicyInfo = newTestPolicyInfo - rsc.StageOutputsToBePersisted.Confirmations = testConfirmation + rsc.StageOutputsToBePersisted.IndexedTransaction = testConfirmation rsc.StageOutputsToBePersisted.Receipt = &testReceipt rsc.StageOutputsToBePersisted.TxUpdates = &baseTypes.BaseTXUpdates{ Status: &newStatus, @@ -440,19 +439,19 @@ func TestStateManagerTxPersistenceManagementTransactionConfirmed(t *testing.T) { GasLimit: newGas, } mTS := testStateManagerWithMocks.mTS - mCL := testStateManagerWithMocks.mCL + mBI := testStateManagerWithMocks.mBI txID := stateManager.GetTxID() mTS.On("AddTransactionConfirmations", mock.Anything, txID, testConfirmation.NewFork, testConfirmation.Confirmations[0]).Return(nil).Once() - mTS.On("SetTransactionReceipt", mock.Anything, txID, &testReceipt).Return(nil).Once() + mTS.On("SetIndexedTransaction", mock.Anything, txID, &testReceipt).Return(nil).Once() mTS.On("AddSubStatusAction", mock.Anything, txID, baseTypes.BaseTxSubStatusTracking, baseTypes.BaseTxActionConfirmTransaction, (*fftypes.JSONAny)(nil), (*fftypes.JSONAny)(nil), mock.Anything).Return(nil).Once() mTS.On("UpdateTransaction", mock.Anything, txID, rsc.StageOutputsToBePersisted.TxUpdates).Return(nil).Once() - mCL.On("Remove", mock.Anything, mock.Anything).Return(nil).Once() + mBI.On("Remove", mock.Anything, mock.Anything).Return(nil).Once() _, _, err = stateManager.PersistTxState(ctx) assert.Nil(t, err) assert.Equal(t, stateManager.GetTxID(), inMemoryTxState.GetTxID()) assert.Equal(t, newTime, inMemoryTxState.GetDeleteRequestedTime()) - assert.Equal(t, testReceipt, *inMemoryTxState.GetReceipt()) + assert.Equal(t, testReceipt, *inMemoryTxState.GetIndexedTransaction()) assert.Equal(t, newTxHash, inMemoryTxState.GetTransactionHash()) assert.Equal(t, newStatus, inMemoryTxState.GetStatus()) assert.Equal(t, newGasPrice.BigInt(), inMemoryTxState.GetGasPriceObject().GasPrice) @@ -506,8 +505,8 @@ func TestStateManagerTxPersistenceManagementTransactionConfirmedRetrieveReceiptA newGas := ethtypes.NewHexInteger64(111) newGasPrice := ethtypes.NewHexInteger64(111) newTestPolicyInfo := &baseTypes.EnterprisePolicyInfo{ - LastWarnTime: newTime, - SubmittedTxHashes: []string{"0x00000", "0x00001", "0x00002", "0x00003"}, + LastWarnTime: newTime, + SubmittedHashes: []string{"0x00000", "0x00001", "0x00002", "0x00003"}, } testReceipt := ethclient.TransactionReceiptResponse{ BlockNumber: fftypes.NewFFBigInt(1233), @@ -525,7 +524,7 @@ func TestStateManagerTxPersistenceManagementTransactionConfirmedRetrieveReceiptA } rsc.StageOutputsToBePersisted.PolicyInfo = newTestPolicyInfo - rsc.StageOutputsToBePersisted.Confirmations = testConfirmation + rsc.StageOutputsToBePersisted.IndexedTransaction = testConfirmation rsc.StageOutputsToBePersisted.TxUpdates = &baseTypes.BaseTXUpdates{ Status: &newStatus, DeleteRequested: newTime, @@ -537,19 +536,19 @@ func TestStateManagerTxPersistenceManagementTransactionConfirmedRetrieveReceiptA } mTS := testStateManagerWithMocks.mTS - mCL := testStateManagerWithMocks.mCL + mBI := testStateManagerWithMocks.mBI txID := stateManager.GetTxID() mTS.On("AddTransactionConfirmations", mock.Anything, txID, testConfirmation.NewFork, testConfirmation.Confirmations[0]).Return(nil).Once() - mTS.On("GetTransactionReceipt", mock.Anything, txID).Return(&testReceipt, nil).Once() + mTS.On("GetIndexedTransaction", mock.Anything, txID).Return(&testReceipt, nil).Once() mTS.On("AddSubStatusAction", mock.Anything, txID, baseTypes.BaseTxSubStatusTracking, baseTypes.BaseTxActionConfirmTransaction, (*fftypes.JSONAny)(nil), (*fftypes.JSONAny)(nil), mock.Anything).Return(nil).Once() mTS.On("UpdateTransaction", mock.Anything, txID, rsc.StageOutputsToBePersisted.TxUpdates).Return(nil).Once() - mCL.On("Remove", mock.Anything, oldHash).Return(nil).Once() + mBI.On("Remove", mock.Anything, oldHash).Return(nil).Once() _, _, err = stateManager.PersistTxState(ctx) assert.Nil(t, err) assert.Equal(t, stateManager.GetTxID(), inMemoryTxState.GetTxID()) assert.Equal(t, newTime, inMemoryTxState.GetDeleteRequestedTime()) - assert.Equal(t, testReceipt, *inMemoryTxState.GetReceipt()) + assert.Equal(t, testReceipt, *inMemoryTxState.GetIndexedTransaction()) assert.Equal(t, newTxHash, inMemoryTxState.GetTransactionHash()) assert.Equal(t, newStatus, inMemoryTxState.GetStatus()) assert.Equal(t, newGasPrice.BigInt(), inMemoryTxState.GetGasPriceObject().GasPrice) @@ -598,8 +597,8 @@ func TestStateManagerTxPersistenceManagementUpdateErrors(t *testing.T) { ProtocolID: "000000000/0023", } newTestPolicyInfo := &baseTypes.EnterprisePolicyInfo{ - LastWarnTime: fftypes.Now(), - SubmittedTxHashes: []string{"0x00000", "0x00001", "0x00002", "0x00003"}, + LastWarnTime: fftypes.Now(), + SubmittedHashes: []string{"0x00000", "0x00001", "0x00002", "0x00003"}, } testConfirmation := &baseTypes.ConfirmationsNotification{ Confirmed: true, @@ -610,16 +609,16 @@ func TestStateManagerTxPersistenceManagementUpdateErrors(t *testing.T) { } rsc.StageOutputsToBePersisted.PolicyInfo = newTestPolicyInfo rsc.StageOutputsToBePersisted.Receipt = &testReceipt - rsc.StageOutputsToBePersisted.Confirmations = testConfirmation + rsc.StageOutputsToBePersisted.IndexedTransaction = testConfirmation mTS := testStateManagerWithMocks.mTS - mCL := testStateManagerWithMocks.mCL + mBI := testStateManagerWithMocks.mBI txID := stateManager.GetTxID() // set receipt fail - mTS.On("SetTransactionReceipt", mock.Anything, txID, &testReceipt).Return(fmt.Errorf("SetTransactionReceipt error")).Once() + mTS.On("SetIndexedTransaction", mock.Anything, txID, &testReceipt).Return(fmt.Errorf("SetIndexedTransaction error")).Once() _, _, err = stateManager.PersistTxState(ctx) assert.NotNil(t, err) - assert.Regexp(t, "SetTransactionReceipt error", err) + assert.Regexp(t, "SetIndexedTransaction error", err) // add transaction confirmations fail rsc.StageOutputsToBePersisted.PolicyInfo = nil @@ -632,14 +631,14 @@ func TestStateManagerTxPersistenceManagementUpdateErrors(t *testing.T) { // get receipt fail mTS.On("AddTransactionConfirmations", mock.Anything, txID, testConfirmation.NewFork, testConfirmation.Confirmations[0]).Return(nil).Once() - mTS.On("GetTransactionReceipt", mock.Anything, txID).Return(nil, fmt.Errorf("GetTransactionReceipt error")).Once() + mTS.On("GetIndexedTransaction", mock.Anything, txID).Return(nil, fmt.Errorf("GetIndexedTransaction error")).Once() _, _, err = stateManager.PersistTxState(ctx) assert.NotNil(t, err) - assert.Regexp(t, "GetTransactionReceipt error", err) + assert.Regexp(t, "GetIndexedTransaction error", err) // history update fail mTS.On("AddTransactionConfirmations", mock.Anything, txID, testConfirmation.NewFork, testConfirmation.Confirmations[0]).Return(nil).Once() - mTS.On("GetTransactionReceipt", mock.Anything, txID).Return(&testReceipt, nil).Once() + mTS.On("GetIndexedTransaction", mock.Anything, txID).Return(&testReceipt, nil).Once() mTS.On("AddSubStatusAction", mock.Anything, txID, baseTypes.BaseTxSubStatusTracking, baseTypes.BaseTxActionConfirmTransaction, (*fftypes.JSONAny)(nil), (*fftypes.JSONAny)(nil), mock.Anything).Return(fmt.Errorf("AddSubStatusAction error")).Once() _, _, err = stateManager.PersistTxState(ctx) assert.NotNil(t, err) @@ -661,7 +660,7 @@ func TestStateManagerTxPersistenceManagementUpdateErrors(t *testing.T) { mTS.On("AddTransactionConfirmations", mock.Anything, txID, testConfirmation.NewFork, testConfirmation.Confirmations[0]).Return(nil).Once() mTS.On("AddSubStatusAction", mock.Anything, txID, baseTypes.BaseTxSubStatusTracking, baseTypes.BaseTxActionConfirmTransaction, (*fftypes.JSONAny)(nil), (*fftypes.JSONAny)(nil), mock.Anything).Return(nil).Once() mTS.On("UpdateTransaction", mock.Anything, txID, rsc.StageOutputsToBePersisted.TxUpdates).Return(nil).Once() - mCL.On("Remove", mock.Anything, mock.Anything).Return(fmt.Errorf("HandleEvent error")).Once() + mBI.On("Remove", mock.Anything, mock.Anything).Return(fmt.Errorf("HandleEvent error")).Once() _, _, err = stateManager.PersistTxState(ctx) assert.Nil(t, err) // failure to delete existing hash tracking is ignored } diff --git a/core/go/internal/engine/baseledgertx/in_memory_tx_state.go b/core/go/internal/engine/baseledgertx/in_memory_tx_state.go index 06b93be0e..9b1007b9c 100644 --- a/core/go/internal/engine/baseledgertx/in_memory_tx_state.go +++ b/core/go/internal/engine/baseledgertx/in_memory_tx_state.go @@ -17,7 +17,6 @@ package baseledgertx import ( "context" - "encoding/json" "math/big" "time" @@ -25,7 +24,7 @@ import ( "github.com/hyperledger/firefly-common/pkg/fftypes" baseTypes "github.com/kaleido-io/paladin/core/internal/engine/enginespi" - "github.com/kaleido-io/paladin/core/pkg/ethclient" + "github.com/kaleido-io/paladin/core/pkg/blockindexer" ) type inMemoryTxState struct { @@ -34,28 +33,22 @@ type inMemoryTxState struct { // the value of the following properties are populated during transaction processing but not during initialization // the process logic will determine whether receipt / confirmations requires to be fetched - Receipt *ethclient.TransactionReceiptResponse - Confirmations []*baseTypes.Confirmation + IndexedTransaction *blockindexer.IndexedTransaction - // History []*baseTypes.TxHistoryStateTransitionEntry , There shouldn't be a scenario to read history entries using the in memory state - // If there is logic requires historical data, consider extending the policyInfo to store them instead - - // Following attributes are metadata derived from the on disk state for coding convenience - policyInfo *baseTypes.EnterprisePolicyInfo // The enterprise policy policyInfo parsed from the managed transaction PolicyInfo JSON object + // submitted transaction hashes are in a separate DB table, we load and manage it in memory in the same object for code convenience + SubmittedHashes []string } func NewInMemoryTxStateMananger(ctx context.Context, mtx *baseTypes.ManagedTX) baseTypes.InMemoryTxStateManager { - var info baseTypes.EnterprisePolicyInfo - _ = json.Unmarshal(mtx.PolicyInfo.Bytes(), &info) return &inMemoryTxState{ - mtx: mtx, - policyInfo: &info, + mtx: mtx, } } -func (imtxs *inMemoryTxState) SetReceipt(ctx context.Context, receipt *ethclient.TransactionReceiptResponse) { - imtxs.Receipt = receipt +func (imtxs *inMemoryTxState) SetIndexedTransaction(ctx context.Context, iTX *blockindexer.IndexedTransaction) { + imtxs.IndexedTransaction = iTX } + func (imtxs *inMemoryTxState) ApplyTxUpdates(ctx context.Context, txUpdates *baseTypes.BaseTXUpdates) { mtx := imtxs.mtx @@ -103,13 +96,6 @@ func (imtxs *inMemoryTxState) ApplyTxUpdates(ctx context.Context, txUpdates *bas mtx.LastSubmit = txUpdates.LastSubmit } - if txUpdates.PolicyInfo != nil { - var info baseTypes.EnterprisePolicyInfo - _ = json.Unmarshal(txUpdates.PolicyInfo.Bytes(), &info) - mtx.PolicyInfo = txUpdates.PolicyInfo - imtxs.policyInfo = &info - } - if txUpdates.Status != nil { mtx.Status = *txUpdates.Status } @@ -170,15 +156,19 @@ func (imtxs *inMemoryTxState) GetGasPriceObject() *baseTypes.GasPriceObject { } return gpo } -func (imtxs *inMemoryTxState) GetPolicyInfo() *baseTypes.EnterprisePolicyInfo { - return imtxs.policyInfo +func (imtxs *inMemoryTxState) GetLastSubmitTime() *fftypes.FFTime { + return imtxs.mtx.LastSubmit +} + +func (imtxs *inMemoryTxState) GetSubmittedHashes() []string { + return imtxs.SubmittedHashes } func (imtxs *inMemoryTxState) GetGasLimit() *big.Int { return imtxs.mtx.GasLimit.BigInt() } -func (imtxs *inMemoryTxState) GetReceipt() *ethclient.TransactionReceiptResponse { - return imtxs.Receipt +func (imtxs *inMemoryTxState) GetIndexedTransaction() *blockindexer.IndexedTransaction { + return imtxs.IndexedTransaction } func (imtxs *inMemoryTxState) IsComplete() bool { diff --git a/core/go/internal/engine/baseledgertx/in_memory_tx_state_test.go b/core/go/internal/engine/baseledgertx/in_memory_tx_state_test.go index 55369e301..127d8e807 100644 --- a/core/go/internal/engine/baseledgertx/in_memory_tx_state_test.go +++ b/core/go/internal/engine/baseledgertx/in_memory_tx_state_test.go @@ -42,8 +42,8 @@ func NewTestInMemoryTxState(t *testing.T) baseTypes.InMemoryTxStateManager { oldGasPrice := ethtypes.NewHexInteger64(10) oldErrorMessage := "old message" oldTestPolicyInfo := &baseTypes.EnterprisePolicyInfo{ - LastWarnTime: oldTime, - SubmittedTxHashes: []string{"0x00000", "0x00001", "0x00002"}, + LastWarnTime: oldTime, + SubmittedHashes: []string{"0x00000", "0x00001", "0x00002"}, } oldTransactionData := ethtypes.MustNewHexBytes0xPrefix(testTransactionData) oldTestPolicyInfoBytes, _ := json.Marshal(oldTestPolicyInfo) @@ -83,8 +83,8 @@ func TestSettersAndGetters(t *testing.T) { oldGasPrice := ethtypes.NewHexInteger64(10) oldErrorMessage := "old message" oldTestPolicyInfo := &baseTypes.EnterprisePolicyInfo{ - LastWarnTime: oldTime, - SubmittedTxHashes: []string{"0x00000", "0x00001", "0x00002"}, + LastWarnTime: oldTime, + SubmittedHashes: []string{"0x00000", "0x00001", "0x00002"}, } oldTransactionData := ethtypes.MustNewHexBytes0xPrefix(testTransactionData) @@ -118,7 +118,7 @@ func TestSettersAndGetters(t *testing.T) { assert.Equal(t, oldTime, inMemoryTxState.GetCreatedTime()) assert.Equal(t, oldTime, inMemoryTxState.GetDeleteRequestedTime()) - assert.Nil(t, inMemoryTxState.GetReceipt()) + assert.Nil(t, inMemoryTxState.GetIndexedTransaction()) assert.Equal(t, oldTxHash, inMemoryTxState.GetTransactionHash()) assert.Equal(t, oldNonce.BigInt(), inMemoryTxState.GetNonce()) assert.Equal(t, oldFrom, inMemoryTxState.GetFrom()) @@ -139,8 +139,8 @@ func TestSettersAndGetters(t *testing.T) { ProtocolID: "000000000/0023", } - inMemoryTxState.SetReceipt(context.Background(), &testReceipt) - assert.Equal(t, testReceipt, *inMemoryTxState.GetReceipt()) + inMemoryTxState.SetIndexedTransaction(context.Background(), &testReceipt) + assert.Equal(t, testReceipt, *inMemoryTxState.GetIndexedTransaction()) successStatus := baseTypes.BaseTxStatusSucceeded newTime := fftypes.Now() newFrom := "0xf1031" @@ -152,8 +152,8 @@ func TestSettersAndGetters(t *testing.T) { newGasPrice := ethtypes.NewHexInteger64(111) newErrorMessage := "new message" newTestPolicyInfo := &baseTypes.EnterprisePolicyInfo{ - LastWarnTime: newTime, - SubmittedTxHashes: []string{"0x00000", "0x00001", "0x00002", "0x00003"}, + LastWarnTime: newTime, + SubmittedHashes: []string{"0x00000", "0x00001", "0x00002", "0x00003"}, } newTestPolicyInfoBytes, _ := json.Marshal(newTestPolicyInfo) inMemoryTxState.ApplyTxUpdates(context.Background(), &baseTypes.BaseTXUpdates{ @@ -177,7 +177,7 @@ func TestSettersAndGetters(t *testing.T) { assert.Equal(t, oldTime, inMemoryTxState.GetCreatedTime()) assert.Equal(t, newTime, inMemoryTxState.GetDeleteRequestedTime()) - assert.Equal(t, testReceipt, *inMemoryTxState.GetReceipt()) + assert.Equal(t, testReceipt, *inMemoryTxState.GetIndexedTransaction()) assert.Equal(t, newTxHash, inMemoryTxState.GetTransactionHash()) assert.Equal(t, successStatus, inMemoryTxState.GetStatus()) assert.Equal(t, newGasPrice.BigInt(), inMemoryTxState.GetGasPriceObject().GasPrice) diff --git a/core/go/internal/engine/baseledgertx/transaction_engine.go b/core/go/internal/engine/baseledgertx/transaction_engine.go index 7c5907148..ffd2c8f3e 100644 --- a/core/go/internal/engine/baseledgertx/transaction_engine.go +++ b/core/go/internal/engine/baseledgertx/transaction_engine.go @@ -29,6 +29,7 @@ import ( "github.com/hyperledger/firefly-signer/pkg/ethsigner" "github.com/kaleido-io/paladin/core/internal/components" baseTypes "github.com/kaleido-io/paladin/core/internal/engine/enginespi" + "github.com/kaleido-io/paladin/core/pkg/blockindexer" "github.com/kaleido-io/paladin/core/pkg/ethclient" "github.com/kaleido-io/paladin/toolkit/pkg/log" @@ -62,7 +63,7 @@ type baseLedgerTxEngine struct { ctx context.Context thMetrics *baseLedgerTxEngineMetrics txStore baseTypes.TransactionStore - txConfirmationListener baseTypes.TransactionConfirmationListener + bIndexer blockindexer.BlockIndexer ethClient ethclient.EthClient managedTXEventNotifier baseTypes.ManagedTxEventNotifier keymgr ethclient.KeyManager @@ -79,6 +80,10 @@ type baseLedgerTxEngine struct { completedTxNoncePerAddress map[string]big.Int completedTxNoncePerAddressMutex sync.Mutex + // a map of signing addresses and the highest nonce of their confirmed transactions seen from the block indexer + confirmedTxNoncePerAddress map[string]*big.Int + confirmedTxNoncePerAddressRWMutex sync.RWMutex + // inbound concurrency control TBD // engine config @@ -145,6 +150,7 @@ func NewTransactionEngine(ctx context.Context, conf config.Section) (baseTypes.B cacheManager: cm, balanceManagerConfig: balanceManagerConfig, completedTxNoncePerAddress: make(map[string]big.Int), + confirmedTxNoncePerAddress: make(map[string]*big.Int), orchestratorConfig: orchestratorConfig, gasPriceIncreaseMax: gasPriceIncreaseMax, gasPriceIncreasePercent: big.NewInt(orchestratorConfig.GetInt64(OrchestratorGasPriceIncreasePercentageInt)), @@ -154,14 +160,14 @@ func NewTransactionEngine(ctx context.Context, conf config.Section) (baseTypes.B return ble, nil } -func (ble *baseLedgerTxEngine) Init(ctx context.Context, ethClient ethclient.EthClient, keymgr ethclient.KeyManager, txStore baseTypes.TransactionStore, managedTXEventNotifier baseTypes.ManagedTxEventNotifier, txConfirmationListener baseTypes.TransactionConfirmationListener) { +func (ble *baseLedgerTxEngine) Init(ctx context.Context, ethClient ethclient.EthClient, keymgr ethclient.KeyManager, txStore baseTypes.TransactionStore, managedTXEventNotifier baseTypes.ManagedTxEventNotifier, blockIndexer blockindexer.BlockIndexer) { log.L(ctx).Debugf("Initializing enterprise transaction handler") ble.ethClient = ethClient ble.keymgr = keymgr ble.txStore = txStore ble.gasPriceClient.Init(ctx, ethClient) ble.managedTXEventNotifier = managedTXEventNotifier - ble.txConfirmationListener = txConfirmationListener + ble.bIndexer = blockIndexer balanceManager, err := NewBalanceManagerWithInMemoryTracking(ctx, ble.balanceManagerConfig, ethClient, ble) if err != nil { @@ -178,6 +184,7 @@ func (ble *baseLedgerTxEngine) Start(ctx context.Context) (done <-chan struct{}, ble.ctx = ctx // set the context for policy loop ble.engineLoopDone = make(chan struct{}) log.L(ctx).Debugf("Kicking off enterprise handler engine loop") + ble.bIndexer.RegisterIndexedTransactionHandler(ctx, ble.HandleIndexTransactions) go ble.engineLoop() } ble.MarkInFlightOrchestratorsStale() @@ -306,6 +313,86 @@ func (ble *baseLedgerTxEngine) createManagedTx(ctx context.Context, txID string, return mtx, nil } +// HandleIndexTransactions +// handover events to the inflight orchestrators for the related signing addresses and record the highest confirmed nonce +// new orchestrators will be created if there are space, orchestrators will use the recorded highest nonce to drive completion logic of transactions +func (ble *baseLedgerTxEngine) HandleIndexTransactions(ctx context.Context, indexedTransactions []*blockindexer.IndexedTransaction) error { + // firstly, we group the indexed transactions by from address + // note: filter out transactions that are before the recorded nonce in confirmedTXNonce map requires multiple reads to a single address (as the loop keep switching between addresses) + // so we delegate the logic to the orchestrator as it will have a list of records for a single address + itMap := make(map[string]map[string]*blockindexer.IndexedTransaction) + itMaxNonce := make(map[string]*big.Int) + for _, it := range indexedTransactions { + itNonce := new(big.Int).SetUint64(it.Nonce) + if itMap[it.From.String()] == nil { + itMap[it.From.String()] = map[string]*blockindexer.IndexedTransaction{itNonce.String(): it} + } else { + itMap[it.From.String()][itNonce.String()] = it + } + if itMaxNonce[it.From.String()] == nil || itMaxNonce[it.From.String()].Cmp(itNonce) == -1 { + itMaxNonce[it.From.String()] = itNonce + } + } + + // secondly, we obtain the lock for the orchestrator map: + ble.InFlightOrchestratorMux.Lock() + defer ble.InFlightOrchestratorMux.Unlock() // note, using lock might cause the event sequence to get lost when this function is invoked concurrently by several go routines, this code assumes the upstream logic does not do that + + // for address that has or could have a running orchestrator, triggers event handlers of each orchestrator in parallel to handle the event + // (logic implemented in orchestrator handler)for the orchestrator handler, it obtains the stage process buffer lock and add the event into the stage process buffer and then exit + + localRWLock := sync.RWMutex{} // could consider switch InFlightOrchestrators to use sync.Map for this logic here as the go routines will only modify disjoint set of keys + eventHandlingErrors := make(chan error, len(itMap)) + for from, its := range itMap { + go func() { + localRWLock.RLock() + inFlightOrchestrator := ble.InFlightOrchestrators[from] + localRWLock.Unlock() + if inFlightOrchestrator == nil { + localRWLock.Lock() + itTotal := len(ble.InFlightOrchestrators) + if itTotal < ble.maxInFlightOrchestrators { + inFlightOrchestrator = NewOrchestrator(ble, from, ble.orchestratorConfig) + ble.InFlightOrchestrators[from] = inFlightOrchestrator + _, _ = inFlightOrchestrator.Start(ble.ctx) + log.L(ctx).Infof("(Event handler) Engine added orchestrator for signing address %s", from) + localRWLock.Unlock() + } else { + // no action can be taken + log.L(ctx).Debug("(Event handler) Cannot add orchestrator for signing address %s due to in-flight queue is full", from) + localRWLock.Unlock() + eventHandlingErrors <- nil + return + } + } + err := inFlightOrchestrator.HandleIndexTransactions(ctx, its, itMaxNonce[from]) + // finally, we update the confirmed nonce for each address to the highest number that is observed ever. This then can be used by the orchestrator to retrospectively fetch missed indexed transaction data. + ble.updateConfirmedTxNonce(from, itMaxNonce[from]) + eventHandlingErrors <- err + }() + } + + resultCount := 0 + var accumulatedError error + + // wait for all add output to complete + for { + select { + case err := <-eventHandlingErrors: + if err != nil { + accumulatedError = err + } + resultCount++ + case <-ctx.Done(): + return i18n.NewError(ctx, msgs.MsgContextCanceled) + } + if resultCount == len(itMap) { + break + } + } + return accumulatedError +} + func (ble *baseLedgerTxEngine) HandleSuspendTransaction(ctx context.Context, txID string) (mtx *baseTypes.ManagedTX, err error) { mtx, err = ble.txStore.GetTransactionByID(ctx, txID) if err != nil { @@ -329,3 +416,18 @@ func (ble *baseLedgerTxEngine) HandleResumeTransaction(ctx context.Context, txID } return res.tx, nil } + +func (ble *baseLedgerTxEngine) getConfirmedTxNonce(addr string) (nonce *big.Int) { + ble.confirmedTxNoncePerAddressRWMutex.RLock() + nonce = ble.confirmedTxNoncePerAddress[addr] + defer ble.confirmedTxNoncePerAddressRWMutex.RUnlock() + return +} + +func (ble *baseLedgerTxEngine) updateConfirmedTxNonce(addr string, nonce *big.Int) { + ble.confirmedTxNoncePerAddressRWMutex.Lock() + defer ble.confirmedTxNoncePerAddressRWMutex.Unlock() + if ble.confirmedTxNoncePerAddress[addr] == nil || ble.confirmedTxNoncePerAddress[addr].Cmp(nonce) != 1 { + ble.confirmedTxNoncePerAddress[addr] = nonce + } +} diff --git a/core/go/internal/engine/baseledgertx/transaction_engine_loop.go b/core/go/internal/engine/baseledgertx/transaction_engine_loop.go index 7e8e22ea5..4d19b1406 100644 --- a/core/go/internal/engine/baseledgertx/transaction_engine_loop.go +++ b/core/go/internal/engine/baseledgertx/transaction_engine_loop.go @@ -130,18 +130,18 @@ func (ble *baseLedgerTxEngine) poll(ctx context.Context) (polled int, total int) } // Run through copying across from the old InFlight list to the new one, those that aren't ready to be deleted - for signingAddress, te := range oldInFlight { - log.L(ctx).Debugf("Engine checking orchestrator for %s: state: %s, state duration: %s, number of transactions: %d", te.signingAddress, te.state, time.Since(te.stateEntryTime), len(te.InFlightTxs)) - if (te.state == OrchestratorStateStale && time.Since(te.stateEntryTime) > ble.maxOrchestratorStale) || - (te.state == OrchestratorStateIdle && time.Since(te.stateEntryTime) > ble.maxOrchestratorIdle) { + for signingAddress, oc := range oldInFlight { + log.L(ctx).Debugf("Engine checking orchestrator for %s: state: %s, state duration: %s, number of transactions: %d", oc.signingAddress, oc.state, time.Since(oc.stateEntryTime), len(oc.InFlightTxs)) + if (oc.state == OrchestratorStateStale && time.Since(oc.stateEntryTime) > ble.maxOrchestratorStale) || + (oc.state == OrchestratorStateIdle && time.Since(oc.stateEntryTime) > ble.maxOrchestratorIdle) { // tell transaction orchestrator to stop, there is a chance we later found new transaction for this address, but we got to make a call at some point // so it's here. The transaction orchestrator won't be removed immediately as the state update is async - te.Stop() + oc.Stop() } - if te.state != OrchestratorStateStopped { - ble.InFlightOrchestrators[signingAddress] = te - te.MarkInFlightTxStale() - stateCounts[string(te.state)] = stateCounts[string(te.state)] + 1 + if oc.state != OrchestratorStateStopped { + ble.InFlightOrchestrators[signingAddress] = oc + oc.MarkInFlightTxStale() + stateCounts[string(oc.state)] = stateCounts[string(oc.state)] + 1 InFlightSigningAddresses = append(InFlightSigningAddresses, signingAddress) } else { log.L(ctx).Infof("Engine removed orchestrator for signing address %s", signingAddress) @@ -187,10 +187,10 @@ func (ble *baseLedgerTxEngine) poll(ctx context.Context) (polled int, total int) for _, mtx := range additionalTxFromNonInFlightSigners { if _, exist := ble.InFlightOrchestrators[string(mtx.From)]; !exist { - te := NewOrchestrator(ble, string(mtx.From), ble.orchestratorConfig) - ble.InFlightOrchestrators[string(mtx.From)] = te - stateCounts[string(te.state)] = stateCounts[string(te.state)] + 1 - _, _ = te.Start(ble.ctx) + oc := NewOrchestrator(ble, string(mtx.From), ble.orchestratorConfig) + ble.InFlightOrchestrators[string(mtx.From)] = oc + stateCounts[string(oc.state)] = stateCounts[string(oc.state)] + 1 + _, _ = oc.Start(ble.ctx) log.L(ctx).Infof("Engine added orchestrator for signing address %s", mtx.From) } else { log.L(ctx).Warnf("Engine fetched extra transactions from signing address %s", mtx.From) @@ -206,10 +206,10 @@ func (ble *baseLedgerTxEngine) poll(ctx context.Context) (polled int, total int) // TODO: don't stop more than required number of slots // Run through the existing running orchestrators and stop the ones that exceeded the max process timeout - for signingAddress, te := range ble.InFlightOrchestrators { - if time.Since(te.orchestratorBirthTime) > ble.maxOverloadProcessTime { + for signingAddress, oc := range ble.InFlightOrchestrators { + if time.Since(oc.orchestratorBirthTime) > ble.maxOverloadProcessTime { log.L(ctx).Infof("Engine pause, attempt to stop orchestrator for signing address %s", signingAddress) - te.Stop() + oc.Stop() ble.SigningAddressesPausedUntil[signingAddress] = time.Now().Add(ble.maxOverloadProcessTime) } } diff --git a/core/go/internal/engine/baseledgertx/transaction_engine_loop_test.go b/core/go/internal/engine/baseledgertx/transaction_engine_loop_test.go index 8730489c8..f158dc197 100644 --- a/core/go/internal/engine/baseledgertx/transaction_engine_loop_test.go +++ b/core/go/internal/engine/baseledgertx/transaction_engine_loop_test.go @@ -39,12 +39,12 @@ func TestNewEngineNoNewOrchestrator(t *testing.T) { ble, _ := NewTestTransactionEngine(t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.maxInFlightOrchestrators = 1 ble.enginePollingInterval = 1 * time.Hour @@ -65,12 +65,12 @@ func TestNewEnginePollingCancelledContext(t *testing.T) { ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.maxInFlightOrchestrators = 1 ble.enginePollingInterval = 1 * time.Hour // already has a running orchestrator for the address so no new orchestrator should be started @@ -107,12 +107,12 @@ func TestNewEnginePollingReAddStoppedOrchestrator(t *testing.T) { ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.maxInFlightOrchestrators = 1 ble.enginePollingInterval = 1 * time.Hour @@ -161,12 +161,12 @@ func TestNewEnginePollingStoppingAnOrchestratorAndSelf(t *testing.T) { ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.maxInFlightOrchestrators = 2 ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour @@ -186,7 +186,7 @@ func TestNewEnginePollingStoppingAnOrchestratorAndSelf(t *testing.T) { txStore: mTS, ethClient: mEC, managedTXEventNotifier: mEN, - txConfirmationListener: mCL, + bIndexer: mBI, maxInFlightTxs: 0, } ble.InFlightOrchestrators = map[string]*orchestrator{ @@ -227,12 +227,12 @@ func TestNewEnginePollingStoppingAnOrchestratorForFairnessControl(t *testing.T) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.maxInFlightOrchestrators = 1 ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour @@ -249,7 +249,7 @@ func TestNewEnginePollingStoppingAnOrchestratorForFairnessControl(t *testing.T) txStore: mTS, ethClient: mEC, managedTXEventNotifier: mEN, - txConfirmationListener: mCL, + bIndexer: mBI, } // already has a running orchestrator for the address so no new orchestrator should be started mTS.On("NewTransactionFilter", mock.Anything).Return(mockTransactionFilter).Maybe() @@ -276,12 +276,12 @@ func TestNewEnginePollingExcludePausedOrchestrator(t *testing.T) { ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.maxInFlightOrchestrators = 1 ble.enginePollingInterval = 1 * time.Hour @@ -314,12 +314,12 @@ func TestNewEngineGetPendingFuelingTxs(t *testing.T) { ble, _ := NewTestTransactionEngine(t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour @@ -353,12 +353,12 @@ func TestNewEngineCheckTxCompleteness(t *testing.T) { ble, _ := NewTestTransactionEngine(t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.ctx = ctx mTS.On("NewTransactionFilter", mock.Anything).Return(mockTransactionFilter).Maybe() ble.enginePollingInterval = 1 * time.Hour diff --git a/core/go/internal/engine/baseledgertx/transaction_engine_test.go b/core/go/internal/engine/baseledgertx/transaction_engine_test.go index 25a2edf72..971416c18 100644 --- a/core/go/internal/engine/baseledgertx/transaction_engine_test.go +++ b/core/go/internal/engine/baseledgertx/transaction_engine_test.go @@ -88,9 +88,8 @@ func TestInit(t *testing.T) { mockTransactionFilter := qFields.NewFilter(ctx) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) - mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) listed := make(chan struct{}) @@ -99,7 +98,7 @@ func TestInit(t *testing.T) { listed <- struct{}{} }).Once() ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.enginePollingInterval = 1 * time.Hour ble.maxInFlightOrchestrators = 1 // starts ok @@ -109,7 +108,7 @@ func TestInit(t *testing.T) { afConfig := ble.balanceManagerConfig.SubSection(BalanceManagerAutoFuelingSection) afConfig.Set(BalanceManagerAutoFuelingSourceAddressMinimumBalanceBigIntString, "not a big int") assert.Panics(t, func() { - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) }) afConfig.Set(BalanceManagerAutoFuelingSourceAddressMinimumBalanceBigIntString, "0") } @@ -120,11 +119,11 @@ func TestHandleNewTransactionForTransferOnly(t *testing.T) { ble, _ := NewTestTransactionEngine(t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) testEthTxInput := ðsigner.Transaction{ From: []byte(testAutoFuelingSourceAddress), To: ethtypes.MustNewAddress(testDestAddress), @@ -228,14 +227,14 @@ func TestHandleNewTransactionTransferOnlyWithProvideGas(t *testing.T) { ctx := context.Background() ble, _ := NewTestTransactionEngine(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) mKM.On("ResolveKey", ctx, testAutoFuelingSourceAddress, algorithms.ECDSA_SECP256K1_PLAINBYTES).Return("", testAutoFuelingSourceAddress, nil) // fall back to connector when get call failed ble.gasPriceClient = NewTestNodeGasPriceClient(t, mEC) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) testEthTxInput := ðsigner.Transaction{ From: []byte(testAutoFuelingSourceAddress), To: ethtypes.MustNewAddress(testDestAddress), @@ -278,12 +277,12 @@ func TestHandleNewTransactionTransferAndInvalidType(t *testing.T) { ble, _ := NewTestTransactionEngine(t) ble.gasPriceClient = NewTestZeroGasPriceChainClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) mKM.On("ResolveKey", ctx, testAutoFuelingSourceAddress, algorithms.ECDSA_SECP256K1_PLAINBYTES).Return("", testAutoFuelingSourceAddress, nil) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) testEthTxInput := ðsigner.Transaction{ From: []byte(testAutoFuelingSourceAddress), To: ethtypes.MustNewAddress(testDestAddress), @@ -335,12 +334,12 @@ func TestHandleNewTransaction(t *testing.T) { ble, _ := NewTestTransactionEngine(t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) mKM.On("ResolveKey", ctx, testDestAddress, algorithms.ECDSA_SECP256K1_PLAINBYTES).Return("", testDestAddress, nil) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) testEthTxInput := ðsigner.Transaction{ From: []byte(testMainSigningAddress), To: ethtypes.MustNewAddress(testDestAddress), @@ -478,12 +477,12 @@ func TestHandleNewDeployment(t *testing.T) { ble, _ := NewTestTransactionEngine(t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) mKM.On("ResolveKey", ctx, testDestAddress, algorithms.ECDSA_SECP256K1_PLAINBYTES).Return("", testDestAddress, nil) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) testEthTxInput := ðsigner.Transaction{ From: []byte(testMainSigningAddress), Data: ethtypes.MustNewHexBytes0xPrefix(""), @@ -603,12 +602,12 @@ func TestEngineSuspend(t *testing.T) { ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) imtxs := NewTestInMemoryTxState(t) mtx := imtxs.GetTx() @@ -650,7 +649,7 @@ func TestEngineSuspend(t *testing.T) { txStore: mTS, ethClient: mEC, managedTXEventNotifier: mEN, - txConfirmationListener: mCL, + bIndexer: mBI, } // orchestrator update error mTS.On("GetTransactionByID", ctx, mtx.ID).Return(mtx, nil).Once() @@ -706,12 +705,12 @@ func TestEngineResume(t *testing.T) { ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) imtxs := NewTestInMemoryTxState(t) mtx := imtxs.GetTx() @@ -753,7 +752,7 @@ func TestEngineResume(t *testing.T) { txStore: mTS, ethClient: mEC, managedTXEventNotifier: mEN, - txConfirmationListener: mCL, + bIndexer: mBI, } // orchestrator update error mTS.On("GetTransactionByID", ctx, mtx.ID).Return(mtx, nil).Once() @@ -809,12 +808,12 @@ func TestEngineCanceledContext(t *testing.T) { ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) mTS := enginemocks.NewTransactionStore(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) imtxs := NewTestInMemoryTxState(t) mtx := imtxs.GetTx() diff --git a/core/go/internal/engine/baseledgertx/transaction_orchestrator.go b/core/go/internal/engine/baseledgertx/transaction_orchestrator.go index 908dda176..6138036c0 100644 --- a/core/go/internal/engine/baseledgertx/transaction_orchestrator.go +++ b/core/go/internal/engine/baseledgertx/transaction_orchestrator.go @@ -29,6 +29,7 @@ import ( "github.com/hyperledger/firefly-common/pkg/retry" baseTypes "github.com/kaleido-io/paladin/core/internal/engine/enginespi" "github.com/kaleido-io/paladin/core/internal/msgs" + "github.com/kaleido-io/paladin/core/pkg/blockindexer" "github.com/kaleido-io/paladin/core/pkg/ethclient" "github.com/kaleido-io/paladin/toolkit/pkg/log" ) @@ -175,7 +176,7 @@ type orchestrator struct { txStore baseTypes.TransactionStore ethClient ethclient.EthClient managedTXEventNotifier baseTypes.ManagedTxEventNotifier - txConfirmationListener baseTypes.TransactionConfirmationListener + bIndexer blockindexer.BlockIndexer turnOffHistory bool transactionSubmissionRetry *retry.Retry @@ -250,7 +251,7 @@ func NewOrchestrator( txStore: ble.txStore, ethClient: ble.ethClient, managedTXEventNotifier: ble.managedTXEventNotifier, - txConfirmationListener: ble.txConfirmationListener, + bIndexer: ble.bIndexer, } log.L(ctx).Debugf("NewOrchestrator for signing address %s created: %+v", newOrchestrator.signingAddress, newOrchestrator) @@ -258,42 +259,42 @@ func NewOrchestrator( return newOrchestrator } -func (te *orchestrator) orchestratorLoop() { - ctx := log.WithLogField(te.ctx, "role", "orchestrator-loop") - log.L(ctx).Infof("Orchestrator for signing address %s started pooling based on interval %s", te.signingAddress, te.orchestratorPollingInterval) +func (oc *orchestrator) orchestratorLoop() { + ctx := log.WithLogField(oc.ctx, "role", "orchestrator-loop") + log.L(ctx).Infof("Orchestrator for signing address %s started pooling based on interval %s", oc.signingAddress, oc.orchestratorPollingInterval) - defer close(te.orchestratorLoopDone) + defer close(oc.orchestratorLoopDone) - ticker := time.NewTicker(te.orchestratorPollingInterval) + ticker := time.NewTicker(oc.orchestratorPollingInterval) for { // an InFlight select { - case <-te.InFlightTxsStale: + case <-oc.InFlightTxsStale: case <-ticker.C: case <-ctx.Done(): - log.L(ctx).Infof("Orchestrator loop exit due to canceled context, it processed %d transaction during its lifetime.", te.totalCompleted) + log.L(ctx).Infof("Orchestrator loop exit due to canceled context, it processed %d transaction during its lifetime.", oc.totalCompleted) return - case <-te.stopProcess: - log.L(ctx).Infof("Orchestrator loop process stopped, it processed %d transaction during its lifetime.", te.totalCompleted) - te.state = OrchestratorStateStopped - te.stateEntryTime = time.Now() - te.MarkInFlightOrchestratorsStale() // trigger engine loop for removal + case <-oc.stopProcess: + log.L(ctx).Infof("Orchestrator loop process stopped, it processed %d transaction during its lifetime.", oc.totalCompleted) + oc.state = OrchestratorStateStopped + oc.stateEntryTime = time.Now() + oc.MarkInFlightOrchestratorsStale() // trigger engine loop for removal return } - polled, total := te.pollAndProcess(ctx) + polled, total := oc.pollAndProcess(ctx) log.L(ctx).Debugf("Orchestrator loop polled %d txs, there are %d txs in total", polled, total) } } -func (te *orchestrator) pollAndProcess(ctx context.Context) (polled int, total int) { +func (oc *orchestrator) pollAndProcess(ctx context.Context) (polled int, total int) { pollStart := time.Now() - te.InFlightTxsMux.Lock() - defer te.InFlightTxsMux.Unlock() + oc.InFlightTxsMux.Lock() + defer oc.InFlightTxsMux.Unlock() queueUpdated := false - oldInFlight := te.InFlightTxs - te.InFlightTxs = make([]*InFlightTransactionStageController, 0, len(oldInFlight)) + oldInFlight := oc.InFlightTxs + oc.InFlightTxs = make([]*InFlightTransactionStageController, 0, len(oldInFlight)) stageCounts := make(map[string]int) for _, stageName := range baseTypes.AllInFlightStages { @@ -302,16 +303,16 @@ func (te *orchestrator) pollAndProcess(ctx context.Context) (polled int, total i } var latestCompleted *baseTypes.ManagedTX - startFromNonce, hasCompletedNonce := te.completedTxNoncePerAddress[te.signingAddress] + startFromNonce, hasCompletedNonce := oc.completedTxNoncePerAddress[oc.signingAddress] // Run through copying across from the old InFlight list to the new one, those that aren't ready to be deleted for _, p := range oldInFlight { if !hasCompletedNonce || p.stateManager.GetNonce().Uint64()-startFromNonce.Uint64() <= 1 { startFromNonce = *p.stateManager.GetNonce() hasCompletedNonce = true } - te.processedTxIDs[p.stateManager.GetTxID()] = true + oc.processedTxIDs[p.stateManager.GetTxID()] = true if p.stateManager.CanBeRemoved(ctx) { - te.totalCompleted = te.totalCompleted + 1 + oc.totalCompleted = oc.totalCompleted + 1 latestCompleted = p.stateManager.GetTx() queueUpdated = true log.L(ctx).Debugf("Orchestrator poll and process, marking %s as complete after: %s", p.stateManager.GetTxID(), time.Since(*p.stateManager.GetCreatedTime().Time())) @@ -319,7 +320,7 @@ func (te *orchestrator) pollAndProcess(ctx context.Context) (polled int, total i log.L(ctx).Debugf("Orchestrator poll and process, removed suspended tx %s after: %s", p.stateManager.GetTxID(), time.Since(*p.stateManager.GetCreatedTime().Time())) } else { log.L(ctx).Debugf("Orchestrator poll and process, continuing tx %s after: %s", p.stateManager.GetTxID(), time.Since(*p.stateManager.GetCreatedTime().Time())) - te.InFlightTxs = append(te.InFlightTxs, p) + oc.InFlightTxs = append(oc.InFlightTxs, p) txStage := p.stateManager.GetStage(ctx) if string(txStage) == "" { txStage = baseTypes.InFlightTxStageQueued @@ -331,58 +332,58 @@ func (te *orchestrator) pollAndProcess(ctx context.Context) (polled int, total i log.L(ctx).Debugf("Orchestrator poll and process, stage counts: %+v", stageCounts) if latestCompleted != nil { - te.updateCompletedTxNonce(latestCompleted) + oc.updateCompletedTxNonce(latestCompleted) } - oldLen := len(te.InFlightTxs) + oldLen := len(oc.InFlightTxs) total = oldLen // check and poll new transactions from the persistence if we can handle more // If we are not at maximum, then query if there are more candidates now - spaces := te.maxInFlightTxs - oldLen + spaces := oc.maxInFlightTxs - oldLen if spaces > 0 { completedTxIDsStillBeingPersisted := make(map[string]bool) - fb := te.txStore.NewTransactionFilter(ctx) + fb := oc.txStore.NewTransactionFilter(ctx) conds := []ffapi.Filter{ - fb.Eq("from", te.signingAddress), + fb.Eq("from", oc.signingAddress), fb.Eq("status", baseTypes.BaseTxStatusPending), } - if len(te.transactionIDsInStatusUpdate) > 0 { - transactionIDInStatusUpdate := make([]driver.Value, 0, len(te.transactionIDsInStatusUpdate)) - for _, txID := range te.transactionIDsInStatusUpdate { + if len(oc.transactionIDsInStatusUpdate) > 0 { + transactionIDInStatusUpdate := make([]driver.Value, 0, len(oc.transactionIDsInStatusUpdate)) + for _, txID := range oc.transactionIDsInStatusUpdate { transactionIDInStatusUpdate = append(transactionIDInStatusUpdate, txID) } conds = append(conds, fb.NotIn(dbsql.ColumnID, transactionIDInStatusUpdate)) } var after string - if len(te.InFlightTxs) > 0 { + if len(oc.InFlightTxs) > 0 { conds = append(conds, fb.Gt("nonce", startFromNonce.String())) } var additional []*baseTypes.ManagedTX // We retry the get from persistence indefinitely (until the context cancels) - err := te.retry.Do(ctx, "get pending transactions", func(attempt int) (retry bool, err error) { + err := oc.retry.Do(ctx, "get pending transactions", func(attempt int) (retry bool, err error) { filter := fb.And(conds...) _ = filter.Limit(uint64(spaces)).Sort("nonce") - additional, _, err = te.txStore.ListTransactions(ctx, filter) + additional, _, err = oc.txStore.ListTransactions(ctx, filter) return true, err }) if err != nil { log.L(ctx).Infof("Orchestrator poll and process: context cancelled while retrying") - return -1, len(te.InFlightTxs) + return -1, len(oc.InFlightTxs) } log.L(ctx).Debugf("Orchestrator poll and process: polled %d items, space: %d", len(additional), spaces) for _, mtx := range additional { - if te.processedTxIDs[mtx.ID] { + if oc.processedTxIDs[mtx.ID] { // already processed, still being persisted completedTxIDsStillBeingPersisted[mtx.ID] = true log.L(ctx).Debugf("Orchestrator polled transaction with ID: %s but it's already being processed before, ignoring it", mtx.ID) } else if mtx.Status == baseTypes.BaseTxStatusPending { queueUpdated = true - it := NewInFlightTransactionStageController(te.baseLedgerTxEngine, te, mtx) - te.InFlightTxs = append(te.InFlightTxs, it) + it := NewInFlightTransactionStageController(oc.baseLedgerTxEngine, oc, mtx) + oc.InFlightTxs = append(oc.InFlightTxs, it) txStage := it.stateManager.GetStage(ctx) if string(txStage) == "" { txStage = baseTypes.InFlightTxStageQueued @@ -391,35 +392,35 @@ func (te *orchestrator) pollAndProcess(ctx context.Context) (polled int, total i log.L(ctx).Debugf("Orchestrator added transaction with ID: %s", mtx.ID) } } - te.processedTxIDs = completedTxIDsStillBeingPersisted - total = len(te.InFlightTxs) + oc.processedTxIDs = completedTxIDsStillBeingPersisted + total = len(oc.InFlightTxs) polled = total - oldLen if polled > 0 { - log.L(ctx).Debugf("InFlight set updated len=%d head-seq=%s tail-seq=%s old-tail=%s", len(te.InFlightTxs), te.InFlightTxs[0].stateManager.GetTx().SequenceID, te.InFlightTxs[total-1].stateManager.GetTx().SequenceID, after) + log.L(ctx).Debugf("InFlight set updated len=%d head-seq=%s tail-seq=%s old-tail=%s", len(oc.InFlightTxs), oc.InFlightTxs[0].stateManager.GetTx().SequenceID, oc.InFlightTxs[total-1].stateManager.GetTx().SequenceID, after) } - te.thMetrics.RecordInFlightTxQueueMetrics(ctx, stageCounts, te.maxInFlightTxs-len(te.InFlightTxs)) + oc.thMetrics.RecordInFlightTxQueueMetrics(ctx, stageCounts, oc.maxInFlightTxs-len(oc.InFlightTxs)) } log.L(ctx).Debugf("Orchestrator polling from DB took %s", time.Since(pollStart)) // now check and process each transaction if total > 0 { - waitingForBalance, _ := te.ProcessInFlightTransaction(ctx, te.InFlightTxs) + waitingForBalance, _ := oc.ProcessInFlightTransaction(ctx, oc.InFlightTxs) if queueUpdated { - te.lastQueueUpdate = time.Now() + oc.lastQueueUpdate = time.Now() } - if time.Since(te.lastQueueUpdate) > te.staleTimeout && te.state != OrchestratorStateStale { - te.state = OrchestratorStateStale - te.stateEntryTime = time.Now() - } else if waitingForBalance && te.state != OrchestratorStateWaiting { - te.state = OrchestratorStateWaiting - te.stateEntryTime = time.Now() - } else if te.state != OrchestratorStateRunning { - te.state = OrchestratorStateRunning - te.stateEntryTime = time.Now() + if time.Since(oc.lastQueueUpdate) > oc.staleTimeout && oc.state != OrchestratorStateStale { + oc.state = OrchestratorStateStale + oc.stateEntryTime = time.Now() + } else if waitingForBalance && oc.state != OrchestratorStateWaiting { + oc.state = OrchestratorStateWaiting + oc.stateEntryTime = time.Now() + } else if oc.state != OrchestratorStateRunning { + oc.state = OrchestratorStateRunning + oc.stateEntryTime = time.Now() } - } else if te.state != OrchestratorStateIdle { - te.state = OrchestratorStateIdle - te.stateEntryTime = time.Now() + } else if oc.state != OrchestratorStateIdle { + oc.state = OrchestratorStateIdle + oc.stateEntryTime = time.Now() } log.L(ctx).Debugf("Orchestrator process loop took %s", time.Since(pollStart)) @@ -427,25 +428,25 @@ func (te *orchestrator) pollAndProcess(ctx context.Context) (polled int, total i } // this function should only have one running instance at any given time -func (te *orchestrator) ProcessInFlightTransaction(ctx context.Context, its []*InFlightTransactionStageController) (waitingForBalance bool, err error) { +func (oc *orchestrator) ProcessInFlightTransaction(ctx context.Context, its []*InFlightTransactionStageController) (waitingForBalance bool, err error) { processStart := time.Now() waitingForBalance = false var addressAccount *baseTypes.AddressAccount - skipBalanceCheck := te.hasZeroGasPrice + skipBalanceCheck := oc.hasZeroGasPrice now := time.Now() - log.L(ctx).Debugf("%s ProcessInFlightTransaction entry for signing address %s", now.String(), te.signingAddress) + log.L(ctx).Debugf("%s ProcessInFlightTransaction entry for signing address %s", now.String(), oc.signingAddress) if !skipBalanceCheck { - log.L(ctx).Debugf("%s: ProcessInFlightTransaction checking balance for %s", now.String(), te.signingAddress) + log.L(ctx).Debugf("%s: ProcessInFlightTransaction checking balance for %s", now.String(), oc.signingAddress) - addressAccount, err = te.balanceManager.GetAddressBalance(te.ctx, te.signingAddress) + addressAccount, err = oc.balanceManager.GetAddressBalance(oc.ctx, oc.signingAddress) if err != nil { - log.L(ctx).Errorf("Failed to retrieve balance for address %s due to %+v", te.signingAddress, err) - if te.unavailableBalanceHandlingStrategy == OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyWait { + log.L(ctx).Errorf("Failed to retrieve balance for address %s due to %+v", oc.signingAddress, err) + if oc.unavailableBalanceHandlingStrategy == OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyWait { // wait till next retry return true, nil - } else if te.unavailableBalanceHandlingStrategy == OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyStop { - te.Stop() + } else if oc.unavailableBalanceHandlingStrategy == OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyStop { + oc.Stop() return true, nil } else { // just continue without any balance check @@ -456,8 +457,9 @@ func (te *orchestrator) ProcessInFlightTransaction(ctx context.Context, its []*I } previousNonceCostUnknown := false + currentConfirmedNonce := oc.getConfirmedTxNonce(oc.signingAddress) for i, it := range its { - log.L(ctx).Debugf("%s ProcessInFlightTransaction for signing address %s processing transaction with ID: %s, index: %d", now.String(), te.signingAddress, it.stateManager.GetTxID(), i) + log.L(ctx).Debugf("%s ProcessInFlightTransaction for signing address %s processing transaction with ID: %s, index: %d", now.String(), oc.signingAddress, it.stateManager.GetTxID(), i) var availableToSpend *big.Int if !skipBalanceCheck { availableToSpend = addressAccount.GetAvailableToSpend(ctx) @@ -465,6 +467,7 @@ func (te *orchestrator) ProcessInFlightTransaction(ctx context.Context, its []*I triggerNextStageOutput := it.ProduceLatestInFlightStageContext(ctx, &baseTypes.OrchestratorContext{ AvailableToSpend: availableToSpend, PreviousNonceCostUnknown: previousNonceCostUnknown, + CurrentConfirmedNonce: currentConfirmedNonce, }) if !skipBalanceCheck { if triggerNextStageOutput.Cost != nil { @@ -481,86 +484,94 @@ func (te *orchestrator) ProcessInFlightTransaction(ctx context.Context, its []*I } } - if !skipBalanceCheck && addressAccount.GetAvailableToSpend(ctx).Sign() == -1 && te.balanceManager.IsAutoFuelingEnabled(ctx) { - log.L(ctx).Debugf("%s Address %s requires top up, credit after estimated cost: %s", now.String(), te.signingAddress, addressAccount.GetAvailableToSpend(ctx).String()) - _, _ = te.balanceManager.TopUpAccount(ctx, addressAccount) + if !skipBalanceCheck && addressAccount.GetAvailableToSpend(ctx).Sign() == -1 && oc.balanceManager.IsAutoFuelingEnabled(ctx) { + log.L(ctx).Debugf("%s Address %s requires top up, credit after estimated cost: %s", now.String(), oc.signingAddress, addressAccount.GetAvailableToSpend(ctx).String()) + _, _ = oc.balanceManager.TopUpAccount(ctx, addressAccount) } - log.L(ctx).Debugf("%s ProcessInFlightTransaction exit for signing address: %s", now.String(), te.signingAddress) + log.L(ctx).Debugf("%s ProcessInFlightTransaction exit for signing address: %s", now.String(), oc.signingAddress) log.L(ctx).Debugf("Orchestrator process loop took %s", time.Since(processStart)) return waitingForBalance, nil } -func (te *orchestrator) Start(c context.Context) (done <-chan struct{}, err error) { - te.orchestratorLoopDone = make(chan struct{}) - go te.orchestratorLoop() - te.MarkInFlightTxStale() - return te.orchestratorLoopDone, nil +func (oc *orchestrator) Start(c context.Context) (done <-chan struct{}, err error) { + oc.orchestratorLoopDone = make(chan struct{}) + go oc.orchestratorLoop() + oc.MarkInFlightTxStale() + return oc.orchestratorLoopDone, nil +} + +// HandleIndexTransactions +// adds confirmed/indexed transactions to the event buffer concurrently for all in-flight transactions +// currently, the submission queue and tracking queue are the same queue, which means the logic should not miss +// confirmations for a pending transaction. +// If the two queues needs to split in the future to allow optimistic submission & delayed confirmation, this logic will need to be updated. +func (oc *orchestrator) HandleIndexTransactions(ctx context.Context, indexedTransactions map[string]*blockindexer.IndexedTransaction, maxNonce *big.Int) error { + recordStart := time.Now() + oc.InFlightTxsMux.Lock() + defer oc.InFlightTxsMux.Unlock() + var pending *InFlightTransactionStageController + addOutputResponse := make(chan struct{}, len(oc.InFlightTxs)) + for _, it := range oc.InFlightTxs { + if it != nil { + currentTx := it + if maxNonce.Cmp(currentTx.stateManager.GetNonce()) != -1 { + // there is a indexed transaction available for this transaction + go func() { + nonceStr := currentTx.stateManager.GetNonce().String() + indexTx := indexedTransactions[nonceStr] + if indexTx != nil { + pending.MarkHistoricalTime("receipt_event_wait_to_be_recorded", recordStart) + pending.MarkTime("receipt_event_wait_to_be_processed") + // Will be picked up on the next orchestrator loop - guaranteed to occur before Confirmed + pending.stateManager.AddConfirmationsOutput(ctx, indexTx) + pending.MarkInFlightTxStale() + } + addOutputResponse <- struct{}{} + }() + } else { + addOutputResponse <- struct{}{} + } + + } else { + addOutputResponse <- struct{}{} + } + } + + resultCount := 0 + + // wait for all add output to complete + for { + select { + case <-addOutputResponse: + resultCount++ + case <-ctx.Done(): + return i18n.NewError(ctx, msgs.MsgContextCanceled) + } + if resultCount == len(oc.InFlightTxs) { + break + } + } + // we've processed all indexed nonce in this batch, update the confirmed nonce so any gaps will be filled in + oc.updateConfirmedTxNonce(oc.signingAddress, maxNonce) + return nil } // Stop the InFlight transaction process. -func (te *orchestrator) Stop() { +func (oc *orchestrator) Stop() { // try to send an item in `stopProcess` channel, which has a buffer of 1 // if it already has an item in the channel, this function does nothing select { - case te.stopProcess <- true: + case oc.stopProcess <- true: default: } } -func (te *orchestrator) MarkInFlightTxStale() { +func (oc *orchestrator) MarkInFlightTxStale() { // try to send an item in `processNow` channel, which has a buffer of 1 // if it already has an item in the channel, this function does nothing select { - case te.InFlightTxsStale <- true: + case oc.InFlightTxsStale <- true: default: } } - -func (te *orchestrator) CreateTransactionConfirmationsHandler(txID string) func(ctx context.Context, txID string, notification *baseTypes.ConfirmationsNotification) (err error) { - return func(ctx context.Context, txID string, notification *baseTypes.ConfirmationsNotification) (err error) { - recordStart := time.Now() - te.InFlightTxsMux.Lock() - defer te.InFlightTxsMux.Unlock() - var pending *InFlightTransactionStageController - for _, p := range te.InFlightTxs { - if p != nil && p.stateManager.GetTxID() == txID { - pending = p - break - } - } - if pending == nil { - err = i18n.NewError(ctx, msgs.MsgTransactionNotFound, txID) - return - } - pending.MarkHistoricalTime("confirm_event_wait_to_be_recorded", recordStart) - pending.MarkTime("confirm_event_wait_to_be_processed") - pending.stateManager.AddConfirmationsOutput(ctx, notification) - pending.MarkInFlightTxStale() - return - } -} -func (te *orchestrator) CreateTransactionReceiptReceivedHandler(txID string) func(ctx context.Context, txID string, receipt *ethclient.TransactionReceiptResponse) (err error) { - return func(ctx context.Context, txID string, receipt *ethclient.TransactionReceiptResponse) (err error) { - recordStart := time.Now() - te.InFlightTxsMux.Lock() - defer te.InFlightTxsMux.Unlock() - var pending *InFlightTransactionStageController - for _, p := range te.InFlightTxs { - if p != nil && p.stateManager.GetTxID() == txID { - pending = p - break - } - } - if pending == nil { - err = i18n.NewError(ctx, msgs.MsgTransactionNotFound, txID) - return - } - pending.MarkHistoricalTime("receipt_event_wait_to_be_recorded", recordStart) - pending.MarkTime("receipt_event_wait_to_be_processed") - // Will be picked up on the next orchestrator loop - guaranteed to occur before Confirmed - pending.stateManager.AddReceiptOutput(ctx, receipt, nil) - pending.MarkInFlightTxStale() - return - } -} diff --git a/core/go/internal/engine/baseledgertx/transaction_orchestrator_test.go b/core/go/internal/engine/baseledgertx/transaction_orchestrator_test.go index ea16ddfd7..6c9e0a6e9 100644 --- a/core/go/internal/engine/baseledgertx/transaction_orchestrator_test.go +++ b/core/go/internal/engine/baseledgertx/transaction_orchestrator_test.go @@ -64,20 +64,20 @@ func TestNewOrchestratorPolling(t *testing.T) { ble, _ := NewTestTransactionEngine(t) mockBM, mEC, _ := NewTestBalanceManager(ctx, t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mTS := enginemocks.NewTransactionStore(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.orchestratorConfig.Set(OrchestratorMaxInFlightTransactionsInt, 10) ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour - te := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) - te.balanceManager = mockBM - mockIT := NewInFlightTransactionStageController(ble, te, mockManagedTx1) - te.InFlightTxs = []*InFlightTransactionStageController{mockIT} - te.transactionIDsInStatusUpdate = []string{"randomID"} + oc := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) + oc.balanceManager = mockBM + mockIT := NewInFlightTransactionStageController(ble, oc, mockManagedTx1) + oc.InFlightTxs = []*InFlightTransactionStageController{mockIT} + oc.transactionIDsInStatusUpdate = []string{"randomID"} mTS.On("AddSubStatusAction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { time.Sleep(1 * time.Hour) // make sure the async action never got returned as the test will mock the events }).Maybe() @@ -87,15 +87,15 @@ func TestNewOrchestratorPolling(t *testing.T) { mTS.On("NewTransactionFilter", mock.Anything).Return(mockTransactionFilter).Maybe() mTS.On("ListTransactions", mock.Anything, mock.Anything).Return([]*baseTypes.ManagedTX{mockManagedTx1, mockManagedTx2}, nil, nil).Once() mTS.On("ListTransactions", mock.Anything, mock.Anything).Return([]*baseTypes.ManagedTX{}, nil, nil).Maybe() - te.InFlightTxs = []*InFlightTransactionStageController{ + oc.InFlightTxs = []*InFlightTransactionStageController{ mockIT} addressBalanceChecked := make(chan bool) mEC.On("GetBalance", mock.Anything, testMainSigningAddress, "latest").Return(ethtypes.NewHexInteger64(100), nil).Run(func(args mock.Arguments) { close(addressBalanceChecked) }).Once() - te.orchestratorPollingInterval = 1 * time.Hour - _, _ = te.Start(ctx) + oc.orchestratorPollingInterval = 1 * time.Hour + _, _ = oc.Start(ctx) <-addressBalanceChecked } @@ -115,24 +115,24 @@ func TestNewOrchestratorPollingContextCancelled(t *testing.T) { ble, _ := NewTestTransactionEngine(t) mockBM, mEC, _ := NewTestBalanceManager(ctx, t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mTS := enginemocks.NewTransactionStore(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.orchestratorConfig.Set(OrchestratorMaxInFlightTransactionsInt, 10) ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour - te := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) + oc := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) - te.balanceManager = mockBM + oc.balanceManager = mockBM mTS.On("NewTransactionFilter", mock.Anything).Return(mockTransactionFilter).Maybe() mTS.On("ListTransactions", mock.Anything, mock.Anything).Return(nil, nil, fmt.Errorf("list transactions error")).Run(func(args mock.Arguments) { cancelCtx() }).Once() - polled, _ := te.pollAndProcess(ctx) + polled, _ := oc.pollAndProcess(ctx) assert.Equal(t, -1, polled) } @@ -163,31 +163,31 @@ func TestNewOrchestratorPollingRemoveCompleted(t *testing.T) { ble, _ := NewTestTransactionEngine(t) mockBM, mEC, _ := NewTestBalanceManager(ctx, t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mTS := enginemocks.NewTransactionStore(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.orchestratorConfig.Set(OrchestratorMaxInFlightTransactionsInt, 10) ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour - te := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) + oc := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) - te.balanceManager = mockBM - mockIT := NewInFlightTransactionStageController(ble, te, mockManagedTx1) + oc.balanceManager = mockBM + mockIT := NewInFlightTransactionStageController(ble, oc, mockManagedTx1) mTS.On("NewTransactionFilter", mock.Anything).Return(mockTransactionFilter).Once() mTS.On("ListTransactions", mock.Anything, mock.Anything).Return([]*baseTypes.ManagedTX{mockManagedTx1, mockManagedTx2}, nil, nil).Once() - te.InFlightTxs = []*InFlightTransactionStageController{ + oc.InFlightTxs = []*InFlightTransactionStageController{ mockIT, } addressBalanceChecked := make(chan bool) mEC.On("GetBalance", mock.Anything, testMainSigningAddress, "latest").Return(nil, fmt.Errorf("failed getting balance")).Run(func(args mock.Arguments) { close(addressBalanceChecked) }).Once() - te.orchestratorPollingInterval = 1 * time.Hour - _, _ = te.Start(ctx) + oc.orchestratorPollingInterval = 1 * time.Hour + _, _ = oc.Start(ctx) <-addressBalanceChecked } @@ -219,24 +219,24 @@ func TestNewOrchestratorPollingRemoveSuspended(t *testing.T) { ble, _ := NewTestTransactionEngine(t) mockBM, mEC, _ := NewTestBalanceManager(ctx, t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mTS := enginemocks.NewTransactionStore(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.orchestratorConfig.Set(OrchestratorMaxInFlightTransactionsInt, 10) ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour - te := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) + oc := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) - te.balanceManager = mockBM - mockIT := NewInFlightTransactionStageController(ble, te, mockManagedTx1) + oc.balanceManager = mockBM + mockIT := NewInFlightTransactionStageController(ble, oc, mockManagedTx1) mTS.On("NewTransactionFilter", mock.Anything).Return(mockTransactionFilter).Once() mTS.On("ListTransactions", mock.Anything, mock.Anything).Return([]*baseTypes.ManagedTX{mockManagedTx1, mockManagedTx2}, nil, nil).Once() - te.InFlightTxs = []*InFlightTransactionStageController{ + oc.InFlightTxs = []*InFlightTransactionStageController{ mockIT, } @@ -244,8 +244,8 @@ func TestNewOrchestratorPollingRemoveSuspended(t *testing.T) { mEC.On("GetBalance", mock.Anything, testMainSigningAddress, "latest").Return(nil, fmt.Errorf("failed getting balance")).Run(func(args mock.Arguments) { close(addressBalanceChecked) }).Once() - te.orchestratorPollingInterval = 1 * time.Hour - _, _ = te.Start(ctx) + oc.orchestratorPollingInterval = 1 * time.Hour + _, _ = oc.Start(ctx) <-addressBalanceChecked } @@ -268,34 +268,34 @@ func TestNewOrchestratorPollingMarkStale(t *testing.T) { ble, _ := NewTestTransactionEngine(t) mockBM, mEC, _ := NewTestBalanceManager(ctx, t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mTS := enginemocks.NewTransactionStore(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.orchestratorConfig.Set(OrchestratorMaxInFlightTransactionsInt, 10) ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour - te := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) + oc := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) - te.balanceManager = mockBM - te.lastQueueUpdate = time.Now().Add(-1 * time.Hour) - mockIT := NewInFlightTransactionStageController(ble, te, mockManagedTx1) + oc.balanceManager = mockBM + oc.lastQueueUpdate = time.Now().Add(-1 * time.Hour) + mockIT := NewInFlightTransactionStageController(ble, oc, mockManagedTx1) mTS.On("NewTransactionFilter", mock.Anything).Return(mockTransactionFilter).Maybe() mTS.On("ListTransactions", mock.Anything, mock.Anything).Return([]*baseTypes.ManagedTX{mockManagedTx1}, nil, nil).Once() - te.InFlightTxs = []*InFlightTransactionStageController{ + oc.InFlightTxs = []*InFlightTransactionStageController{ mockIT, } addressBalanceChecked := make(chan bool) mEC.On("GetBalance", mock.Anything, testMainSigningAddress, "latest").Return(nil, fmt.Errorf("failed getting balance")).Run(func(args mock.Arguments) { close(addressBalanceChecked) }).Once() - te.orchestratorPollingInterval = 1 * time.Hour - te.pollAndProcess(ctx) + oc.orchestratorPollingInterval = 1 * time.Hour + oc.pollAndProcess(ctx) <-addressBalanceChecked - assert.Equal(t, OrchestratorStateStale, te.state) + assert.Equal(t, OrchestratorStateStale, oc.state) } func TestOrchestratorStop(t *testing.T) { @@ -330,25 +330,25 @@ func TestOrchestratorStop(t *testing.T) { mTS.On("NewTransactionFilter", mock.Anything).Return(mockTransactionFilter).Maybe() mTS.On("ListTransactions", mock.Anything, mock.Anything).Return([]*baseTypes.ManagedTX{mockManagedTx1, mockManagedTx2}, nil, nil).Maybe() ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mEC := componentmocks.NewEthClient(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.orchestratorConfig.Set(OrchestratorMaxInFlightTransactionsInt, 10) ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour - te := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) - te.orchestratorPollingInterval = 1 * time.Hour - te.ctx = ctx - te.orchestratorLoopDone = make(chan struct{}) - go te.orchestratorLoop() + oc := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) + oc.orchestratorPollingInterval = 1 * time.Hour + oc.ctx = ctx + oc.orchestratorLoopDone = make(chan struct{}) + go oc.orchestratorLoop() //stops OK cancelCtx() - <-te.orchestratorLoopDone + <-oc.orchestratorLoopDone } func TestOrchestratorStopWhenBalanceUnavailable(t *testing.T) { @@ -366,45 +366,45 @@ func TestOrchestratorStopWhenBalanceUnavailable(t *testing.T) { ble, _ := NewTestTransactionEngine(t) mockBM, mEC, _ := NewTestBalanceManager(ctx, t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mTS := enginemocks.NewTransactionStore(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.orchestratorConfig.Set(OrchestratorMaxInFlightTransactionsInt, 10) ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour - te := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) - te.state = OrchestratorStateRunning - te.balanceManager = mockBM - mockIT := NewInFlightTransactionStageController(ble, te, mockManagedTx1) + oc := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) + oc.state = OrchestratorStateRunning + oc.balanceManager = mockBM + mockIT := NewInFlightTransactionStageController(ble, oc, mockManagedTx1) // continue process mEC.On("GetBalance", mock.Anything, testMainSigningAddress, "latest").Return(nil, fmt.Errorf("failed getting balance")).Once() - te.unavailableBalanceHandlingStrategy = OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyContinue - waitingForBalance, err := te.ProcessInFlightTransaction(ctx, []*InFlightTransactionStageController{mockIT}) + oc.unavailableBalanceHandlingStrategy = OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyContinue + waitingForBalance, err := oc.ProcessInFlightTransaction(ctx, []*InFlightTransactionStageController{mockIT}) require.NoError(t, err) assert.False(t, waitingForBalance) - assert.Equal(t, OrchestratorStateRunning, te.state) + assert.Equal(t, OrchestratorStateRunning, oc.state) // wait for the next round mEC.On("GetBalance", mock.Anything, testMainSigningAddress, "latest").Return(nil, fmt.Errorf("failed getting balance")).Once() - te.unavailableBalanceHandlingStrategy = OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyWait - waitingForBalance, err = te.ProcessInFlightTransaction(ctx, []*InFlightTransactionStageController{mockIT}) + oc.unavailableBalanceHandlingStrategy = OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyWait + waitingForBalance, err = oc.ProcessInFlightTransaction(ctx, []*InFlightTransactionStageController{mockIT}) require.NoError(t, err) assert.True(t, waitingForBalance) - assert.Equal(t, OrchestratorStateRunning, te.state) + assert.Equal(t, OrchestratorStateRunning, oc.state) // stop the orchestrator mEC.On("GetBalance", mock.Anything, testMainSigningAddress, "latest").Return(nil, fmt.Errorf("failed getting balance")).Once() - te.unavailableBalanceHandlingStrategy = OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyStop - te.stopProcess = make(chan bool, 1) - waitingForBalance, err = te.ProcessInFlightTransaction(ctx, []*InFlightTransactionStageController{mockIT}) + oc.unavailableBalanceHandlingStrategy = OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyStop + oc.stopProcess = make(chan bool, 1) + waitingForBalance, err = oc.ProcessInFlightTransaction(ctx, []*InFlightTransactionStageController{mockIT}) require.NoError(t, err) assert.True(t, waitingForBalance) - <-te.stopProcess + <-oc.stopProcess } func TestOrchestratorTriggerTopUp(t *testing.T) { @@ -424,36 +424,36 @@ func TestOrchestratorTriggerTopUp(t *testing.T) { ble, _ := NewTestTransactionEngine(t) mockBM, mEC, mockAFTxEngine := NewTestBalanceManager(ctx, t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mTS := enginemocks.NewTransactionStore(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.orchestratorConfig.Set(OrchestratorMaxInFlightTransactionsInt, 10) ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour - te := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) - te.state = OrchestratorStateRunning - te.balanceManager = mockBM - mockIT := NewInFlightTransactionStageController(ble, te, mockManagedTx1) + oc := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) + oc.state = OrchestratorStateRunning + oc.balanceManager = mockBM + mockIT := NewInFlightTransactionStageController(ble, oc, mockManagedTx1) // trigger top up if cost is known mockAFTxEngine.On("GetPendingFuelingTransaction", mock.Anything, "0x4e598f6e918321dd47c86e7a077b4ab0e7414846", testMainSigningAddress).Return(nil, fmt.Errorf("cannot get tx")).Once() mEC.On("GetBalance", mock.Anything, testMainSigningAddress, "latest").Return(ethtypes.NewHexInteger64(100), nil).Once() - te.unavailableBalanceHandlingStrategy = OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyContinue - waitingForBalance, err := te.ProcessInFlightTransaction(ctx, []*InFlightTransactionStageController{mockIT}) + oc.unavailableBalanceHandlingStrategy = OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyContinue + waitingForBalance, err := oc.ProcessInFlightTransaction(ctx, []*InFlightTransactionStageController{mockIT}) require.NoError(t, err) assert.True(t, waitingForBalance) - assert.Equal(t, OrchestratorStateRunning, te.state) + assert.Equal(t, OrchestratorStateRunning, oc.state) // skip top up when cost is unknown mockManagedTx1.GasPrice = nil - te.unavailableBalanceHandlingStrategy = OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyContinue - waitingForBalance, err = te.ProcessInFlightTransaction(ctx, []*InFlightTransactionStageController{mockIT}) + oc.unavailableBalanceHandlingStrategy = OrchestratorBalanceCheckUnavailableBalanceHandlingStrategyContinue + waitingForBalance, err = oc.ProcessInFlightTransaction(ctx, []*InFlightTransactionStageController{mockIT}) require.NoError(t, err) assert.False(t, waitingForBalance) - assert.Equal(t, OrchestratorStateRunning, te.state) + assert.Equal(t, OrchestratorStateRunning, oc.state) } func TestOrchestratorReceiptHandler(t *testing.T) { @@ -480,23 +480,23 @@ func TestOrchestratorReceiptHandler(t *testing.T) { ble, _ := NewTestTransactionEngine(t) _, mEC, _ := NewTestBalanceManager(ctx, t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mTS := enginemocks.NewTransactionStore(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.orchestratorConfig.Set(OrchestratorMaxInFlightTransactionsInt, 10) ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour - te := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) - mockIT1 := NewInFlightTransactionStageController(ble, te, mockManagedTx1) + oc := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) + mockIT1 := NewInFlightTransactionStageController(ble, oc, mockManagedTx1) mockIT1.timeLineLoggingEnabled = true - mockIT2 := NewInFlightTransactionStageController(ble, te, mockManagedTx2) + mockIT2 := NewInFlightTransactionStageController(ble, oc, mockManagedTx2) mockIT2.timeLineLoggingEnabled = true - te.InFlightTxs = []*InFlightTransactionStageController{mockIT1, mockIT2} - receiptHandler := te.CreateTransactionReceiptReceivedHandler(mockManagedTx1.ID) + oc.InFlightTxs = []*InFlightTransactionStageController{mockIT1, mockIT2} + receiptHandler := oc.CreateTransactionReceiptReceivedHandler(mockManagedTx1.ID) testReceipt := ðclient.TransactionReceiptResponse{BlockNumber: fftypes.NewFFBigInt(1)} // added receipt @@ -504,10 +504,10 @@ func TestOrchestratorReceiptHandler(t *testing.T) { require.NoError(t, err) time.Sleep(200 * time.Millisecond) iftxs := mockIT1.stateManager.(*inFlightTransactionState) - assert.Equal(t, testReceipt, iftxs.bufferedStageOutputs[0].ReceiptOutput.Receipt) + assert.Equal(t, testReceipt, iftxs.bufferedStageOutputs[0].ConfirmationOutput.Receipt) // transaction no longer in queue - te.InFlightTxs = []*InFlightTransactionStageController{mockIT2} + oc.InFlightTxs = []*InFlightTransactionStageController{mockIT2} err = receiptHandler(ctx, mockIT1.stateManager.GetTxID(), testReceipt) assert.Regexp(t, "PD011924", err) } @@ -536,21 +536,21 @@ func TestOrchestratorConfirmationHandler(t *testing.T) { ble, _ := NewTestTransactionEngine(t) _, mEC, _ := NewTestBalanceManager(ctx, t) ble.gasPriceClient = NewTestFixedPriceGasPriceClient(t) - mCL := enginemocks.NewTransactionConfirmationListener(t) + mBI := componentmocks.NewBlockIndexer(t) mTS := enginemocks.NewTransactionStore(t) mEN := enginemocks.NewManagedTxEventNotifier(t) mKM := componentmocks.NewKeyManager(t) - ble.Init(ctx, mEC, mKM, mTS, mEN, mCL) + ble.Init(ctx, mEC, mKM, mTS, mEN, mBI) ble.orchestratorConfig.Set(OrchestratorMaxInFlightTransactionsInt, 10) ble.ctx = ctx ble.enginePollingInterval = 1 * time.Hour - te := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) - mockIT1 := NewInFlightTransactionStageController(ble, te, mockManagedTx1) - mockIT2 := NewInFlightTransactionStageController(ble, te, mockManagedTx2) - te.InFlightTxs = []*InFlightTransactionStageController{mockIT1, mockIT2} - confirmationHandler := te.CreateTransactionConfirmationsHandler(mockManagedTx1.ID) + oc := NewOrchestrator(ble, string(mockManagedTx1.From), ble.orchestratorConfig) + mockIT1 := NewInFlightTransactionStageController(ble, oc, mockManagedTx1) + mockIT2 := NewInFlightTransactionStageController(ble, oc, mockManagedTx2) + oc.InFlightTxs = []*InFlightTransactionStageController{mockIT1, mockIT2} + confirmationHandler := oc.CreateTransactionConfirmationsHandler(mockManagedTx1.ID) testConfirmation := &baseTypes.ConfirmationsNotification{Confirmed: true} // added confirmation @@ -561,7 +561,7 @@ func TestOrchestratorConfirmationHandler(t *testing.T) { assert.Equal(t, testConfirmation, iftxs.bufferedStageOutputs[0].ConfirmationOutput.Confirmations) // transaction no longer in queue - te.InFlightTxs = []*InFlightTransactionStageController{mockIT2} + oc.InFlightTxs = []*InFlightTransactionStageController{mockIT2} err = confirmationHandler(ctx, mockIT1.stateManager.GetTxID(), testConfirmation) assert.Regexp(t, "PD011924", err) } diff --git a/core/go/internal/engine/enginespi/base_ledger_types.go b/core/go/internal/engine/enginespi/base_ledger_types.go index 80ffb5e62..1b4dc00c2 100644 --- a/core/go/internal/engine/enginespi/base_ledger_types.go +++ b/core/go/internal/engine/enginespi/base_ledger_types.go @@ -26,6 +26,7 @@ import ( "github.com/hyperledger/firefly-signer/pkg/ethsigner" "github.com/hyperledger/firefly-signer/pkg/ethtypes" "github.com/kaleido-io/paladin/core/internal/msgs" + "github.com/kaleido-io/paladin/core/pkg/blockindexer" "github.com/kaleido-io/paladin/core/pkg/ethclient" "github.com/kaleido-io/paladin/toolkit/pkg/log" @@ -42,6 +43,8 @@ const ( BaseTxStatusSucceeded BaseTxStatus = "Succeeded" // BaseTxStatusFailed happens when an error is reported by the infrastructure runtime BaseTxStatusFailed BaseTxStatus = "Failed" + // BaseTxStatusFailed happens when the indexed transaction hash doesn't match any of the submitted hashes + BaseTxStatusConflict BaseTxStatus = "Conflict" // BaseTxStatusSuspended indicates we are not actively doing any work with this transaction right now, until it's resumed to pending again BaseTxStatusSuspended BaseTxStatus = "Suspended" ) @@ -76,8 +79,6 @@ const ( BaseTxActionRetrieveGasPrice BaseTxAction = "RetrieveGasPrice" // BaseTxActionSubmitTransaction indicates that the transaction has been submitted BaseTxActionSubmitTransaction BaseTxAction = "SubmitTransaction" - // BaseTxActionReceiveReceipt indicates that we have received a receipt for the transaction - BaseTxActionReceiveReceipt BaseTxAction = "ReceiveReceipt" // BaseTxActionConfirmTransaction indicates that the transaction has been confirmed BaseTxActionConfirmTransaction BaseTxAction = "Confirm" ) @@ -106,10 +107,10 @@ type BaseTXUpdates struct { MaxFeePerGas *ethtypes.HexInteger `json:"maxFeePerGas,omitempty"` GasLimit *ethtypes.HexInteger `json:"gas,omitempty"` // note this is required for some methods (eth_estimateGas) TransactionHash *string `json:"transactionHash,omitempty"` - PolicyInfo *fftypes.JSONAny `json:"policyInfo"` FirstSubmit *fftypes.FFTime `json:"firstSubmit,omitempty"` LastSubmit *fftypes.FFTime `json:"lastSubmit,omitempty"` ErrorMessage *string `json:"errorMessage,omitempty"` + SubmittedHashes []string `json:"submittedHashes,omitempty"` } type TransactionHeaders struct { @@ -128,11 +129,10 @@ type ManagedTX struct { DeleteRequested *fftypes.FFTime `json:"deleteRequested,omitempty"` SequenceID string `json:"sequenceId,omitempty"` *ethsigner.Transaction - TransactionHash string `json:"transactionHash,omitempty"` - PolicyInfo *fftypes.JSONAny `json:"policyInfo"` - FirstSubmit *fftypes.FFTime `json:"firstSubmit,omitempty"` - LastSubmit *fftypes.FFTime `json:"lastSubmit,omitempty"` - ErrorMessage string `json:"errorMessage,omitempty"` + TransactionHash string `json:"transactionHash,omitempty"` + FirstSubmit *fftypes.FFTime `json:"firstSubmit,omitempty"` + LastSubmit *fftypes.FFTime `json:"lastSubmit,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` } type BalanceManager interface { @@ -198,10 +198,9 @@ type TransactionStore interface { UpdateTransaction(ctx context.Context, txID string, updates *BaseTXUpdates) error DeleteTransaction(ctx context.Context, txID string) error - GetTransactionReceipt(ctx context.Context, txID string) (receipt *ethclient.TransactionReceiptResponse, err error) - SetTransactionReceipt(ctx context.Context, txID string, receipt *ethclient.TransactionReceiptResponse) error + GetIndexedTransaction(ctx context.Context, txID string) (iTX *blockindexer.IndexedTransaction, err error) + SetIndexedTransaction(ctx context.Context, txID string, iTX *blockindexer.IndexedTransaction) error - AddTransactionConfirmations(ctx context.Context, txID string, clearExisting bool, confirmations ...*Confirmation) error AddSubStatusAction(ctx context.Context, txID string, subStatus BaseTxSubStatus, action BaseTxAction, info *fftypes.JSONAny, err *fftypes.JSONAny, actionOccurred *fftypes.FFTime) error ListTransactions(ctx context.Context, filter ffapi.AndFilter) ([]*ManagedTX, *ffapi.FilterResult, error) @@ -224,7 +223,7 @@ type BaseLedgerTxEngine interface { // Init - setting a set of initialized toolkit plugins in the constructed transaction handler object. Safe checks & initialization // can take place inside this function as well. It also enables toolkit plugins to be able to embed a reference to its parent // transaction handler instance. - Init(ctx context.Context, ethClient ethclient.EthClient, keymgr ethclient.KeyManager, txStore TransactionStore, managedTXEventNotifier ManagedTxEventNotifier, txConfirmationListener TransactionConfirmationListener) + Init(ctx context.Context, ethClient ethclient.EthClient, keymgr ethclient.KeyManager, txStore TransactionStore, managedTXEventNotifier ManagedTxEventNotifier, blockIndexer blockindexer.BlockIndexer) // Start - starting the transaction handler to handle inbound events. // It takes in a context, of which upon cancellation will stop the transaction handler. @@ -252,11 +251,6 @@ type ManagedTxEventNotifier interface { Notify(ctx context.Context, e ManagedTransactionEvent) error } -type TransactionConfirmationListener interface { - Add(ctx context.Context, txID, txHash string, rH ReceiptHandler, cH ConfirmationHandler) error - Remove(ctx context.Context, txHash string) error -} - type RequestOptions struct { ID *uuid.UUID SignerID string @@ -284,17 +278,10 @@ const ( ) type ManagedTransactionEvent struct { - Type ManagedTransactionEventType - Tx *ManagedTX - Receipt *ethclient.TransactionReceiptResponse + Type ManagedTransactionEventType + Tx *ManagedTX } -// ReceiptHandler can be passed on the event as a closure with extra variables -type ReceiptHandler func(ctx context.Context, txID string, receipt *ethclient.TransactionReceiptResponse) error - -// ConfirmationHandler can be passed on the event as a closure with extra variables -type ConfirmationHandler func(ctx context.Context, txID string, notification *ConfirmationsNotification) error - type GasPriceObject struct { MaxPriorityFeePerGas *big.Int `json:"maxPriorityFeePerGas,omitempty"` MaxFeePerGas *big.Int `json:"maxFeePerGas,omitempty"` @@ -393,24 +380,20 @@ const ( SubmissionOutcomeFailedRequiresRetry SubmissionOutcome = "errRequiresRetry" ) -type EnterprisePolicyInfo struct { - LastWarnTime *fftypes.FFTime `json:"lastWarnTime"` - SubmittedTxHashes []string `json:"submittedTxHashes,omitempty"` -} - type InMemoryTxStateReadOnly interface { GetTxID() string GetCreatedTime() *fftypes.FFTime GetDeleteRequestedTime() *fftypes.FFTime // get the transaction receipt from the in-memory state (note: the returned value should not be modified) - GetReceipt() *ethclient.TransactionReceiptResponse + GetIndexedTransaction() *blockindexer.IndexedTransaction GetTransactionHash() string GetNonce() *big.Int GetFrom() string GetStatus() BaseTxStatus GetGasPriceObject() *GasPriceObject GetFirstSubmit() *fftypes.FFTime - GetPolicyInfo() *EnterprisePolicyInfo + GetLastSubmitTime() *fftypes.FFTime + GetSubmittedHashes() []string GetTx() *ManagedTX //TODO: remove the need of this function @@ -424,7 +407,7 @@ type InMemoryTxStateManager interface { } type InMemoryTxStateSetters interface { - SetReceipt(ctx context.Context, receipt *ethclient.TransactionReceiptResponse) + SetIndexedTransaction(ctx context.Context, iTX *blockindexer.IndexedTransaction) ApplyTxUpdates(ctx context.Context, txUpdates *BaseTXUpdates) } @@ -439,8 +422,6 @@ type StageOutput struct { GasPriceOutput *GasPriceOutput - ReceiptOutput *ReceiptOutputs - ConfirmationOutput *ConfirmationOutputs } @@ -462,15 +443,9 @@ type GasPriceOutput struct { Err error } -type ReceiptOutputs struct { - Receipt *ethclient.TransactionReceiptResponse - ReceiptNotify *fftypes.FFTime - Err error -} - type ConfirmationOutputs struct { - Confirmations *ConfirmationsNotification - ConfirmNotify *fftypes.FFTime + IndexedTransaction *blockindexer.IndexedTransaction + Err error } type PersistenceOutput struct { @@ -480,7 +455,6 @@ type PersistenceOutput struct { type InFlightStageActionTriggers interface { TriggerRetrieveGasPrice(ctx context.Context) error - TriggerTracking(ctx context.Context) error TriggerSignTx(ctx context.Context) error TriggerSubmitTx(ctx context.Context, signedMessage []byte) error TriggerStatusUpdate(ctx context.Context) error @@ -519,16 +493,14 @@ func (ctx *RunningStageContext) SetNewPersistenceUpdateOutput() { } type RunningStageContextPersistenceOutput struct { - UpdateType PersistenceUpdateType - InMemoryTx InMemoryTxStateReadOnly - SubStatus BaseTxSubStatus - Ctx context.Context - - PolicyInfo *EnterprisePolicyInfo - TxUpdates *BaseTXUpdates - Receipt *ethclient.TransactionReceiptResponse - HistoryUpdates []func(p TransactionStore) error - Confirmations *ConfirmationsNotification + UpdateType PersistenceUpdateType + InMemoryTx InMemoryTxStateReadOnly + SubStatus BaseTxSubStatus + Ctx context.Context + TxUpdates *BaseTXUpdates + HistoryUpdates []func(p TransactionStore) error + IndexedTransaction *blockindexer.IndexedTransaction + MissedIndexedTransaction bool } func (sOut *RunningStageContextPersistenceOutput) AddSubStatusAction(action BaseTxAction, info *fftypes.JSONAny, err *fftypes.JSONAny) { @@ -550,6 +522,7 @@ type OrchestratorContext struct { // input from transaction engine AvailableToSpend *big.Int PreviousNonceCostUnknown bool + CurrentConfirmedNonce *big.Int } // output of some stages doesn't get written into the database @@ -583,8 +556,7 @@ type InFlightTransactionStateManager interface { AddSubmitOutput(ctx context.Context, txHash string, submissionTime *fftypes.FFTime, submissionOutcome SubmissionOutcome, errorReason ethclient.ErrorReason, err error) AddSignOutput(ctx context.Context, signedMessage []byte, txHash string, err error) AddGasPriceOutput(ctx context.Context, gasPriceObject *GasPriceObject, err error) - AddReceiptOutput(ctx context.Context, rpt *ethclient.TransactionReceiptResponse, err error) - AddConfirmationsOutput(ctx context.Context, cmfs *ConfirmationsNotification) + AddConfirmationsOutput(ctx context.Context, indexedTx *blockindexer.IndexedTransaction) AddPanicOutput(ctx context.Context, stage InFlightTxStage) PersistTxState(ctx context.Context) (stage InFlightTxStage, persistenceTime time.Time, err error) diff --git a/core/go/internal/msgs/en_errors.go b/core/go/internal/msgs/en_errors.go index 2a6527455..c8b33ce4d 100644 --- a/core/go/internal/msgs/en_errors.go +++ b/core/go/internal/msgs/en_errors.go @@ -268,23 +268,16 @@ var ( MsgInvalidGasClientConfig = ffe("PD011908", "Invalid gas client config: %s") MsgInvalidGasPriceIncreaseMax = ffe("PD011909", "Invalid max gas price increase price string %s") MsgMissingTransactionID = ffe("PD011910", "Transaction ID must be provided") - MsgUnsupportedAction = ffe("PD011911", "%s action is not supported") - MsgReturnValueNotDecoded = ffe("PD011912", "Error return value for custom error: %s") - MsgReturnValueNotAvailable = ffe("PD011913", "Error return value unavailable") - MsgReceiptNotAvailable = ffe("PD011914", "Receipt not available for transaction '%s'") MsgGasPriceError = ffe("PD011917", `The gasPrice '%s' could not be parsed. Must be a numeric string, or an object with 'gasPrice' field, or 'maxFeePerGas'/'maxPriorityFeePerGas' fields (EIP-1559), error: %s`) MsgPersistError = ffe("PD011918", "Unexpected internal error, cannot persist stage.") MsgInvalidStageOutput = ffe("PD011919", "Stage output object is missing %s: %+v") MsgInvalidGasLimit = ffe("PD011920", "Invalid gas limit, must be a positive number") MsgStatusUpdateForbidden = ffe("PD011921", "Cannot update status of a completed transaction") - MsgConfirmationHandlerNotFound = ffe("PD011922", "Unexpected internal error, no handler set for confirmation") - MsgReceiptHandlerNotFound = ffe("PD011923", "Unexpected internal error, no handler set for receipt") MsgTransactionNotFound = ffe("PD011924", "Transaction '%s' not found") - MsgInvalidChainID = ffe("PD011925", "Invalid chain ID string %s") MsgTransactionEngineRequestTimeout = ffe("PD011926", "The transaction handler did not acknowledge the request after %.2fs") - MsgErrorFromSigningService = ffe("PD011927", "Error from signing service: %s") MsgErrorMissingSignerID = ffe("PD011928", "Signer Identifier must be provided") MsgInvalidTransactionType = ffe("PD011929", "Transaction type invalid") + MsgMissingIndexedTransaction = ffe("PD011930", "Transaction %s with nonce smaller than the recorded confirmed nonce cannot find an indexed transaction.") // TransportManager module PD0120XX MsgTransportInvalidMessage = ffe("PD012000", "Invalid message") diff --git a/core/go/pkg/blockindexer/block_indexer.go b/core/go/pkg/blockindexer/block_indexer.go index 2ebd0ec74..d15ab1aa6 100644 --- a/core/go/pkg/blockindexer/block_indexer.go +++ b/core/go/pkg/blockindexer/block_indexer.go @@ -46,9 +46,10 @@ import ( type BlockIndexer interface { Start(internalStreams ...*InternalEventStream) error Stop() - RegisterIndexedTransactionHandler(ctx context.Context, hookFunction func(indexedTransactions []*IndexedTransaction) error) error + RegisterIndexedTransactionHandler(ctx context.Context, hookFunction func(ctx context.Context, indexedTransactions []*IndexedTransaction) error) error GetIndexedBlockByNumber(ctx context.Context, number uint64) (*IndexedBlock, error) GetIndexedTransactionByHash(ctx context.Context, hash tktypes.Bytes32) (*IndexedTransaction, error) + GetIndexedTransactionByNonce(ctx context.Context, from tktypes.EthAddress, nonce uint64) (*IndexedTransaction, error) GetBlockTransactionsByNumber(ctx context.Context, blockNumber int64) ([]*IndexedTransaction, error) GetTransactionEventsByHash(ctx context.Context, hash tktypes.Bytes32) ([]*IndexedEvent, error) ListTransactionEvents(ctx context.Context, lastBlock int64, lastIndex, limit int, withTransaction, withBlock bool) ([]*IndexedEvent, error) @@ -70,7 +71,7 @@ type BlockIndexer interface { type blockIndexer struct { parentCtxForReset context.Context hookRegisterMutex sync.Mutex - indexedTransactionPrePersistHook func(indexedTransactions []*IndexedTransaction) error + indexedTransactionPrePersistHook func(ctx context.Context, indexedTransactions []*IndexedTransaction) error cancelFunc func() persistence persistence.Persistence blockListener *blockListener @@ -148,7 +149,7 @@ func (bi *blockIndexer) Start(internalStreams ...*InternalEventStream) error { return nil } -func (bi *blockIndexer) RegisterIndexedTransactionHandler(ctx context.Context, hookFunction func(indexedTransactions []*IndexedTransaction) error) (err error) { +func (bi *blockIndexer) RegisterIndexedTransactionHandler(ctx context.Context, hookFunction func(ctx context.Context, indexedTransactions []*IndexedTransaction) error) (err error) { bi.hookRegisterMutex.Lock() defer bi.hookRegisterMutex.Unlock() if bi.indexedTransactionPrePersistHook != nil { @@ -544,7 +545,7 @@ func (bi *blockIndexer) writeBatch(ctx context.Context, batch *blockWriterBatch) } if bi.indexedTransactionPrePersistHook != nil { - if prePersistHookError = bi.indexedTransactionPrePersistHook(transactions); prePersistHookError != nil { + if prePersistHookError = bi.indexedTransactionPrePersistHook(ctx, transactions); prePersistHookError != nil { // if pre persist hook failed, don't write to db return prePersistHookError } @@ -747,6 +748,22 @@ func (bi *blockIndexer) GetIndexedBlockByNumber(ctx context.Context, number uint return blocks[0], nil } +func (bi *blockIndexer) GetIndexedTransactionByNonce(ctx context.Context, from tktypes.EthAddress, nonce uint64) (*IndexedTransaction, error) { + var txns []*IndexedTransaction + db := bi.persistence.DB() + err := db. + WithContext(ctx). + Table("indexed_transactions"). + Where("from = ?", from). + Where("nonce = ?", nonce). + Find(&txns). + Error + if err != nil || len(txns) < 1 { + return nil, err + } + return txns[0], nil +} + func (bi *blockIndexer) GetIndexedTransactionByHash(ctx context.Context, hash tktypes.Bytes32) (*IndexedTransaction, error) { return bi.getIndexedTransactionByHash(ctx, hash) } diff --git a/core/go/pkg/blockindexer/block_indexer_test.go b/core/go/pkg/blockindexer/block_indexer_test.go index 07022adba..ce28f2fbd 100644 --- a/core/go/pkg/blockindexer/block_indexer_test.go +++ b/core/go/pkg/blockindexer/block_indexer_test.go @@ -718,7 +718,7 @@ func TestBlockIndexerResetsWhenPrePersistHookFails(t *testing.T) { return blocks, receipts }) - err := bi.RegisterIndexedTransactionHandler(ctx, func(indexedTransactions []*IndexedTransaction) error { + err := bi.RegisterIndexedTransactionHandler(ctx, func(ctx context.Context, indexedTransactions []*IndexedTransaction) error { if !prePersistHookFailed && indexedTransactions[0].Hash == tktypes.NewBytes32FromSlice(blocks[2].Transactions[0].Hash) { prePersistHookFailed = true return fmt.Errorf("pop") @@ -727,7 +727,7 @@ func TestBlockIndexerResetsWhenPrePersistHookFails(t *testing.T) { }) assert.NoError(t, err) - err = bi.RegisterIndexedTransactionHandler(ctx, func(indexedTransactions []*IndexedTransaction) error { + err = bi.RegisterIndexedTransactionHandler(ctx, func(ctx context.Context, indexedTransactions []*IndexedTransaction) error { panic("should not override the first hook") }) assert.Regexp(t, "PD011309", err) diff --git a/core/go/pkg/ethclient/client.go b/core/go/pkg/ethclient/client.go index e26409546..aac757e28 100644 --- a/core/go/pkg/ethclient/client.go +++ b/core/go/pkg/ethclient/client.go @@ -16,15 +16,11 @@ package ethclient import ( - "bytes" "context" - "encoding/hex" "encoding/json" "fmt" "math/big" - "strings" - "github.com/hyperledger/firefly-common/pkg/fftypes" "github.com/hyperledger/firefly-common/pkg/i18n" "github.com/hyperledger/firefly-signer/pkg/abi" "github.com/hyperledger/firefly-signer/pkg/ethsigner" @@ -56,7 +52,6 @@ type EthClient interface { GetBalance(ctx context.Context, address string, block string) (balance *ethtypes.HexInteger, err error) GasEstimate(ctx context.Context, tx *ethsigner.Transaction) (gasLimit *ethtypes.HexInteger, err error) GetTransactionCount(ctx context.Context, fromAddr string) (transactionCount *ethtypes.HexUint64, err error) - GetTransactionReceipt(ctx context.Context, txHash string) (*TransactionReceiptResponse, error) CallContract(ctx context.Context, from *string, tx *ethsigner.Transaction, block string) (data tktypes.HexBytes, err error) BuildRawTransaction(ctx context.Context, txVersion EthTXVersion, from string, tx *ethsigner.Transaction) (tktypes.HexBytes, error) SendRawTransaction(ctx context.Context, rawTX tktypes.HexBytes) (*tktypes.Bytes32, error) @@ -151,59 +146,6 @@ func (ec *ethClient) GasPrice(ctx context.Context) (*ethtypes.HexInteger, error) return &gasPrice, nil } -func (ec *ethClient) GetTransactionReceipt(ctx context.Context, txHash string) (*TransactionReceiptResponse, error) { - - // Get the receipt in the back-end JSON/RPC format - var ethReceipt *txReceiptJSONRPC - rpcErr := ec.rpc.CallRPC(ctx, ðReceipt, "eth_getTransactionReceipt", txHash) - if rpcErr != nil { - return nil, rpcErr.Error() - } - if ethReceipt == nil { - return nil, i18n.NewError(ctx, msgs.MsgReceiptNotAvailable, txHash) - } - isSuccess := (ethReceipt.Status != nil && ethReceipt.Status.BigInt().Int64() > 0) - - var returnDataString *string - var transactionErrorMessage *string - - if !isSuccess { - returnDataString, transactionErrorMessage = ec.getErrorInfo(ctx, ethReceipt.RevertReason) - } - - fullReceipt, _ := json.Marshal(&receiptExtraInfo{ - ContractAddress: ethReceipt.ContractAddress, - CumulativeGasUsed: (*fftypes.FFBigInt)(ethReceipt.CumulativeGasUsed), - From: ethReceipt.From, - To: ethReceipt.To, - GasUsed: (*fftypes.FFBigInt)(ethReceipt.GasUsed), - Status: (*fftypes.FFBigInt)(ethReceipt.Status), - ReturnValue: returnDataString, - ErrorMessage: transactionErrorMessage, - }) - - var txIndex int64 - if ethReceipt.TransactionIndex != nil { - txIndex = ethReceipt.TransactionIndex.BigInt().Int64() - } - receiptResponse := &TransactionReceiptResponse{ - BlockNumber: (*fftypes.FFBigInt)(ethReceipt.BlockNumber), - TransactionIndex: fftypes.NewFFBigInt(txIndex), - BlockHash: ethReceipt.BlockHash.String(), - Success: isSuccess, - ProtocolID: ProtocolIDForReceipt((*fftypes.FFBigInt)(ethReceipt.BlockNumber), fftypes.NewFFBigInt(txIndex)), - ExtraInfo: fftypes.JSONAnyPtrBytes(fullReceipt), - } - - if ethReceipt.ContractAddress != nil { - location, _ := json.Marshal(map[string]string{ - "address": ethReceipt.ContractAddress.String(), - }) - receiptResponse.ContractLocation = fftypes.JSONAnyPtrBytes(location) - } - return receiptResponse, nil -} - func (ec *ethClient) GasEstimate(ctx context.Context, tx *ethsigner.Transaction) (*ethtypes.HexInteger, error) { var gasEstimate ethtypes.HexInteger if rpcErr := ec.rpc.CallRPC(ctx, &gasEstimate, "eth_estimateGas", tx); rpcErr != nil { @@ -324,30 +266,3 @@ func logJSON(v interface{}) string { } return ret } -func (ec *ethClient) getErrorInfo(ctx context.Context, revertFromReceipt *ethtypes.HexBytes0xPrefix) (pReturnValue *string, pErrorMessage *string) { - - var revertReason string - if revertFromReceipt != nil { - revertReason = revertFromReceipt.String() - } - - // See if the return value is using the default error you get from "revert" - var errorMessage string - returnDataBytes, _ := hex.DecodeString(strings.TrimPrefix(revertReason, "0x")) - if len(returnDataBytes) > 4 && bytes.Equal(returnDataBytes[0:4], defaultErrorID) { - value, err := defaultError.DecodeCallDataCtx(ctx, returnDataBytes) - if err == nil { - errorMessage = value.Children[0].Value.(string) - } - } - - // Otherwise we can't decode it, so put it directly in the error - if errorMessage == "" { - if len(returnDataBytes) > 0 { - errorMessage = i18n.NewError(ctx, msgs.MsgReturnValueNotDecoded, revertReason).Error() - } else { - errorMessage = i18n.NewError(ctx, msgs.MsgReturnValueNotAvailable).Error() - } - } - return &revertReason, &errorMessage -} diff --git a/core/go/pkg/ethclient/client_test.go b/core/go/pkg/ethclient/client_test.go index 82d00bddd..ad132849d 100644 --- a/core/go/pkg/ethclient/client_test.go +++ b/core/go/pkg/ethclient/client_test.go @@ -21,7 +21,6 @@ import ( "math/big" "testing" - "github.com/hyperledger/firefly-common/pkg/fftypes" "github.com/hyperledger/firefly-signer/pkg/ethsigner" "github.com/hyperledger/firefly-signer/pkg/ethtypes" "github.com/kaleido-io/paladin/core/pkg/proto" @@ -258,126 +257,3 @@ func TestSendRawFail(t *testing.T) { assert.Regexp(t, "pop", err) } - -const testTxHash = "0x7d48ae971faf089878b57e3c28e3035540d34f38af395958d2c73c36c57c83a2" - -func TestGetReceiptOkSuccess(t *testing.T) { - sampleJSONRPCReceipt := &txReceiptJSONRPC{ - BlockNumber: ethtypes.NewHexInteger64(1988), - TransactionIndex: ethtypes.NewHexInteger64(30), - Status: ethtypes.NewHexInteger64(1), - ContractAddress: ethtypes.MustNewAddress("0x87ae94ab290932c4e6269648bb47c86978af4436"), - } - ctx, ec, done := newTestClientAndServer(t, &mockEth{ - eth_getTransactionReceipt: func(context.Context, ethtypes.HexBytes0xPrefix) (*txReceiptJSONRPC, error) { - return sampleJSONRPCReceipt, nil - }, - }) - defer done() - - receipt, err := ec.HTTPClient().GetTransactionReceipt(ctx, testTxHash) - require.NoError(t, err) - - assert.True(t, receipt.Success) - assert.Equal(t, int64(1988), receipt.BlockNumber.Int64()) - assert.Equal(t, int64(30), receipt.TransactionIndex.Int64()) -} - -func TestGetReceiptOkFailed(t *testing.T) { - revertReasonTooSmallHex := ethtypes.MustNewHexBytes0xPrefix("0x08c379a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001d5468652073746f7265642076616c756520697320746f6f20736d616c6c000000") - sampleJSONRPCReceipt := &txReceiptJSONRPC{ - BlockNumber: ethtypes.NewHexInteger64(1988), - TransactionIndex: ethtypes.NewHexInteger64(30), - Status: ethtypes.NewHexInteger64(0), - ContractAddress: ethtypes.MustNewAddress("0x87ae94ab290932c4e6269648bb47c86978af4436"), - RevertReason: &revertReasonTooSmallHex, - } - ctx, ec, done := newTestClientAndServer(t, &mockEth{ - eth_getTransactionReceipt: func(context.Context, ethtypes.HexBytes0xPrefix) (*txReceiptJSONRPC, error) { - return sampleJSONRPCReceipt, nil - }, - }) - defer done() - - receipt, err := ec.HTTPClient().GetTransactionReceipt(ctx, testTxHash) - require.NoError(t, err) - - assert.False(t, receipt.Success) - assert.Contains(t, receipt.ExtraInfo.String(), "The stored value is too small") -} - -func TestGetReceiptOkFailedMissingReason(t *testing.T) { - sampleJSONRPCReceipt := &txReceiptJSONRPC{ - BlockNumber: ethtypes.NewHexInteger64(1988), - TransactionIndex: ethtypes.NewHexInteger64(30), - Status: ethtypes.NewHexInteger64(0), - ContractAddress: ethtypes.MustNewAddress("0x87ae94ab290932c4e6269648bb47c86978af4436"), - } - ctx, ec, done := newTestClientAndServer(t, &mockEth{ - eth_getTransactionReceipt: func(context.Context, ethtypes.HexBytes0xPrefix) (*txReceiptJSONRPC, error) { - return sampleJSONRPCReceipt, nil - }, - }) - defer done() - - receipt, err := ec.HTTPClient().GetTransactionReceipt(ctx, testTxHash) - require.NoError(t, err) - - assert.False(t, receipt.Success) - assert.Contains(t, receipt.ExtraInfo.String(), "PD011913") -} - -func TestGetReceiptOkFailedCustomReason(t *testing.T) { - revertCustomHex := ethtypes.MustNewHexBytes0xPrefix("0x08c379a0000000000000000000000000000000000000000000000000000000000000003e000000000000000000000000000000000000000000000000000000000000001d5468652073746f7265642076616c756520697320746f6f20736d616c6c000000") - - sampleJSONRPCReceipt := &txReceiptJSONRPC{ - BlockNumber: ethtypes.NewHexInteger64(1988), - TransactionIndex: ethtypes.NewHexInteger64(30), - Status: ethtypes.NewHexInteger64(0), - ContractAddress: ethtypes.MustNewAddress("0x87ae94ab290932c4e6269648bb47c86978af4436"), - RevertReason: &revertCustomHex, - } - ctx, ec, done := newTestClientAndServer(t, &mockEth{ - eth_getTransactionReceipt: func(context.Context, ethtypes.HexBytes0xPrefix) (*txReceiptJSONRPC, error) { - return sampleJSONRPCReceipt, nil - }, - }) - defer done() - - receipt, err := ec.HTTPClient().GetTransactionReceipt(ctx, testTxHash) - require.NoError(t, err) - - assert.False(t, receipt.Success) - assert.Contains(t, receipt.ExtraInfo.String(), revertCustomHex.String()) -} - -func TestGetReceiptError(t *testing.T) { - - ctx, ec, done := newTestClientAndServer(t, &mockEth{ - eth_getTransactionReceipt: func(context.Context, ethtypes.HexBytes0xPrefix) (*txReceiptJSONRPC, error) { - return nil, fmt.Errorf("pop") - }, - }) - defer done() - - _, err := ec.HTTPClient().GetTransactionReceipt(ctx, testTxHash) - assert.Regexp(t, "pop", err) -} - -func TestGetReceiptNotFound(t *testing.T) { - - ctx, ec, done := newTestClientAndServer(t, &mockEth{ - eth_getTransactionReceipt: func(context.Context, ethtypes.HexBytes0xPrefix) (*txReceiptJSONRPC, error) { - return nil, nil - }, - }) - defer done() - - _, err := ec.HTTPClient().GetTransactionReceipt(ctx, testTxHash) - assert.Regexp(t, "PD011914", err) - -} -func TestProtocolIDForReceipt(t *testing.T) { - assert.Equal(t, "000000012345/000042", ProtocolIDForReceipt(fftypes.NewFFBigInt(12345), fftypes.NewFFBigInt(42))) - assert.Equal(t, "", ProtocolIDForReceipt(nil, nil)) -} diff --git a/core/go/pkg/ethclient/factory_test.go b/core/go/pkg/ethclient/factory_test.go index ba57b2489..9b870530a 100644 --- a/core/go/pkg/ethclient/factory_test.go +++ b/core/go/pkg/ethclient/factory_test.go @@ -35,15 +35,14 @@ import ( ) type mockEth struct { - eth_getBalance func(context.Context, ethtypes.Address0xHex, string) (ethtypes.HexInteger, error) - eth_gasPrice func(context.Context) (ethtypes.HexInteger, error) - eth_gasLimit func(context.Context, ethsigner.Transaction) (ethtypes.HexInteger, error) - eth_chainId func(context.Context) (ethtypes.HexUint64, error) - eth_getTransactionCount func(context.Context, ethtypes.Address0xHex, string) (ethtypes.HexUint64, error) - eth_getTransactionReceipt func(context.Context, ethtypes.HexBytes0xPrefix) (*txReceiptJSONRPC, error) - eth_estimateGas func(context.Context, ethsigner.Transaction) (ethtypes.HexInteger, error) - eth_sendRawTransaction func(context.Context, tktypes.HexBytes) (tktypes.HexBytes, error) - eth_call func(context.Context, ethsigner.Transaction, string) (tktypes.HexBytes, error) + eth_getBalance func(context.Context, ethtypes.Address0xHex, string) (ethtypes.HexInteger, error) + eth_gasPrice func(context.Context) (ethtypes.HexInteger, error) + eth_gasLimit func(context.Context, ethsigner.Transaction) (ethtypes.HexInteger, error) + eth_chainId func(context.Context) (ethtypes.HexUint64, error) + eth_getTransactionCount func(context.Context, ethtypes.Address0xHex, string) (ethtypes.HexUint64, error) + eth_estimateGas func(context.Context, ethsigner.Transaction) (ethtypes.HexInteger, error) + eth_sendRawTransaction func(context.Context, tktypes.HexBytes) (tktypes.HexBytes, error) + eth_call func(context.Context, ethsigner.Transaction, string) (tktypes.HexBytes, error) } func newTestServer(t *testing.T, ctx context.Context, isWS bool, mEth *mockEth) (rpcServer rpcserver.RPCServer, done func()) { @@ -84,7 +83,6 @@ func newTestServer(t *testing.T, ctx context.Context, isWS bool, mEth *mockEth) rpcServer.Register(rpcserver.NewRPCModule("eth"). Add("eth_chainId", checkNil(mEth.eth_chainId, rpcserver.RPCMethod0)). Add("eth_getTransactionCount", checkNil(mEth.eth_getTransactionCount, rpcserver.RPCMethod2)). - Add("eth_getTransactionReceipt", checkNil(mEth.eth_getTransactionReceipt, rpcserver.RPCMethod1)). Add("eth_estimateGas", checkNil(mEth.eth_estimateGas, rpcserver.RPCMethod1)). Add("eth_sendRawTransaction", checkNil(mEth.eth_sendRawTransaction, rpcserver.RPCMethod1)). Add("eth_call", checkNil(mEth.eth_call, rpcserver.RPCMethod2)). diff --git a/core/go/pkg/ethclient/types.go b/core/go/pkg/ethclient/types.go index 0b898be15..188a60f4c 100644 --- a/core/go/pkg/ethclient/types.go +++ b/core/go/pkg/ethclient/types.go @@ -17,9 +17,6 @@ package ethclient import ( "strings" - - "github.com/hyperledger/firefly-common/pkg/fftypes" - "github.com/hyperledger/firefly-signer/pkg/ethtypes" ) // ErrorReason are a set of standard error conditions that a blockchain connector can return @@ -89,68 +86,3 @@ func MapError(err error) ErrorReason { return "" } } - -// txReceiptJSONRPC is the receipt obtained over JSON/RPC from the ethereum client, with gas used, logs and contract address -type txReceiptJSONRPC struct { - BlockHash ethtypes.HexBytes0xPrefix `json:"blockHash"` - BlockNumber *ethtypes.HexInteger `json:"blockNumber"` - ContractAddress *ethtypes.Address0xHex `json:"contractAddress"` - CumulativeGasUsed *ethtypes.HexInteger `json:"cumulativeGasUsed"` - From *ethtypes.Address0xHex `json:"from"` - GasUsed *ethtypes.HexInteger `json:"gasUsed"` - Logs []*logJSONRPC `json:"logs"` - Status *ethtypes.HexInteger `json:"status"` - To *ethtypes.Address0xHex `json:"to"` - TransactionHash ethtypes.HexBytes0xPrefix `json:"transactionHash"` - TransactionIndex *ethtypes.HexInteger `json:"transactionIndex"` - RevertReason *ethtypes.HexBytes0xPrefix `json:"revertReason"` -} - -type logJSONRPC struct { - Removed bool `json:"removed"` - LogIndex *ethtypes.HexInteger `json:"logIndex"` - TransactionIndex *ethtypes.HexInteger `json:"transactionIndex"` - BlockNumber *ethtypes.HexInteger `json:"blockNumber"` - TransactionHash ethtypes.HexBytes0xPrefix `json:"transactionHash"` - BlockHash ethtypes.HexBytes0xPrefix `json:"blockHash"` - Address *ethtypes.Address0xHex `json:"address"` - Data ethtypes.HexBytes0xPrefix `json:"data"` - Topics []ethtypes.HexBytes0xPrefix `json:"topics"` -} - -type TransactionReceiptResponse struct { - BlockNumber *fftypes.FFBigInt `json:"blockNumber"` - TransactionIndex *fftypes.FFBigInt `json:"transactionIndex"` - BlockHash string `json:"blockHash"` - Success bool `json:"success"` - ProtocolID string `json:"protocolId"` - ExtraInfo *fftypes.JSONAny `json:"extraInfo,omitempty"` - ContractLocation *fftypes.JSONAny `json:"contractLocation,omitempty"` - Logs []fftypes.JSONAny `json:"logs,omitempty"` // all raw un-decoded logs should be included if includeLogs=true -} - -// receiptExtraInfo is the version of the receipt we store under the TX. -// - We omit the full logs from the JSON/RPC -// - We omit fields already in the standardized cross-blockchain section -// - We format numbers as decimals -type receiptExtraInfo struct { - ContractAddress *ethtypes.Address0xHex `json:"contractAddress"` - CumulativeGasUsed *fftypes.FFBigInt `json:"cumulativeGasUsed"` - From *ethtypes.Address0xHex `json:"from"` - To *ethtypes.Address0xHex `json:"to"` - GasUsed *fftypes.FFBigInt `json:"gasUsed"` - Status *fftypes.FFBigInt `json:"status"` - ErrorMessage *string `json:"errorMessage"` - ReturnValue *string `json:"returnValue,omitempty"` -} - -type StructLog struct { - PC *fftypes.FFBigInt `json:"pc"` - Op *string `json:"op"` - Gas *fftypes.FFBigInt `json:"gas"` - GasCost *fftypes.FFBigInt `json:"gasCost"` - Depth *fftypes.FFBigInt `json:"depth"` - Stack []*string `json:"stack"` - Memory []*string `json:"memory"` - Reason *string `json:"reason"` -} diff --git a/core/go/pkg/ethclient/utils.go b/core/go/pkg/ethclient/utils.go deleted file mode 100644 index fd85b286b..000000000 --- a/core/go/pkg/ethclient/utils.go +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright © 2024 Kaleido, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package ethclient - -import ( - "fmt" - - "github.com/hyperledger/firefly-common/pkg/fftypes" - "github.com/hyperledger/firefly-signer/pkg/abi" -) - -var ( - // See https://docs.soliditylang.org/en/v0.8.14/control-structures.html#revert - // There default error for `revert("some error")` is a function Error(string) - defaultError = &abi.Entry{ - Type: abi.Error, - Name: "Error", - Inputs: abi.ParameterArray{ - { - Type: "string", - }, - }, - } - defaultErrorID = defaultError.FunctionSelectorBytes() -) - -func ProtocolIDForReceipt(blockNumber, transactionIndex *fftypes.FFBigInt) string { - if blockNumber != nil && transactionIndex != nil { - return fmt.Sprintf("%.12d/%.6d", blockNumber.Int(), transactionIndex.Int()) - } - return "" -} diff --git a/core/go/pkg/signer/signing_module_test.go b/core/go/pkg/signer/signing_module_test.go index 5b3f1109f..3c065ab45 100644 --- a/core/go/pkg/signer/signing_module_test.go +++ b/core/go/pkg/signer/signing_module_test.go @@ -36,8 +36,8 @@ type testExtension struct { keyStore func(ctx context.Context, config *api.StoreConfig) (store api.KeyStore, err error) } -func (te *testExtension) KeyStore(ctx context.Context, config *api.StoreConfig) (store api.KeyStore, err error) { - return te.keyStore(ctx, config) +func (oc *testExtension) KeyStore(ctx context.Context, config *api.StoreConfig) (store api.KeyStore, err error) { + return oc.keyStore(ctx, config) } type testKeyStoreAll struct { @@ -74,7 +74,7 @@ func (tk *testKeyStoreAll) Close() { func TestExtensionInitFail(t *testing.T) { - te := &testExtension{ + oc := &testExtension{ keyStore: func(ctx context.Context, config *api.StoreConfig) (store api.KeyStore, err error) { assert.Equal(t, "ext-store", config.Type) return nil, fmt.Errorf("pop") @@ -85,21 +85,21 @@ func TestExtensionInitFail(t *testing.T) { KeyStore: api.StoreConfig{ Type: "ext-store", }, - }, te) + }, oc) assert.Regexp(t, "pop", err) } func TestKeystoreTypeUnknown(t *testing.T) { - te := &testExtension{ + oc := &testExtension{ keyStore: func(ctx context.Context, config *api.StoreConfig) (store api.KeyStore, err error) { return nil, nil }, } _, err := NewSigningModule(context.Background(), &api.Config{ KeyStore: api.StoreConfig{ Type: "unknown", }, - }, te) + }, oc) assert.Regexp(t, "PD011407", err) } @@ -140,7 +140,7 @@ func TestExtensionKeyStoreListOK(t *testing.T) { return testRes, nil }, } - te := &testExtension{ + oc := &testExtension{ keyStore: func(ctx context.Context, config *api.StoreConfig) (store api.KeyStore, err error) { assert.Equal(t, "ext-store", config.Type) return tk, nil @@ -151,7 +151,7 @@ func TestExtensionKeyStoreListOK(t *testing.T) { KeyStore: api.StoreConfig{ Type: "ext-store", }, - }, te) + }, oc) require.NoError(t, err) res, err := sm.List(context.Background(), &proto.ListKeysRequest{ @@ -178,7 +178,7 @@ func TestExtensionKeyStoreListFail(t *testing.T) { return nil, fmt.Errorf("pop") }, } - te := &testExtension{ + oc := &testExtension{ keyStore: func(ctx context.Context, config *api.StoreConfig) (store api.KeyStore, err error) { assert.Equal(t, "ext-store", config.Type) return tk, nil @@ -189,7 +189,7 @@ func TestExtensionKeyStoreListFail(t *testing.T) { KeyStore: api.StoreConfig{ Type: "ext-store", }, - }, te) + }, oc) require.NoError(t, err) _, err = sm.List(context.Background(), &proto.ListKeysRequest{ @@ -213,7 +213,7 @@ func TestExtensionKeyStoreResolveSignSECP256K1OK(t *testing.T) { return &secp256k1.SignatureData{V: big.NewInt(1), R: big.NewInt(2), S: big.NewInt(3)}, nil }, } - te := &testExtension{ + oc := &testExtension{ keyStore: func(ctx context.Context, config *api.StoreConfig) (store api.KeyStore, err error) { assert.Equal(t, "ext-store", config.Type) return tk, nil @@ -224,7 +224,7 @@ func TestExtensionKeyStoreResolveSignSECP256K1OK(t *testing.T) { KeyStore: api.StoreConfig{ Type: "ext-store", }, - }, te) + }, oc) require.NoError(t, err) resResolve, err := sm.Resolve(context.Background(), &proto.ResolveKeyRequest{ @@ -251,7 +251,7 @@ func TestExtensionKeyStoreResolveSECP256K1Fail(t *testing.T) { return nil, "", fmt.Errorf("pop") }, } - te := &testExtension{ + oc := &testExtension{ keyStore: func(ctx context.Context, config *api.StoreConfig) (store api.KeyStore, err error) { assert.Equal(t, "ext-store", config.Type) return tk, nil @@ -262,7 +262,7 @@ func TestExtensionKeyStoreResolveSECP256K1Fail(t *testing.T) { KeyStore: api.StoreConfig{ Type: "ext-store", }, - }, te) + }, oc) require.NoError(t, err) _, err = sm.Resolve(context.Background(), &proto.ResolveKeyRequest{ @@ -283,7 +283,7 @@ func TestExtensionKeyStoreSignSECP256K1Fail(t *testing.T) { return nil, fmt.Errorf("pop") }, } - te := &testExtension{ + oc := &testExtension{ keyStore: func(ctx context.Context, config *api.StoreConfig) (store api.KeyStore, err error) { assert.Equal(t, "ext-store", config.Type) return tk, nil @@ -294,7 +294,7 @@ func TestExtensionKeyStoreSignSECP256K1Fail(t *testing.T) { KeyStore: api.StoreConfig{ Type: "ext-store", }, - }, te) + }, oc) require.NoError(t, err) _, err = sm.Sign(context.Background(), &proto.SignRequest{ diff --git a/toolkit/go/pkg/tktypes/enum_persisted_test.go b/toolkit/go/pkg/tktypes/enum_persisted_test.go index f1632ec12..c44f70663 100644 --- a/toolkit/go/pkg/tktypes/enum_persisted_test.go +++ b/toolkit/go/pkg/tktypes/enum_persisted_test.go @@ -32,7 +32,7 @@ const ( TestEnumOption3 TestEnum = "option3" ) -func (te TestEnum) Options() []string { +func (oc TestEnum) Options() []string { return []string{ string(TestEnumOption1), string(TestEnumOption2), @@ -40,7 +40,7 @@ func (te TestEnum) Options() []string { } } -func (te TestEnum) Default() string { +func (oc TestEnum) Default() string { return string(TestEnumOption2) }