From 5c35437bf865c3457d015a873a8ef07789c4ffe9 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 15 May 2026 13:46:23 +0200 Subject: [PATCH 001/100] new mempool draft --- sei-tendermint/internal/mempool/mempool.go | 759 +++--------------- .../internal/mempool/priority_queue.go | 49 -- sei-tendermint/internal/mempool/tx.go | 442 +++++----- 3 files changed, 297 insertions(+), 953 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index c038a05b90..1bf3563aad 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -1,7 +1,6 @@ package mempool import ( - "bytes" "context" "crypto/sha256" "errors" @@ -197,35 +196,7 @@ type TxMempool struct { // txStore defines the main storage of valid transactions. Indexes are built // on top of this store. - txStore *TxStore - - // gossipIndex defines the gossiping index of valid transactions via a - // thread-safe linked-list. We also use the gossip index as a cursor for - // rechecking transactions already in the mempool. - gossipIndex *clist.CList[*WrappedTx] - - // recheckCursor and recheckEnd are used as cursors based on the gossip index - // to recheck transactions that are already in the mempool. Iteration is not - // thread-safe and transaction may be mutated in serial order. - // - // XXX/TODO: It might be somewhat of a codesmell to use the gossip index for - // iterator and cursor management when rechecking transactions. If the gossip - // index changes or is removed in a future refactor, this will have to be - // refactored. Instead, we should consider just keeping a slice of a snapshot - // of the mempool's current transactions during Update and an integer cursor - // into that slice. This, however, requires additional O(n) space complexity. - recheckCursor *clist.CElement[*WrappedTx] // next expected response - recheckEnd *clist.CElement[*WrappedTx] // re-checking stops here - - // priorityIndex defines the priority index of valid transactions via a - // thread-safe priority queue. - priorityIndex *TxPriorityQueue - - // pendingTxs stores transactions that are not valid yet but might become valid - // once nonce ordering or sender balance catches up. - pendingTxs *PendingTxs - - byAddrNonce utils.Mutex[map[evmAddrNonce]*WrappedTx] + txStore *txStoreV2 // A read/write lock is used to safe guard updates, insertions and deletions // from the mempool. A read-lock is implicitly acquired when executing CheckTx, @@ -253,10 +224,6 @@ func NewTxMempool( blockFailedTxs: NopTxCache{}, metrics: metrics, txStore: NewTxStore(), - gossipIndex: clist.New[*WrappedTx](), - priorityIndex: NewTxPriorityQueue(), - pendingTxs: NewPendingTxs(cfg), - byAddrNonce: utils.NewMutex(map[evmAddrNonce]*WrappedTx{}), txConstraintsFetcher: txConstraintsFetcher, priorityReservoir: reservoir.New[int64](cfg.DropPriorityReservoirSize, cfg.DropPriorityThreshold, nil), // Use non-deterministic RNG } @@ -278,46 +245,10 @@ func (txmp *TxMempool) Config() *Config { return txmp.config } func (txmp *TxMempool) App() *proxy.Proxy { return txmp.app } func (txmp *TxMempool) EvmNextPendingNonce(addr common.Address) uint64 { - an := evmAddrNonce{addr, txmp.app.EvmNonce(addr)} - for byAddrNonce := range txmp.byAddrNonce.Lock() { - for { - if _, ok := byAddrNonce[an]; !ok { - break - } - an.Nonce += 1 - } - } - return an.Nonce + return txmp.txStore.NextNonce(addr) } -func (txmp *TxMempool) addNonce(wtx *WrappedTx) { - evm, ok := wtx.evm.Get() - if !ok { - return - } - an := evmAddrNonce{evm.address, evm.nonce} - for byAddrNonce := range txmp.byAddrNonce.Lock() { - if old, ok := byAddrNonce[an]; ok && old.priority >= wtx.priority { - return - } - byAddrNonce[an] = wtx - } -} - -func (txmp *TxMempool) removeNonce(wtx *WrappedTx) { - evm, ok := wtx.evm.Get() - if !ok { - return - } - an := evmAddrNonce{evm.address, evm.nonce} - for byAddrNonce := range txmp.byAddrNonce.Lock() { - if byAddrNonce[an] == wtx { - delete(byAddrNonce, an) - } - } -} - -func (txmp *TxMempool) TxStore() *TxStore { return txmp.txStore } +func (txmp *TxMempool) TxStore() *txStoreV2 { return txmp.txStore } // Lock obtains a write-lock on the mempool. A caller must be sure to explicitly // release the lock when finished. @@ -336,39 +267,35 @@ func (txmp *TxMempool) utilisation() float64 { return float64(txmp.NumTxsNotPending()) / float64(txmp.config.Size) } -func (txmp *TxMempool) NumTxsNotPending() int { - return txmp.txStore.Size() -} - -func (txmp *TxMempool) BytesNotPending() int64 { - return txmp.txStore.AllTxsBytes() -} - -func (txmp *TxMempool) TotalTxsBytesSize() int64 { - return txmp.BytesNotPending() + txmp.pendingTxs.SizeBytes() -} +func (txmp *TxMempool) NumTxsNotPending() int { return txmp.txStore.Size() } +func (txmp *TxMempool) BytesNotPending() uint64 { return txmp.txStore.AllTxsBytes() } +func (txmp *TxMempool) TotalTxsBytesSize() uint64 { return txmp.txStore.TotalBytes() } // PendingSize returns the number of pending transactions in the mempool. -func (txmp *TxMempool) PendingSize() int { return txmp.pendingTxs.Size() } -func (txmp *TxMempool) PendingSizeBytes() int64 { return txmp.pendingTxs.SizeBytes() } +func (txmp *TxMempool) PendingSize() int { return txmp.txStore.PendingSize() } +func (txmp *TxMempool) PendingSizeBytes() uint64 { return txmp.txStore.PendingBytes() } // SizeBytes return the total sum in bytes of all the valid transactions in the // mempool. It is thread-safe. -func (txmp *TxMempool) SizeBytes() int64 { return txmp.txStore.AllTxsBytes() } +func (txmp *TxMempool) SizeBytes() uint64 { return txmp.txStore.AllTxsBytes() } // WaitForNextTx waits until the next transaction is available for gossip. // Returns the next valid transaction to gossip. func (txmp *TxMempool) WaitForNextTx(ctx context.Context) (*clist.CElement[*WrappedTx], error) { - return txmp.gossipIndex.WaitFront(ctx) + return txmp.txStore.readyTxs.WaitFront(ctx) } // TxsAvailable returns a channel which fires once for every height, and only // when transactions are available in the mempool. It is thread-safe. -func (txmp *TxMempool) TxsAvailable() <-chan struct{} { - return txmp.txsAvailable +func (txmp *TxMempool) TxsAvailable() <-chan struct{} { return txmp.txsAvailable } + +func (txmp *TxMempool) removeTx(txHash types.TxHash) { + if txmp.txStore.Remove(txHash) { + txmp.metrics.RemovedTxs.Add(1) + } } -func (txmp *TxMempool) checkResponseState(wtx *WrappedTx) error { +func (txmp *TxMempool) checkTxConstraints(wtx *WrappedTx) error { constraints, err := txmp.txConstraintsFetcher() if err != nil { return err @@ -383,7 +310,6 @@ func (txmp *TxMempool) checkResponseState(wtx *WrappedTx) error { if wtx.gasWanted > constraints.MaxGas { return fmt.Errorf("gas wanted exceeds max gas: gas wanted %d is greater than max gas %d", wtx.gasWanted, constraints.MaxGas) } - return nil } @@ -412,15 +338,17 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) txmp.mtx.RLock() defer txmp.mtx.RUnlock() + // Early exit if tx is too large. if txSize := len(tx); txSize > txmp.config.MaxTxBytes { return nil, fmt.Errorf("%w: max size is %d, but got %d", ErrTxTooLarge, txmp.config.MaxTxBytes, txSize) } + hTx := newHashedTx(tx) constraints, err := txmp.txConstraintsFetcher() if err != nil { return nil, fmt.Errorf("txmp.txConstraintsFetcher(): %w", err) } - if txSize := types.ComputeProtoSizeForTxs([]types.Tx{tx}); txSize > constraints.MaxDataBytes { - return nil, fmt.Errorf("%w: tx size is too big: %d, max: %d", ErrTxTooLarge, txSize, constraints.MaxDataBytes) + if hTx.protoSize > constraints.MaxDataBytes { + return nil, fmt.Errorf("%w: tx size is too big: %d, max: %d", ErrTxTooLarge, hTx.protoSize, constraints.MaxDataBytes) } // Reject low priority transactions when the mempool is more than @@ -442,13 +370,12 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) return nil, errors.New("priority not high enough for mempool") } } - txHash := tx.Hash() // We add the transaction to the mempool's cache and if the // transaction is already present in the cache, i.e. false is returned, then we // check if we've seen this transaction and error if we have. - if !txmp.cache.Push(txHash) { - txmp.txStore.GetOrSetPeerByTxHash(txHash, txInfo.SenderID) + if !txmp.cache.Push(hTx.Hash()) { + txmp.txStore.GetOrSetPeerByTxHash(hTx.Hash(), txInfo.SenderID) return nil, ErrTxInCache } txmp.metrics.CacheSize.Set(float64(txmp.cache.Size())) @@ -456,7 +383,7 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) // Check TTL cache to see if we've recently processed this transaction // Only execute TTL cache logic if we're using a real TTL cache (not NOP) if c, ok := txmp.duplicateTxsCache.Get(); ok { - c.Increment(txHash) + c.Increment(hTx.Hash()) } if len(txInfo.SenderNodeID) == 0 { @@ -466,7 +393,7 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) if err != nil || !res.IsOK() { txmp.metrics.NumberOfFailedCheckTxs.Add(1) txmp.metrics.observeCheckTxPriorityDistribution(0, false, txInfo.SenderNodeID, true) - txmp.cache.Remove(txHash) + txmp.cache.Remove(hTx.Hash()) } if err != nil { return nil, err @@ -478,7 +405,7 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) txmp.metrics.observeCheckTxPriorityDistribution(res.Priority, false, txInfo.SenderNodeID, false) wtx := &WrappedTx{ - hashedTx: newHashedTx(tx), + hashedTx: hTx, timestamp: time.Now().UTC(), height: txmp.height, priority: res.Priority, @@ -494,38 +421,41 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) requiredBalance: res.EVMRequiredBalance, }) } - - // only add new transaction if checkTx passes and is not pending - if !txmp.isPending(wtx) { - if err := txmp.addNewTransaction(wtx); err != nil { - return nil, err - } - } else { - // otherwise add to pending txs store - if err := txmp.canAddPendingTx(wtx); err != nil { - // TODO: eviction strategy for pending transactions - txmp.cache.Remove(txHash) - return nil, err - } - if err := txmp.pendingTxs.Insert(wtx); err != nil { - txmp.cache.Remove(txHash) - return nil, err - } + // Update transaction priority reservoir with the true Tx priority + // as determined by the application. + // + // NOTE: This is done before potentially rejecting the transaction due to + // mempool being full. This is to ensure that the reservoir contains a + // representative sample of all transactions that have been processed by + // CheckTx. + // + // However, this is NOT done if the tx is pending, since a spammer could + // throw off the correct priority percentiles otherwise. + // + // We do not use the priority hint here as it may be misleading and + // inaccurate. The true priority as determined by the application is the + // most accurate. + txmp.priorityReservoir.Add(wtx.priority) + + if err := txmp.checkTxConstraints(wtx); err != nil { + // ignore bad transactions + logger.Info("rejected bad transaction", "priority", wtx.priority, "tx", wtx.Hash(), "post_check_err", err) + txmp.metrics.FailedTxs.Add(1) + return nil, err } - txmp.addNonce(wtx) + + txmp.txStore.Insert(wtx) + + txmp.metrics.InsertedTxs.Add(1) + txmp.metrics.TxSizeBytes.Add(float64(wtx.Size())) + txmp.metrics.Size.Set(float64(txmp.NumTxsNotPending())) + txmp.metrics.PendingSize.Set(float64(txmp.PendingSize())) + txmp.metrics.TotalTxsSizeBytes.Set(float64(txmp.TotalTxsBytesSize())) + + txmp.notifyTxsAvailable() return res.ResponseCheckTx, nil } -func (txmp *TxMempool) isInMempool(txHash types.TxHash) bool { - return !txmp.txStore.IsTxRemovedByHash(txHash) -} - -func (txmp *TxMempool) HasTx(txHash types.TxHash) bool { - txmp.mtx.RLock() - defer txmp.mtx.RUnlock() - return txmp.txStore.GetTxByHash(txHash) != nil -} - func (txmp *TxMempool) GetTxsForHashes(txHashes []types.TxHash) types.Txs { txmp.mtx.RLock() defer txmp.mtx.RUnlock() @@ -565,7 +495,7 @@ func (txmp *TxMempool) Flush() { txmp.mtx.Lock() defer txmp.mtx.Unlock() for _, wtx := range txmp.txStore.GetAllTxs() { - txmp.removeTx(wtx, false, false, true) + txmp.removeTx(wtx.Hash()) } txmp.cache.Reset() } @@ -622,27 +552,22 @@ func (txmp *TxMempool) reapTxs(l ReapLimits) (types.Txs, int64) { if maxGasEstimated < 0 { maxGasEstimated = utils.Max[int64]() } - var ( - totalGasWanted int64 - totalGasEstimated int64 - totalSize int64 - ) - + totalGasWanted := int64(0) + totalGasEstimated := int64(0) + totalSize := int64(0) numTxs := uint64(0) encounteredGasUnfit := false + if uint64(txmp.NumTxsNotPending()) < txmp.config.TxNotifyThreshold { //nolint:gosec // NumTxsNotPending returns non-negative value // do not reap anything if threshold is not met return []types.Tx{}, 0 } - totalTxs := txmp.priorityIndex.NumTxs() - evmTxs := make([]types.Tx, 0, totalTxs) - nonEvmTxs := make([]types.Tx, 0, totalTxs) - txmp.priorityIndex.ForEachTx(func(wtx *WrappedTx) bool { - size := types.ComputeProtoSizeForTxs([]types.Tx{wtx.Tx()}) - + var evmTxs []types.Tx + var nonEvmTxs []types.Tx + for wtx := range txmp.txStore.IterByPriority() { // bytes limit is a hard stop - if totalSize+size > maxBytes || numTxs+1 > maxTxs { - return false + if wtx.protoSize > maxBytes-totalSize || numTxs >= maxTxs { + break } // if the tx doesn't have a gas estimate, fallback to gas wanted @@ -654,27 +579,23 @@ func (txmp *TxMempool) reapTxs(l ReapLimits) (types.Txs, int64) { txGasEstimate = wtx.gasWanted } - // prospective totals - prospectiveGasWanted := totalGasWanted + wtx.gasWanted - prospectiveGasEstimated := totalGasEstimated + txGasEstimate - - maxGasWantedExceeded := prospectiveGasWanted > maxGasWanted - maxGasEstimatedExceeded := prospectiveGasEstimated > maxGasEstimated + limitExceeded := (maxGasWanted - totalGasWanted < wtx.gasWanted) || + (maxGasEstimated - totalGasEstimated < txGasEstimate) - if maxGasWantedExceeded || maxGasEstimatedExceeded { + if limitExceeded { // skip this unfit-by-gas tx once and attempt to pull up to 10 smaller ones if !encounteredGasUnfit && numTxs < MinTxsToPeek { encounteredGasUnfit = true - return true + continue } - return false + break } // include tx and update totals numTxs += 1 - totalSize += size - totalGasWanted = prospectiveGasWanted - totalGasEstimated = prospectiveGasEstimated + totalSize += wtx.protoSize + totalGasWanted += wtx.gasWanted + totalGasEstimated += txGasEstimate if wtx.evm.IsPresent() { evmTxs = append(evmTxs, wtx.Tx()) @@ -682,10 +603,9 @@ func (txmp *TxMempool) reapTxs(l ReapLimits) (types.Txs, int64) { nonEvmTxs = append(nonEvmTxs, wtx.Tx()) } if encounteredGasUnfit && numTxs >= MinTxsToPeek { - return false + break } - return true - }) + } return append(evmTxs, nonEvmTxs...), totalGasEstimated } @@ -696,38 +616,11 @@ func (txmp *TxMempool) PopTxs(l ReapLimits) (types.Txs, int64) { defer txmp.Unlock() txs, gasEstimated := txmp.reapTxs(l) for _, tx := range txs { - if wtx := txmp.txStore.GetTxByHash(tx.Hash()); wtx != nil { - txmp.removeTx(wtx, false, false, true) - } + txmp.removeTx(tx.Hash()) } return txs, gasEstimated } -// ReapMaxTxs returns a list of transactions within the provided number of -// transactions bound. Transaction are retrieved in priority order. -// -// NOTE: -// - Transactions returned are not removed from the mempool transaction -// store or indexes. -func (txmp *TxMempool) ReapMaxTxs(max int) types.Txs { - txmp.mtx.Lock() - defer txmp.mtx.Unlock() - - wTxs := txmp.priorityIndex.PeekTxs(max) - txs := make([]types.Tx, 0, len(wTxs)) - for _, wtx := range wTxs { - txs = append(txs, wtx.Tx()) - } - if len(txs) < max { - // retrieve more from pending txs - pending := txmp.pendingTxs.Peek(max - len(txs)) - for _, ptx := range pending { - txs = append(txs, ptx.Tx()) - } - } - return txs -} - // Update iterates over all the transactions provided by the block producer, // removes them from the cache (if applicable), and removes // the transactions from the main transaction store and associated indexes. @@ -796,6 +689,8 @@ func (txmp *TxMempool) Update( for i, tx := range blockTxs { txHash := tx.Hash() + // Remove transaction from the mempool, no matter if it succeeded, or not. + txmp.removeTx(txHash) if execTxResult[i].Code == abci.CodeTypeOK { // add the valid committed transaction to the cache (if missing) _ = txmp.cache.Push(txHash) @@ -807,248 +702,23 @@ func (txmp *TxMempool) Update( } // Subsequent failures: leave in cache to prevent infinite re-entry } - - // remove the committed transaction from the transaction store and indexes - if wtx := txmp.txStore.GetTxByHash(txHash); wtx != nil { - txmp.removeTx(wtx, false, false, true) - } - if execTxResult[i].EvmTxInfo != nil { - // remove any tx that has the same nonce (because the committed tx - // may be from block proposal and is never in the local mempool) - if wtx, _ := txmp.priorityIndex.TxByAddrNonce( - common.HexToAddress(execTxResult[i].EvmTxInfo.SenderAddress), - execTxResult[i].EvmTxInfo.Nonce, - ); wtx != nil { - txmp.removeTx(wtx, false, false, true) - } - } } - - txmp.purgeExpiredTxs(blockHeight) - txmp.handlePendingTransactions() + txmp.txStore.UpdateHeight(blockHeight) // If there any uncommitted transactions left in the mempool, we either // initiate re-CheckTx per remaining transaction or notify that remaining // transactions are left. - if txmp.Size() > 0 { - if recheck { - logger.Debug( - "executing re-CheckTx for all remaining transactions", - "num_txs", txmp.Size(), - "height", blockHeight, - ) - txmp.updateReCheckTxs(ctx) - } else { - txmp.notifyTxsAvailable() - } + if recheck { + txmp.updateReCheckTxs(ctx) } + txmp.notifyTxsAvailable() txmp.metrics.Size.Set(float64(txmp.NumTxsNotPending())) txmp.metrics.TotalTxsSizeBytes.Set(float64(txmp.TotalTxsBytesSize())) txmp.metrics.PendingSize.Set(float64(txmp.PendingSize())) return nil } -// addNewTransaction is invoked for a new unique transaction after CheckTx -// has been executed by the ABCI application for the first time on that transaction. -// CheckTx can be called again for the same transaction later when re-checking; -// however, this function will not be called. A recheck after a block is committed -// goes to handleRecheckResult. -// -// addNewTransaction runs after the ABCI application executes CheckTx. -// It runs the consensus-derived post-check for the current state snapshot. -// If the post-check reports an error, the transaction is rejected. Otherwise, -// we attempt to insert the transaction into the mempool. CheckTx response codes -// are filtered earlier in CheckTx. -// -// When inserting a transaction, we first check if there is sufficient capacity. -// If there is, the transaction is added to the txStore and all indexes. -// Otherwise, if the mempool is full, we attempt to find a lower priority transaction -// to evict in place of the new incoming transaction. If no such transaction exists, -// the new incoming transaction is rejected. -// -// NOTE: -// - An explicit lock is NOT required. -func (txmp *TxMempool) addNewTransaction(wtx *WrappedTx) error { - // Update transaction priority reservoir with the true Tx priority - // as determined by the application. - // - // NOTE: This is done before potentially rejecting the transaction due to - // mempool being full. This is to ensure that the reservoir contains a - // representative sample of all transactions that have been processed by - // CheckTx. - // - // However, this is NOT done if the tx is pending, since a spammer could - // throw off the correct priority percentiles otherwise. - // - // We do not use the priority hint here as it may be misleading and - // inaccurate. The true priority as determined by the application is the - // most accurate. - txmp.priorityReservoir.Add(wtx.priority) - err := txmp.checkResponseState(wtx) - if err != nil { - // ignore bad transactions - logger.Info( - "rejected bad transaction", - "priority", wtx.priority, - "tx", wtx.Hash(), - "post_check_err", err, - ) - txmp.metrics.FailedTxs.Add(1) - if !txmp.config.KeepInvalidTxsInCache { - txmp.cache.Remove(wtx.Hash()) - } - return err - } - if err := txmp.canAddTx(wtx); err != nil { - evictTxs := txmp.priorityIndex.GetEvictableTxs( - wtx.priority, - int64(wtx.Size()), - txmp.SizeBytes(), - txmp.config.MaxTxsBytes, - ) - if len(evictTxs) == 0 { - // No room for the new incoming transaction so we just remove it from - // the cache. - txmp.cache.Remove(wtx.Hash()) - logger.Error( - "rejected incoming good transaction; mempool full", - "tx", wtx.Hash(), - "err", err, - ) - txmp.metrics.RejectedTxs.Add(1) - return nil - } - - // evict an existing transaction(s) - // - // NOTE: - // - The transaction, toEvict, can be removed while a concurrent - // reCheckTx callback is being executed for the same transaction. - for _, toEvict := range evictTxs { - txmp.removeTx(toEvict, true, true, true) - logger.Debug( - "evicted existing good transaction; mempool full", - "old_tx", fmt.Sprintf("%X", toEvict.Hash()), - "old_priority", toEvict.priority, - "new_tx", wtx.Hash(), - "new_priority", wtx.priority, - ) - txmp.metrics.EvictedTxs.Add(1) - } - } - - if txmp.isInMempool(wtx.Hash()) { - return nil - } - - if txmp.insertTx(wtx) { - logger.Debug( - "inserted good transaction", - "priority", wtx.priority, - "tx", wtx.Hash(), - "height", txmp.height, - "num_txs", txmp.NumTxsNotPending(), - ) - txmp.notifyTxsAvailable() - } - - return nil -} - -// handleRecheckResult handles the responses from ABCI CheckTx calls issued -// during the recheck phase of a block Update. It removes any transactions -// invalidated by the application. -// -// The caller must hold a mempool write-lock (via Lock()) and when -// executing Update(), if the mempool is non-empty and Recheck is -// enabled, then all remaining transactions will be rechecked via -// CheckTx. The order transactions are rechecked must be the same as -// the order in which this callback is called. -// -// This method is NOT executed for the initial CheckTx on a new transaction; -// that case is handled by addNewTransaction instead. -func (txmp *TxMempool) handleRecheckResult(tx types.Tx, res *abci.ResponseCheckTxV2) { - if txmp.recheckCursor == nil { - return - } - - txmp.metrics.RecheckTimes.Add(1) - - wtx := txmp.recheckCursor.Value() - - // Search through the remaining list of tx to recheck for a transaction that matches - // the one we received from the ABCI application. - for !bytes.Equal(tx, wtx.Tx()) { - - logger.Debug( - "re-CheckTx transaction mismatch", - "got", wtx.Hash(), - "expected", tx.Hash(), - ) - - if txmp.recheckCursor == txmp.recheckEnd { - // we reached the end of the recheckTx list without finding a tx - // matching the one we received from the ABCI application. - // Return without processing any tx. - txmp.recheckCursor = nil - return - } - - txmp.recheckCursor = txmp.recheckCursor.Next() - wtx = txmp.recheckCursor.Value() - } - - // Only evaluate transactions that have not been removed. This can happen - // if an existing transaction is evicted during CheckTx and while this - // callback is being executed for the same evicted transaction. - if !txmp.txStore.IsTxRemoved(wtx) { - err := txmp.checkResponseState(wtx) - if evm, ok := wtx.evm.Get(); ok { - evm.requiredBalance = new(big.Int).Set(res.EVMRequiredBalance) - wtx.evm = utils.Some(evm) - } - - // we will treat a transaction that turns pending in a recheck as invalid and evict it - if res.Code == abci.CodeTypeOK && err == nil && !txmp.isPending(wtx) { - wtx.priority = res.Priority - } else { - logger.Debug( - "existing transaction no longer valid; failed re-CheckTx callback", - "priority", wtx.priority, - "tx", wtx.Hash(), - "err", err, - "code", res.Code, - ) - - if wtx.gossipEl != txmp.recheckCursor { - panic("corrupted reCheckTx cursor") - } - - txmp.removeTx(wtx, !txmp.config.KeepInvalidTxsInCache, true, true) - } - } - - // move reCheckTx cursor to next element - if txmp.recheckCursor == txmp.recheckEnd { - txmp.recheckCursor = nil - } else { - txmp.recheckCursor = txmp.recheckCursor.Next() - } - - if txmp.recheckCursor == nil { - logger.Debug("finished rechecking transactions") - - if txmp.NumTxsNotPending() > 0 { - txmp.notifyTxsAvailable() - } - } - - txmp.metrics.Size.Set(float64(txmp.NumTxsNotPending())) - txmp.metrics.PendingSize.Set(float64(txmp.PendingSize())) - txmp.metrics.TotalTxsSizeBytes.Set(float64(txmp.TotalTxsBytesSize())) -} - // updateReCheckTxs updates the recheck cursors using the gossipIndex. For // each transaction, it executes CheckTx. The global callback defined on // the app will be executed for each transaction after CheckTx is @@ -1057,211 +727,46 @@ func (txmp *TxMempool) handleRecheckResult(tx types.Tx, res *abci.ResponseCheckT // NOTE: // - The caller must have a write-lock when executing updateReCheckTxs. func (txmp *TxMempool) updateReCheckTxs(ctx context.Context) { - if txmp.Size() == 0 { - panic("attempted to update re-CheckTx txs when mempool is empty") - } logger.Debug( "executing re-CheckTx for all remaining transactions", "num_txs", txmp.Size(), "height", txmp.height, ) - txmp.recheckCursor = txmp.gossipIndex.Front() - txmp.recheckEnd = txmp.gossipIndex.Back() - - for e := txmp.gossipIndex.Front(); e != nil; e = e.Next() { + for e := txmp.txStore.readyTxs.Front(); e != nil; e = e.Next() { wtx := e.Value() - - // Only execute CheckTx if the transaction is not marked as removed which - // could happen if the transaction was evicted. - if !txmp.txStore.IsTxRemoved(wtx) { - res, err := txmp.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{ - Tx: wtx.Tx(), - Type: abci.CheckTxTypeV2Recheck, - }) - if err == nil { - err = res.Err() - } - if err != nil { - // no need in retrying since the tx will be rechecked after the next block - - logger.Debug("failed to execute CheckTx during recheck", "err", err, "hash", wtx.Hash()) - continue - } - txmp.handleRecheckResult(wtx.Tx(), res) - } - } -} - -// canAddTx returns an error if we cannot insert the provided *WrappedTx into -// the mempool due to mempool configured constraints. If it returns nil, -// the transaction can be inserted into the mempool. -func (txmp *TxMempool) canAddTx(wtx *WrappedTx) error { - var ( - numTxs = txmp.NumTxsNotPending() - sizeBytes = txmp.SizeBytes() - ) - - if numTxs >= txmp.config.Size || int64(wtx.Size())+sizeBytes > txmp.config.MaxTxsBytes { - return fmt.Errorf("mempool is full: number of txs %d (max: %d), total txs bytes %d (max: %d)", - numTxs, - txmp.config.Size, - sizeBytes, - txmp.config.MaxTxsBytes, - ) - } - - return nil -} - -func (txmp *TxMempool) canAddPendingTx(wtx *WrappedTx) error { - var ( - numTxs = txmp.PendingSize() - sizeBytes = txmp.PendingSizeBytes() - ) - - if numTxs >= txmp.config.PendingSize || int64(wtx.Size())+sizeBytes > txmp.config.MaxPendingTxsBytes { - return fmt.Errorf("mempool pending set is full: number of txs %d (max: %d), total txs bytes %d (max: %d)", - numTxs, - txmp.config.PendingSize, - sizeBytes, - txmp.config.MaxPendingTxsBytes, - ) - } - - return nil -} - -func (txmp *TxMempool) insertTx(wtx *WrappedTx) bool { - replacedTx, inserted := txmp.priorityIndex.PushTx(wtx) - if !inserted { - return false - } - txmp.metrics.TxSizeBytes.Add(float64(wtx.Size())) - txmp.metrics.Size.Set(float64(txmp.NumTxsNotPending())) - txmp.metrics.PendingSize.Set(float64(txmp.PendingSize())) - txmp.metrics.TotalTxsSizeBytes.Set(float64(txmp.TotalTxsBytesSize())) - - if replacedTx != nil { - txmp.removeTx(replacedTx, true, false, false) - } - - txmp.txStore.SetTx(wtx) - - // Insert the transaction into the gossip index and mark the reference to the - // linked-list element, which will be needed at a later point when the - // transaction is removed. - gossipEl := txmp.gossipIndex.PushBack(wtx) - wtx.gossipEl = gossipEl - - txmp.metrics.InsertedTxs.Add(1) - return true -} - -func (txmp *TxMempool) removeTx(wtx *WrappedTx, removeFromCache bool, shouldReenqueue bool, updatePriorityIndex bool) { - if txmp.txStore.IsTxRemoved(wtx) { - return - } - - txmp.removeNonce(wtx) - txmp.txStore.RemoveTx(wtx) - toBeReenqueued := []*WrappedTx{} - if updatePriorityIndex { - toBeReenqueued = txmp.priorityIndex.RemoveTx(wtx, shouldReenqueue) - } - - // Remove the transaction from the gossip index and cleanup the linked-list - // element so it can be garbage collected. - txmp.gossipIndex.Remove(wtx.gossipEl) - wtx.gossipEl.DetachPrev() - - txmp.metrics.RemovedTxs.Add(1) - if removeFromCache { - txmp.cache.Remove(wtx.Hash()) - } - - if shouldReenqueue { - for _, reenqueue := range toBeReenqueued { - txmp.removeTx(reenqueue, removeFromCache, false, true) + res, err := txmp.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{ + Tx: wtx.Tx(), + Type: abci.CheckTxTypeV2Recheck, + }) + if err == nil { + err = res.Err() } - for _, reenqueue := range toBeReenqueued { - rtx := reenqueue.Tx() - go func() { - if _, err := txmp.CheckTx(context.Background(), rtx, TxInfo{}); err != nil { - logger.Error("failed to reenqueue transaction", "tx-hash", rtx.Hash(), "err", err) - } - }() + if err != nil { + // no need in retrying since the tx will be rechecked after the next block + logger.Debug("failed to execute CheckTx during recheck", "err", err, "hash", wtx.Hash()) + continue } - } -} - -func (txmp *TxMempool) expire(blockHeight int64, wtx *WrappedTx) { - txmp.metrics.ExpiredTxs.Add(1) - txmp.logExpiredTx(blockHeight, wtx) - if !txmp.config.KeepInvalidTxsInCache { - txmp.cache.Remove(wtx.Hash()) - } -} - -func (txmp *TxMempool) logExpiredTx(blockHeight int64, wtx *WrappedTx) { - // defensive check - if wtx == nil { - return - } - - logger.Info( - "transaction expired", - "priority", wtx.priority, - "tx", wtx.Hash(), - "address", func() string { - evm, ok := wtx.evm.Get() - if !ok { - return "" - } - return evm.address.Hex() - }(), - "evm", wtx.evm.IsPresent(), - "nonce", wtx.EVMNonce(), - "height", blockHeight, - "tx_height", wtx.height, - "tx_timestamp", wtx.timestamp, - "age", time.Since(wtx.timestamp), - ) -} - -// purgeExpiredTxs removes all transactions that have exceeded their respective -// height- and/or time-based TTLs from their respective indexes. Every expired -// transaction will be removed from the mempool, but preserved in the cache (except for pending txs). -// -// NOTE: purgeExpiredTxs must only be called during TxMempool#Update in which -// the caller has a write-lock on the mempool and so we can safely iterate over -// the height and time based indexes. -func (txmp *TxMempool) purgeExpiredTxs(blockHeight int64) { - now := time.Now() - - minHeight := utils.None[int64]() - if n := txmp.config.TTLNumBlocks; n > 0 && blockHeight > n { - minHeight = utils.Some(blockHeight - n) - } - minTime := utils.None[time.Time]() - if d := txmp.config.TTLDuration; d > 0 { - minTime = utils.Some(time.Now().Add(-d)) - } - expiredTxs := txmp.txStore.GetOlderThan(minTime, minHeight) + txmp.metrics.RecheckTimes.Add(1) - for _, wtx := range expiredTxs { - if txmp.config.RemoveExpiredTxsFromQueue { - txmp.removeTx(wtx, !txmp.config.KeepInvalidTxsInCache, false, true) - } else { - txmp.expire(blockHeight, wtx) + // we will treat a transaction that turns pending in a recheck as invalid and evict it + if err := txmp.checkTxConstraints(wtx); err != nil || res.Code != abci.CodeTypeOK { + logger.Debug( + "existing transaction no longer valid; failed re-CheckTx callback", + "priority", wtx.priority, + "tx", wtx.Hash(), + "err", err, + "code", res.Code, + ) + txmp.removeTx(wtx.Hash()) } - } - // remove pending txs that have expired - txmp.pendingTxs.PurgeExpired(blockHeight, now, func(wtx *WrappedTx) { - txmp.removeNonce(wtx) - txmp.expire(blockHeight, wtx) - }) + wtx.priority = res.Priority + if evm, ok := wtx.evm.Get(); ok { + evm.requiredBalance = new(big.Int).Set(res.EVMRequiredBalance) + wtx.evm = utils.Some(evm) + } + } } func (txmp *TxMempool) notifyTxsAvailable() { @@ -1275,46 +780,6 @@ func (txmp *TxMempool) notifyTxsAvailable() { } } -func (txmp *TxMempool) isPending(wtx *WrappedTx) bool { - evm, ok := wtx.evm.Get() - if !ok { - return false - } - if evm.nonce > txmp.EvmNextPendingNonce(evm.address) { - return true - } - balance := txmp.app.EvmBalance(evm.address, evm.seiAddress) - return balance.Cmp(evm.requiredBalance) < 0 -} - -func (txmp *TxMempool) handlePendingTransactions() { - accepted, rejected := txmp.pendingTxs.EvaluatePendingTransactions(func(wtx *WrappedTx) abci.PendingTxCheckerResponse { - evm, ok := wtx.evm.Get() - if !ok { - return abci.Accepted - } - if evm.nonce < txmp.app.EvmNonce(evm.address) { - return abci.Rejected - } - if txmp.isPending(wtx) { - return abci.Pending - } - return abci.Accepted - }) - for _, tx := range accepted { - if err := txmp.addNewTransaction(tx); err != nil { - txmp.removeNonce(tx) - logger.Error("error adding pending transaction", "err", err) - } - } - for _, tx := range rejected { - txmp.removeNonce(tx) - if !txmp.config.KeepInvalidTxsInCache { - txmp.cache.Remove(tx.Hash()) - } - } -} - // Run executes mempool background tasks. func (txmp *TxMempool) Run(ctx context.Context) error { c, ok := txmp.duplicateTxsCache.Get() diff --git a/sei-tendermint/internal/mempool/priority_queue.go b/sei-tendermint/internal/mempool/priority_queue.go index 48913afa30..1e2a721ac3 100644 --- a/sei-tendermint/internal/mempool/priority_queue.go +++ b/sei-tendermint/internal/mempool/priority_queue.go @@ -62,33 +62,6 @@ func (pq *TxPriorityQueue) txByAddrNonceUnsafe(addr common.Address, nonce uint64 return nil, -1 } -func (pq *TxPriorityQueue) tryReplacementUnsafe(tx *WrappedTx) (replaced *WrappedTx, shouldDrop bool) { - evm, ok := tx.evm.Get() - if !ok { - return nil, false - } - queue := pq.evmQueue[evm.address] - if len(queue) == 0 { - return nil, false - } - existing, idx := pq.txByAddrNonceUnsafe(evm.address, evm.nonce) - if existing == nil { - return nil, false - } - if tx.priority <= existing.priority { - // tx should be dropped since it's dominated by an existing tx - return nil, true - } - // should replace - // replace heap if applicable - if hi, ok := pq.findTxIndexUnsafe(existing); ok { - heap.Remove(pq, hi) - heap.Push(pq, tx) // need to be in the heap since it has the same nonce - } - pq.evmQueue[evm.address][idx] = tx // replace queue item in-place - return existing, false -} - // GetEvictableTxs attempts to find and return a list of *WrappedTx than can be // evicted to make room for another *WrappedTx with higher priority. If no such // list of *WrappedTx exists, nil will be returned. The returned list of *WrappedTx @@ -240,28 +213,6 @@ func (pq *TxPriorityQueue) pushTxUnsafe(tx *WrappedTx) { pq.evmQueue[evm.address] = insertToEVMQueue(queue, tx, idx) } -// PushTx adds a valid transaction to the priority queue. It is thread safe. -func (pq *TxPriorityQueue) PushTx(tx *WrappedTx) (*WrappedTx, bool) { - pq.mtx.Lock() - defer pq.mtx.Unlock() - - replacedTx, shouldDrop := pq.tryReplacementUnsafe(tx) - - // tx was not inserted, and nothing was replaced - if shouldDrop { - return nil, false - } - - // tx replaced an existing transaction - if replacedTx != nil { - return replacedTx, true - } - - // tx was not inserted yet, so insert it - pq.pushTxUnsafe(tx) - return nil, true -} - func (pq *TxPriorityQueue) popTxUnsafe() *WrappedTx { if len(pq.txs) == 0 { return nil diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index e057e93e76..e07eb7de32 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -2,13 +2,14 @@ package mempool import ( "context" - "errors" + "slices" + "maps" "math/big" - "sync/atomic" "time" + "cmp" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/ethereum/go-ethereum/common" - abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/libs/clist" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/types" @@ -29,15 +30,18 @@ type TxInfo struct { type hashedTx struct { tx types.Tx hash types.TxHash + protoSize int64 } func newHashedTx(tx types.Tx) hashedTx { - return hashedTx{tx: tx, hash: tx.Hash()} + return hashedTx{tx: tx, hash: tx.Hash(), + protoSize: types.ComputeProtoSizeForTxs([]types.Tx{tx}), + } } func (ktx *hashedTx) Tx() types.Tx { return ktx.tx } func (ktx *hashedTx) Hash() types.TxHash { return ktx.hash } -func (ktx *WrappedTx) Size() int { return len(ktx.tx) } +func (ktx *hashedTx) Size() int { return len(ktx.tx) } // WrappedTx defines a wrapper around a raw transaction with additional metadata // that is used for indexing. @@ -66,17 +70,8 @@ type WrappedTx struct { // peers records a mapping of all peers that sent a given transaction peers map[uint16]struct{} - // heapIndex defines the index of the item in the heap - heapIndex int - // gossipEl references the linked-list element in the gossip index - gossipEl *clist.CElement[*WrappedTx] - - // removed marks the transaction as removed from the mempool. This is set - // during RemoveTx and is needed due to the fact that a given existing - // transaction in the mempool can be evicted when it is simultaneously having - // a reCheckTx callback executed. - removed bool + readyEl utils.Option[*clist.CElement[*WrappedTx]] // evm properties that aid in prioritization evm utils.Option[evmTx] @@ -99,157 +94,200 @@ func (wtx *WrappedTx) EVMNonce() uint64 { return 0 } -type txStoreInner struct { - byHash map[types.TxHash]*WrappedTx // primary index - sizeBytes utils.AtomicSend[int64] +type evmAccount struct { + balance *big.Int + firstNonce uint64 + nextNonce uint64 +} + +type txStoreState struct { + readyCount int + readyBytes uint64 + pendingCount int + pendingBytes uint64 +} + +type txStoreV2Inner struct { + byHash map[types.TxHash]*WrappedTx + byNonce map[evmAddrNonce]*WrappedTx + accounts map[common.Address]*evmAccount + + state utils.AtomicSend[txStoreState] } -// TxStore implements a thread-safe mapping of valid transaction(s). -// -// NOTE: -// - Concurrent read-only access to a *WrappedTx object is OK. However, mutative -// access is not allowed. Regardless, it is not expected for the mempool to -// need mutative access. -type TxStore struct { - inner utils.RWMutex[*txStoreInner] - sizeBytes utils.AtomicRecv[int64] +type txStoreV2 struct { + config *Config + proxy *proxy.Proxy + inner utils.RWMutex[*txStoreV2Inner] + state utils.AtomicRecv[txStoreState] + // gossipIndex defines the gossiping index of valid transactions via a + // thread-safe linked-list. We also use the gossip index as a cursor for + // rechecking transactions already in the mempool. + readyTxs *clist.CList[*WrappedTx] } -func NewTxStore() *TxStore { - inner := &txStoreInner{ - byHash: make(map[types.TxHash]*WrappedTx), - sizeBytes: utils.NewAtomicSend[int64](0), +func NewTxStore() *txStoreV2 { + inner := &txStoreV2Inner{ + byHash: map[types.TxHash]*WrappedTx{}, + accounts: map[common.Address]*evmAccount{}, + state: utils.NewAtomicSend(txStoreState{}), } - return &TxStore{ - inner: utils.NewRWMutex(inner), - sizeBytes: inner.sizeBytes.Subscribe(), + return &txStoreV2{ + inner: utils.NewRWMutex(inner), + readyTxs: clist.New[*WrappedTx](), + state: inner.state.Subscribe(), } } // Size returns the total number of transactions in the store. -func (txs *TxStore) Size() int { - for inner := range txs.inner.RLock() { - return len(inner.byHash) - } - panic("unreachable") -} +func (txs *txStoreV2) Size() int { return txs.state.Load().readyCount } // AllTxsBytes returns the total size in bytes of all transactions in the store. -func (txs *TxStore) AllTxsBytes() int64 { - return txs.sizeBytes.Load() +func (txs *txStoreV2) AllTxsBytes() uint64 { return txs.state.Load().readyBytes } +func (txs *txStoreV2) TotalBytes() uint64 { + state := txs.state.Load() + return state.pendingBytes + state.readyBytes } // WaitForTxs waits until the store becomes non-empty. -func (txs *TxStore) WaitForTxs(ctx context.Context) error { - _, err := txs.sizeBytes.Wait(ctx, func(sizeBytes int64) bool { return sizeBytes > 0 }) +func (txs *txStoreV2) WaitForTxs(ctx context.Context) error { + _, err := txs.state.Wait(ctx, func(s txStoreState) bool { return s.readyCount > 0 }) return err } // GetAllTxs returns all the transactions currently in the store. -func (txs *TxStore) GetAllTxs() []*WrappedTx { +func (txs *txStoreV2) GetAllTxs() []*WrappedTx { for inner := range txs.inner.RLock() { - wTxs := make([]*WrappedTx, len(inner.byHash)) - i := 0 - for _, wtx := range inner.byHash { - wTxs[i] = wtx - i++ - } - return wTxs + return slices.Collect(maps.Values(inner.byHash)) } panic("unreachable") } -// GetOlderThan have older timestamp than minTime OR lower height than minHeight. -func (txs *TxStore) GetOlderThan(minTime utils.Option[time.Time], minHeight utils.Option[int64]) []*WrappedTx { - var older []*WrappedTx - for inner := range txs.inner.Lock() { - for _, wtx := range inner.byHash { - isOlder := func() bool { - if t, ok := minTime.Get(); ok && wtx.timestamp.Before(t) { - return true - } - if h, ok := minHeight.Get(); ok && wtx.height < h { - return true - } - return false - }() - if isOlder { - older = append(older, wtx) - } - } - } - return older -} - // GetTxByHash returns a *WrappedTx by the transaction's hash. -func (txs *TxStore) GetTxByHash(key types.TxHash) *WrappedTx { +func (txs *txStoreV2) GetTxByHash(key types.TxHash) *WrappedTx { for inner := range txs.inner.RLock() { return inner.byHash[key] } panic("unreachable") } -func (txs *TxStore) IsTxRemovedByHash(txHash types.TxHash) bool { - for inner := range txs.inner.RLock() { - wtx, ok := inner.byHash[txHash] - return !ok || wtx.removed +func (txs *txStoreV2) insert(inner *txStoreV2Inner, wtx *WrappedTx) { + if _,ok := inner.byHash[wtx.Hash()]; ok { return } + if evm,ok := wtx.evm.Get(); ok { + an := evmAddrNonce{evm.address,evm.nonce} + if old,ok := inner.byNonce[an]; ok { + if old.priority >= wtx.priority { return } + // TODO: replace logic + } + inner.byNonce[an] = wtx + account,ok := inner.accounts[evm.address] + if !ok { + b := txs.proxy.EvmBalance(evm.address,evm.seiAddress) + n := txs.proxy.EvmNonce(evm.address) + account = &evmAccount{b,n,n} + inner.accounts[evm.address] = account + } + for { + an.Nonce = account.nextNonce + if _,ok := inner.byNonce[an]; !ok { break } + account.nextNonce += 1 + } } - panic("unreachable") + inner.byHash[wtx.Hash()] = wtx + if !wtx.readyEl.IsPresent() { + wtx.readyEl = utils.Some(txs.readyTxs.PushBack(wtx)) + } + // TODO: update status } -// IsTxRemoved returns true if a transaction by hash is marked as removed and -// false otherwise. -func (txs *TxStore) IsTxRemoved(wtx *WrappedTx) bool { - for inner := range txs.inner.RLock() { - // if this instance has already been marked, return true - if wtx.removed { - return true - } - // otherwise if the same hash exists, return its state - wtx, ok := inner.byHash[wtx.Hash()] - if ok { - return wtx.removed +func (txs *txStoreV2) compact(inner *txStoreV2Inner) { + // split into ready and not-ready txs + var notReady []*WrappedTx + var ready []*WrappedTx + for _,wtx := range inner.byHash { + // TODO: apply balance and monotone priority checks + // earlier nonce has too high requiredBalance => not-ready + // earlier nonce has low prio => prio - our prio is capped + // order by (inc prio, dec nonce) + if evm,ok := wtx.evm.Get(); ok && evm.nonce >= inner.accounts[evm.address].nextNonce { + notReady = append(notReady,wtx) + } else { + ready = append(ready,wtx) } } - // otherwise we haven't seen this tx - return false + cmpPrio := func(a,b *WrappedTx) int { return cmp.Compare(a.priority,b.priority) } + // remove not-ready by priority + slices.SortFunc(notReady, cmpPrio) + for _,wtx := range notReady { + if !lowLimitExceeded {} + delete(inner.byHash,wtx.Hash()) + } + // remove ready by priority + slices.SortFunc(notReady, cmpPrio) + for _,wtx := range ready { + if !lowLimitExceeded {} + delete(inner.byHash,wtx.Hash()) + } + txs.recompute(inner) +} + +func (txs *txStoreV2) ReapTxs(l ReapLimits, remove bool) (types.Txs, int64) { + // find ready and sort like in compact() + // reap until limits + // if remove { removeTxs(); recompute() } } // SetTx stores a *WrappedTx by its hash. -func (txs *TxStore) SetTx(wtx *WrappedTx) { +func (txs *txStoreV2) Insert(wtx *WrappedTx) { for inner := range txs.inner.Lock() { - existing := inner.byHash[wtx.Hash()] - inner.byHash[wtx.Hash()] = wtx - if existing == nil { - inner.sizeBytes.Store(inner.sizeBytes.Load() + int64(wtx.Size())) + txs.insert(inner,wtx) + state := inner.state.Load() + state.readyCount += 1 + state.readyBytes += uint64(wtx.Size()) + inner.state.Store(state) + if highlimitExceeded { + txs.compact(inner) } } } +func (txs *txStoreV2) recompute(inner *txStoreV2Inner) { + byHash := inner.byHash + inner.byHash = map[types.TxHash]*WrappedTx{} + inner.byNonce = map[evmAddrNonce]*WrappedTx{} + for _, account := range inner.accounts { + account.nextNonce = account.firstNonce + } + // TODO: reset status + for _,wtx := range byHash { + txs.insert(inner,wtx) + } +} + // RemoveTx removes a *WrappedTx from the transaction store. It deletes all // indexes of the transaction. -func (txs *TxStore) RemoveTx(wtx *WrappedTx) { - for inner := range txs.inner.Lock() { - if _, ok := inner.byHash[wtx.Hash()]; ok { - delete(inner.byHash, wtx.Hash()) - inner.sizeBytes.Store(inner.sizeBytes.Load() - int64(wtx.Size())) +func (txs *txStoreV2) removeTxs(inner *txStoreV2Inner, txHashes []types.TxHash) { + for _,txHash := range txHashes { + wtx, ok := inner.byHash[txHash] + if !ok { continue } + // TODO: update status + delete(inner.byHash,txHash) + if el,ok := wtx.readyEl.Get(); ok { + txs.readyTxs.Remove(el) } - wtx.removed = true } } // TxHasPeer returns true if a transaction by hash has a given peer ID and false // otherwise. If the transaction does not exist, false is returned. -func (txs *TxStore) TxHasPeer(key types.TxHash, peerID uint16) bool { +func (txs *txStoreV2) TxHasPeer(txHash types.TxHash, peerID uint16) bool { for inner := range txs.inner.RLock() { - wtx := inner.byHash[key] - if wtx == nil { - return false + if wtx,ok := inner.byHash[txHash]; ok { + _, ok := wtx.peers[peerID] + return ok } - _, ok := wtx.peers[peerID] - return ok } - panic("unreachable") + return false } // GetOrSetPeerByTxHash looks up a WrappedTx by transaction hash and adds the @@ -257,162 +295,52 @@ func (txs *TxStore) TxHasPeer(key types.TxHash, peerID uint16) bool { // We return true if we've already recorded the given peer for this transaction // and false otherwise. If the transaction does not exist by hash, we return // (nil, false). -func (txs *TxStore) GetOrSetPeerByTxHash(hash types.TxHash, peerID uint16) (*WrappedTx, bool) { +func (txs *txStoreV2) GetOrSetPeerByTxHash(hash types.TxHash, peerID uint16) (*WrappedTx, bool) { for inner := range txs.inner.Lock() { - wtx := inner.byHash[hash] - if wtx == nil { - return nil, false - } - - if wtx.peers == nil { - wtx.peers = make(map[uint16]struct{}) - } - - if _, ok := wtx.peers[peerID]; ok { - return wtx, true - } - - wtx.peers[peerID] = struct{}{} - return wtx, false - } - panic("unreachable") -} - -type PendingTxs struct { - inner utils.RWMutex[*pendingTxsInner] - config *Config - sizeBytes atomic.Int64 -} - -type pendingTxsInner struct { - txs []*WrappedTx -} - -func NewPendingTxs(conf *Config) *PendingTxs { - return &PendingTxs{ - inner: utils.NewRWMutex(&pendingTxsInner{}), - config: conf, - } -} - -func (p *PendingTxs) EvaluatePendingTransactions( - evaluate func(*WrappedTx) abci.PendingTxCheckerResponse, -) ( - acceptedTxs []*WrappedTx, - rejectedTxs []*WrappedTx, -) { - poppedIndices := []int{} - for inner := range p.inner.Lock() { - for i := 0; i < len(inner.txs); i++ { - result := evaluate(inner.txs[i]) - switch result { - case abci.Accepted: - acceptedTxs = append(acceptedTxs, inner.txs[i]) - poppedIndices = append(poppedIndices, i) - case abci.Rejected: - rejectedTxs = append(rejectedTxs, inner.txs[i]) - poppedIndices = append(poppedIndices, i) + if wtx,ok := inner.byHash[hash]; ok { + if _, ok := wtx.peers[peerID]; ok { + return wtx, true } + wtx.peers[peerID] = struct{}{} + return wtx, false } - p.popTxsAtIndices(inner, poppedIndices) - return } - panic("unreachable") + return nil, false } -// Assumes the pending tx store is already write-locked. -func (p *PendingTxs) popTxsAtIndices(inner *pendingTxsInner, indices []int) { - if len(indices) == 0 { - return +func (txs *txStoreV2) UpdateHeight(now time.Time, blockHeight int64, blockTxs []types.TxHash) { + minHeight := utils.None[int64]() + if n := txs.config.TTLNumBlocks; n > 0 && blockHeight > n { + minHeight = utils.Some(blockHeight - n) } - newTxs := make([]*WrappedTx, 0, max(0, len(inner.txs)-len(indices))) - start := 0 - for _, idx := range indices { - if idx <= start-1 { - panic("indices popped from pending tx store should be sorted without duplicate") - } - if idx >= len(inner.txs) { - panic("indices popped from pending tx store out of range") - } - p.sizeBytes.Add(int64(-inner.txs[idx].Size())) - newTxs = append(newTxs, inner.txs[start:idx]...) - start = idx + 1 + minTime := utils.None[time.Time]() + if d := txs.config.TTLDuration; d > 0 { + minTime = utils.Some(now.Add(-d)) } - newTxs = append(newTxs, inner.txs[start:]...) - inner.txs = newTxs -} - -func (p *PendingTxs) Insert(tx *WrappedTx) error { - for inner := range p.inner.Lock() { - if len(inner.txs) >= p.config.PendingSize || int64(tx.Size())+p.sizeBytes.Load() > p.config.MaxPendingTxsBytes { - return errors.New("pending store is full") - } - inner.txs = append(inner.txs, tx) - p.sizeBytes.Add(int64(tx.Size())) - return nil - } - panic("unreachable") -} - -func (p *PendingTxs) SizeBytes() int64 { return p.sizeBytes.Load() } - -func (p *PendingTxs) Peek(max int) []*WrappedTx { - for inner := range p.inner.RLock() { - // priority is fifo - if max > len(inner.txs) { - return inner.txs - } - return inner.txs[:max] - } - panic("unreachable") -} - -func (p *PendingTxs) Size() int { - for inner := range p.inner.RLock() { - return len(inner.txs) - } - panic("unreachable") -} - -func (p *PendingTxs) PurgeExpired(blockHeight int64, now time.Time, cb func(wtx *WrappedTx)) { - for inner := range p.inner.Lock() { - if len(inner.txs) == 0 { - return - } - - // txs retains the ordering of insertion - if p.config.TTLNumBlocks > 0 { - idxFirstNotExpiredTx := len(inner.txs) - for i, ptx := range inner.txs { - // once found, we can break because these are ordered - if (blockHeight - ptx.height) <= p.config.TTLNumBlocks { - idxFirstNotExpiredTx = i - break + for inner := range txs.inner.Lock() { + // All account states need to be reevaluated. + inner.accounts = map[common.Address]*evmAccount{} + // Sequenced txs are pruned. + txs.removeTxs(inner, blockTxs) + // Old txs are pruned. + for _, wtx := range inner.byHash { + isOlder := func() bool { + if t, ok := minTime.Get(); ok && wtx.timestamp.Before(t) { + return true } - cb(ptx) - p.sizeBytes.Add(int64(-ptx.Size())) - } - inner.txs = inner.txs[idxFirstNotExpiredTx:] - } - - if len(inner.txs) == 0 { - return - } - - if p.config.TTLDuration > 0 { - idxFirstNotExpiredTx := len(inner.txs) - for i, ptx := range inner.txs { - // once found, we can break because these are ordered - if now.Sub(ptx.timestamp) <= p.config.TTLDuration { - idxFirstNotExpiredTx = i - break + if h, ok := minHeight.Get(); ok && wtx.height < h { + return true } - cb(ptx) - p.sizeBytes.Add(int64(-ptx.Size())) + return false + }() + if isOlder && (pending || txs.config.RemoveExpiredTxsFromQueue) { + // TODO: remove } - inner.txs = inner.txs[idxFirstNotExpiredTx:] } - return + // if recheck { ... } + txs.recompute(inner) } - panic("unreachable") } + +func (txs *txStoreV2) PendingBytes() uint64 { return txs.state.Load().pendingBytes } +func (txs *txStoreV2) PendingSize() int { return txs.state.Load().pendingCount } From e4a1280b60b0d117a5b390554a64da85998cdb97 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 15 May 2026 21:00:22 +0200 Subject: [PATCH 002/100] WIP --- sei-tendermint/internal/mempool/cache.go | 13 +- sei-tendermint/internal/mempool/mempool.go | 75 +++----- sei-tendermint/internal/mempool/tx.go | 200 ++++++++++----------- 3 files changed, 119 insertions(+), 169 deletions(-) diff --git a/sei-tendermint/internal/mempool/cache.go b/sei-tendermint/internal/mempool/cache.go index 43ef786454..9555283cdc 100644 --- a/sei-tendermint/internal/mempool/cache.go +++ b/sei-tendermint/internal/mempool/cache.go @@ -74,6 +74,9 @@ func (c *LRUTxCache) Reset() { } func (c *LRUTxCache) Push(txHash types.TxHash) bool { + if c.size <= 0 { + return true + } c.mtx.Lock() defer c.mtx.Unlock() @@ -122,16 +125,6 @@ func (c *LRUTxCache) toCacheKey(key types.TxHash) cacheKey { return cacheKey(trimToSize(key, c.maxKeyLen)) } -// NopTxCache defines a no-op raw transaction cache. -type NopTxCache struct{} - -var _ TxCache = (*NopTxCache)(nil) - -func (NopTxCache) Reset() {} -func (NopTxCache) Push(types.TxHash) bool { return true } -func (NopTxCache) Remove(types.TxHash) {} -func (NopTxCache) Size() int { return 0 } - // DuplicateTxCache implements TxCacheWithTTL using go-cache type DuplicateTxCache struct { maxSize int diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 1bf3563aad..46220558b0 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -70,21 +70,11 @@ type Config struct { // NOTE: the max size of a tx transmitted over the network is {max-tx-bytes}. MaxTxBytes int - // TTLDuration, if non-zero, defines the maximum amount of time a transaction - // can exist for in the mempool. - // - // Note, if TTLNumBlocks is also defined, a transaction will be removed if it - // has existed in the mempool at least TTLNumBlocks number of blocks or if it's - // insertion time into the mempool is beyond TTLDuration. - TTLDuration time.Duration - - // TTLNumBlocks, if non-zero, defines the maximum number of blocks a transaction - // can exist for in the mempool. - // - // Note, if TTLDuration is also defined, a transaction will be removed if it - // has existed in the mempool at least TTLNumBlocks number of blocks or if - // it's insertion time into the mempool is beyond TTLDuration. - TTLNumBlocks int64 + // time after which transaction is removed from mempool. + TTLDuration utils.Option[time.Duration] + + // number of blocks after which a transaction is removed from mempool. + TTLNumBlocks utils.Option[int64] // TxNotifyThreshold, if non-zero, defines the minimum number of transactions // needed to trigger a notification in mempool's Tx notifier @@ -148,8 +138,8 @@ func DefaultConfig() *Config { CacheSize: 10000, DuplicateTxsCacheSize: 100000, MaxTxBytes: 1024 * 1024, // 1MB - TTLDuration: 5 * time.Second, // prevent stale txs from filling mempool - TTLNumBlocks: 10, // remove txs after 10 blocks + TTLDuration: utils.Some(5 * time.Second), // prevent stale txs from filling mempool + TTLNumBlocks: utils.Some(int64(10)), // remove txs after 10 blocks TxNotifyThreshold: 0, PendingSize: 5000, MaxPendingTxsBytes: 1024 * 1024 * 1024, // 1GB @@ -183,12 +173,12 @@ type TxMempool struct { // cache defines a fixed-size cache of already seen transactions as this // reduces pressure on the proxyApp. - cache TxCache + cache *LRUTxCache // blockFailedTxs tracks tx hashes that have previously failed during // block execution. Used to prevent infinite re-entry of txs that // consistently fail before fee charging in DeliverTx. - blockFailedTxs TxCache + blockFailedTxs *LRUTxCache // A TTL cache which keeps all txs that we have seen before over the TTL window. // Currently, this can be used for tracking whether checkTx is always serving the same tx or not. @@ -196,7 +186,7 @@ type TxMempool struct { // txStore defines the main storage of valid transactions. Indexes are built // on top of this store. - txStore *txStoreV2 + txStore *txStore // A read/write lock is used to safe guard updates, insertions and deletions // from the mempool. A read-lock is implicitly acquired when executing CheckTx, @@ -208,29 +198,30 @@ type TxMempool struct { priorityReservoir *reservoir.Sampler[int64] } +func (txmp *TxMempool) SizeBytes() uint64 { return txmp.txStore.State().total.bytes } +func (txmp *TxMempool) NumTxsNotPending() int { return txmp.txStore.State().ready.count } +func (txmp *TxMempool) BytesNotPending() uint64 { return txmp.txStore.State().ready.bytes } +func (txmp *TxMempool) TotalTxsBytesSize() uint64 { return txmp.txStore.State().total.bytes } +func (txmp *TxMempool) PendingSize() int { return txmp.txStore.State().PendingCount() } +func (txmp *TxMempool) PendingSizeBytes() uint64 { return txmp.txStore.State().PendingBytes() } + func NewTxMempool( cfg *Config, app *proxy.Proxy, metrics *Metrics, txConstraintsFetcher TxConstraintsFetcher, ) *TxMempool { - txmp := &TxMempool{ config: cfg, app: app, txsAvailable: make(chan struct{}, 1), height: -1, - cache: NopTxCache{}, - blockFailedTxs: NopTxCache{}, metrics: metrics, txStore: NewTxStore(), txConstraintsFetcher: txConstraintsFetcher, priorityReservoir: reservoir.New[int64](cfg.DropPriorityReservoirSize, cfg.DropPriorityThreshold, nil), // Use non-deterministic RNG - } - - if cfg.CacheSize > 0 { - txmp.cache = NewLRUTxCache(cfg.CacheSize, maxCacheKeySize) - txmp.blockFailedTxs = NewLRUTxCache(cfg.CacheSize, maxCacheKeySize) + cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), + blockFailedTxs: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), } if cfg.DuplicateTxsCacheSize > 0 { @@ -241,14 +232,12 @@ func NewTxMempool( } func (txmp *TxMempool) Config() *Config { return txmp.config } - func (txmp *TxMempool) App() *proxy.Proxy { return txmp.app } - func (txmp *TxMempool) EvmNextPendingNonce(addr common.Address) uint64 { return txmp.txStore.NextNonce(addr) } -func (txmp *TxMempool) TxStore() *txStoreV2 { return txmp.txStore } +func (txmp *TxMempool) TxStore() *txStore { return txmp.txStore } // Lock obtains a write-lock on the mempool. A caller must be sure to explicitly // release the lock when finished. @@ -260,24 +249,13 @@ func (txmp *TxMempool) Unlock() { txmp.mtx.Unlock() } // Size returns the number of valid transactions in the mempool. It is // thread-safe. func (txmp *TxMempool) Size() int { - return txmp.NumTxsNotPending() + txmp.PendingSize() + return txmp.txStore.State().total.count } func (txmp *TxMempool) utilisation() float64 { return float64(txmp.NumTxsNotPending()) / float64(txmp.config.Size) } -func (txmp *TxMempool) NumTxsNotPending() int { return txmp.txStore.Size() } -func (txmp *TxMempool) BytesNotPending() uint64 { return txmp.txStore.AllTxsBytes() } -func (txmp *TxMempool) TotalTxsBytesSize() uint64 { return txmp.txStore.TotalBytes() } - -// PendingSize returns the number of pending transactions in the mempool. -func (txmp *TxMempool) PendingSize() int { return txmp.txStore.PendingSize() } -func (txmp *TxMempool) PendingSizeBytes() uint64 { return txmp.txStore.PendingBytes() } - -// SizeBytes return the total sum in bytes of all the valid transactions in the -// mempool. It is thread-safe. -func (txmp *TxMempool) SizeBytes() uint64 { return txmp.txStore.AllTxsBytes() } // WaitForNextTx waits until the next transaction is available for gossip. // Returns the next valid transaction to gossip. @@ -374,10 +352,7 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) // We add the transaction to the mempool's cache and if the // transaction is already present in the cache, i.e. false is returned, then we // check if we've seen this transaction and error if we have. - if !txmp.cache.Push(hTx.Hash()) { - txmp.txStore.GetOrSetPeerByTxHash(hTx.Hash(), txInfo.SenderID) - return nil, ErrTxInCache - } + if !txmp.cache.Push(hTx.Hash()) { return nil, ErrTxInCache } txmp.metrics.CacheSize.Set(float64(txmp.cache.Size())) // Check TTL cache to see if we've recently processed this transaction @@ -411,7 +386,6 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) priority: res.Priority, estimatedGas: res.GasEstimated, gasWanted: res.GasWanted, - peers: map[uint16]struct{}{txInfo.SenderID: {}}, } if res.IsEVM { wtx.evm = utils.Some(evmTx{ @@ -693,7 +667,7 @@ func (txmp *TxMempool) Update( txmp.removeTx(txHash) if execTxResult[i].Code == abci.CodeTypeOK { // add the valid committed transaction to the cache (if missing) - _ = txmp.cache.Push(txHash) + txmp.cache.Push(txHash) txmp.blockFailedTxs.Remove(txHash) } else if !txmp.config.KeepInvalidTxsInCache { if txmp.blockFailedTxs.Push(txHash) { @@ -727,6 +701,8 @@ func (txmp *TxMempool) Update( // NOTE: // - The caller must have a write-lock when executing updateReCheckTxs. func (txmp *TxMempool) updateReCheckTxs(ctx context.Context) { + // TODO(gprusak): this whole recheck thing is basically doing TxMempool.CheckTx for all remaining + // txs without restarting the gossip though. logger.Debug( "executing re-CheckTx for all remaining transactions", "num_txs", txmp.Size(), @@ -743,6 +719,7 @@ func (txmp *TxMempool) updateReCheckTxs(ctx context.Context) { err = res.Err() } if err != nil { + // TODO(gprusak): check if it would be safer to just remove the tx here, instead of waiting for retry. // no need in retrying since the tx will be rechecked after the next block logger.Debug("failed to execute CheckTx during recheck", "err", err, "hash", wtx.Hash()) continue diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index e07eb7de32..8098edc53d 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -41,40 +41,19 @@ func newHashedTx(tx types.Tx) hashedTx { func (ktx *hashedTx) Tx() types.Tx { return ktx.tx } func (ktx *hashedTx) Hash() types.TxHash { return ktx.hash } -func (ktx *hashedTx) Size() int { return len(ktx.tx) } +func (ktx *hashedTx) Size() uint64 { return uint64(len(ktx.tx)) } // WrappedTx defines a wrapper around a raw transaction with additional metadata // that is used for indexing. type WrappedTx struct { - // hashedTx represents the raw binary transaction data and its memoized hash. hashedTx - - // height defines the height at which the transaction was validated at - height int64 - - // gasWanted defines the amount of gas the transaction sender requires - gasWanted int64 - - // estimatedGas defines the amount of gas that the transaction is estimated to use - estimatedGas int64 - - // priority defines the transaction's priority as specified by the application - // in the ResponseCheckTx response. - priority int64 - - // timestamp is the time at which the node first received the transaction from - // a peer. It is used as a second dimension is prioritizing transactions when - // two transactions have the same priority. - timestamp time.Time - - // peers records a mapping of all peers that sent a given transaction - peers map[uint16]struct{} - - // gossipEl references the linked-list element in the gossip index - readyEl utils.Option[*clist.CElement[*WrappedTx]] - - // evm properties that aid in prioritization - evm utils.Option[evmTx] + height int64 // height defines the height at which the transaction was validated at + gasWanted int64 // gasWanted defines the amount of gas the transaction sender requires + estimatedGas int64 // estimatedGas defines the amount of gas that the transaction is estimated to use + priority int64 // ResponseCheckTx.priority + timestamp time.Time // time at which the transaction was received + readyEl utils.Option[*clist.CElement[*WrappedTx]] // linked-list element in the gossip index + evm utils.Option[evmTx] // evm transaction info } type evmTx struct { @@ -100,14 +79,30 @@ type evmAccount struct { nextNonce uint64 } +type txCounter struct { + count int + bytes uint64 +} + +func (c *txCounter) Inc(bytes uint64) { + c.count += 1 + c.bytes += bytes +} + +func (c *txCounter) Dec(bytes uint64) { + c.count -= 1 + c.bytes -= bytes +} + type txStoreState struct { - readyCount int - readyBytes uint64 - pendingCount int - pendingBytes uint64 + ready txCounter + total txCounter } -type txStoreV2Inner struct { +func (s txStoreState) PendingBytes() uint64 { return s.total.bytes - s.ready.bytes } +func (s txStoreState) PendingCount() int { return s.total.count - s.ready.count } + +type txStoreInner struct { byHash map[types.TxHash]*WrappedTx byNonce map[evmAddrNonce]*WrappedTx accounts map[common.Address]*evmAccount @@ -115,10 +110,10 @@ type txStoreV2Inner struct { state utils.AtomicSend[txStoreState] } -type txStoreV2 struct { +type txStore struct { config *Config - proxy *proxy.Proxy - inner utils.RWMutex[*txStoreV2Inner] + app *proxy.Proxy + inner utils.RWMutex[*txStoreInner] state utils.AtomicRecv[txStoreState] // gossipIndex defines the gossiping index of valid transactions via a // thread-safe linked-list. We also use the gossip index as a cursor for @@ -126,13 +121,13 @@ type txStoreV2 struct { readyTxs *clist.CList[*WrappedTx] } -func NewTxStore() *txStoreV2 { - inner := &txStoreV2Inner{ +func NewTxStore() *txStore { + inner := &txStoreInner{ byHash: map[types.TxHash]*WrappedTx{}, accounts: map[common.Address]*evmAccount{}, state: utils.NewAtomicSend(txStoreState{}), } - return &txStoreV2{ + return &txStore{ inner: utils.NewRWMutex(inner), readyTxs: clist.New[*WrappedTx](), state: inner.state.Subscribe(), @@ -140,23 +135,25 @@ func NewTxStore() *txStoreV2 { } // Size returns the total number of transactions in the store. -func (txs *txStoreV2) Size() int { return txs.state.Load().readyCount } - -// AllTxsBytes returns the total size in bytes of all transactions in the store. -func (txs *txStoreV2) AllTxsBytes() uint64 { return txs.state.Load().readyBytes } -func (txs *txStoreV2) TotalBytes() uint64 { - state := txs.state.Load() - return state.pendingBytes + state.readyBytes -} +func (txs *txStore) State() txStoreState { return txs.state.Load() } // WaitForTxs waits until the store becomes non-empty. -func (txs *txStoreV2) WaitForTxs(ctx context.Context) error { - _, err := txs.state.Wait(ctx, func(s txStoreState) bool { return s.readyCount > 0 }) +func (txs *txStore) WaitForTxs(ctx context.Context) error { + _, err := txs.state.Wait(ctx, func(s txStoreState) bool { return s.ready.count > 0 }) return err } +func (txs *txStore) NextNonce(addr common.Address) uint64 { + for inner := range txs.inner.RLock() { + if acc,ok := inner.accounts[addr]; ok { + return acc.nextNonce + } + } + return txs.app.EvmNonce(addr) +} + // GetAllTxs returns all the transactions currently in the store. -func (txs *txStoreV2) GetAllTxs() []*WrappedTx { +func (txs *txStore) GetAllTxs() []*WrappedTx { for inner := range txs.inner.RLock() { return slices.Collect(maps.Values(inner.byHash)) } @@ -164,43 +161,66 @@ func (txs *txStoreV2) GetAllTxs() []*WrappedTx { } // GetTxByHash returns a *WrappedTx by the transaction's hash. -func (txs *txStoreV2) GetTxByHash(key types.TxHash) *WrappedTx { +func (txs *txStore) GetTxByHash(key types.TxHash) *WrappedTx { for inner := range txs.inner.RLock() { return inner.byHash[key] } panic("unreachable") } -func (txs *txStoreV2) insert(inner *txStoreV2Inner, wtx *WrappedTx) { +func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) { if _,ok := inner.byHash[wtx.Hash()]; ok { return } - if evm,ok := wtx.evm.Get(); ok { + state := inner.state.Load() + if evm,ok := wtx.evm.Get(); ok { + // Fetch the evm account state. + account,ok := inner.accounts[evm.address] + if !ok { + // TODO(gprusak): consider whether we should move these queries out of the mutex. + b := txs.app.EvmBalance(evm.address,evm.seiAddress) + n := txs.app.EvmNonce(evm.address) + account = &evmAccount{b,n,n} + inner.accounts[evm.address] = account + } an := evmAddrNonce{evm.address,evm.nonce} if old,ok := inner.byNonce[an]; ok { + // If the old tx is ready but the new tx is not, then reject new tx. + if old.evm.OrPanic("non-evm tx").nonce < account.nextNonce && account.balance.Cmp(evm.requiredBalance) < 0 { + return + } + // If the old tx has >= priority, then reject new tx. if old.priority >= wtx.priority { return } - // TODO: replace logic + // Remove the old transaction. + delete(inner.byHash,old.Hash()) + if el,ok := wtx.readyEl.Get(); ok { + txs.readyTxs.Remove(el) + } + state.ready.Dec(old.Size()) + state.total.Dec(old.Size()) + state.ready.Inc(wtx.Size()) } inner.byNonce[an] = wtx - account,ok := inner.accounts[evm.address] - if !ok { - b := txs.proxy.EvmBalance(evm.address,evm.seiAddress) - n := txs.proxy.EvmNonce(evm.address) - account = &evmAccount{b,n,n} - inner.accounts[evm.address] = account - } + // Update account ready txs. for { an.Nonce = account.nextNonce - if _,ok := inner.byNonce[an]; !ok { break } + wtx,ok := inner.byNonce[an] + if !ok || account.balance.Cmp(wtx.evm.OrPanic("non-evm tx").requiredBalance) < 0 { break } account.nextNonce += 1 + state.ready.Inc(wtx.Size()) } } + // TODO: non-evm txs are ready + state.total.Inc(wtx.Size()) inner.byHash[wtx.Hash()] = wtx if !wtx.readyEl.IsPresent() { wtx.readyEl = utils.Some(txs.readyTxs.PushBack(wtx)) } - // TODO: update status + inner.state.Store(state) + if highlimitExceeded { + txs.compact(inner) + } } -func (txs *txStoreV2) compact(inner *txStoreV2Inner) { +func (txs *txStore) compact(inner *txStoreInner) { // split into ready and not-ready txs var notReady []*WrappedTx var ready []*WrappedTx @@ -231,27 +251,20 @@ func (txs *txStoreV2) compact(inner *txStoreV2Inner) { txs.recompute(inner) } -func (txs *txStoreV2) ReapTxs(l ReapLimits, remove bool) (types.Txs, int64) { +func (txs *txStore) ReapTxs(l ReapLimits, remove bool) (types.Txs, int64) { // find ready and sort like in compact() // reap until limits // if remove { removeTxs(); recompute() } } // SetTx stores a *WrappedTx by its hash. -func (txs *txStoreV2) Insert(wtx *WrappedTx) { +func (txs *txStore) Insert(wtx *WrappedTx) { for inner := range txs.inner.Lock() { txs.insert(inner,wtx) - state := inner.state.Load() - state.readyCount += 1 - state.readyBytes += uint64(wtx.Size()) - inner.state.Store(state) - if highlimitExceeded { - txs.compact(inner) - } } } -func (txs *txStoreV2) recompute(inner *txStoreV2Inner) { +func (txs *txStore) recompute(inner *txStoreInner) { byHash := inner.byHash inner.byHash = map[types.TxHash]*WrappedTx{} inner.byNonce = map[evmAddrNonce]*WrappedTx{} @@ -266,7 +279,7 @@ func (txs *txStoreV2) recompute(inner *txStoreV2Inner) { // RemoveTx removes a *WrappedTx from the transaction store. It deletes all // indexes of the transaction. -func (txs *txStoreV2) removeTxs(inner *txStoreV2Inner, txHashes []types.TxHash) { +func (txs *txStore) removeTxs(inner *txStoreInner, txHashes []types.TxHash) { for _,txHash := range txHashes { wtx, ok := inner.byHash[txHash] if !ok { continue } @@ -278,37 +291,7 @@ func (txs *txStoreV2) removeTxs(inner *txStoreV2Inner, txHashes []types.TxHash) } } -// TxHasPeer returns true if a transaction by hash has a given peer ID and false -// otherwise. If the transaction does not exist, false is returned. -func (txs *txStoreV2) TxHasPeer(txHash types.TxHash, peerID uint16) bool { - for inner := range txs.inner.RLock() { - if wtx,ok := inner.byHash[txHash]; ok { - _, ok := wtx.peers[peerID] - return ok - } - } - return false -} - -// GetOrSetPeerByTxHash looks up a WrappedTx by transaction hash and adds the -// given peerID to the WrappedTx's set of peers that sent us this transaction. -// We return true if we've already recorded the given peer for this transaction -// and false otherwise. If the transaction does not exist by hash, we return -// (nil, false). -func (txs *txStoreV2) GetOrSetPeerByTxHash(hash types.TxHash, peerID uint16) (*WrappedTx, bool) { - for inner := range txs.inner.Lock() { - if wtx,ok := inner.byHash[hash]; ok { - if _, ok := wtx.peers[peerID]; ok { - return wtx, true - } - wtx.peers[peerID] = struct{}{} - return wtx, false - } - } - return nil, false -} - -func (txs *txStoreV2) UpdateHeight(now time.Time, blockHeight int64, blockTxs []types.TxHash) { +func (txs *txStore) UpdateHeight(now time.Time, blockHeight int64, blockTxs []types.TxHash) { minHeight := utils.None[int64]() if n := txs.config.TTLNumBlocks; n > 0 && blockHeight > n { minHeight = utils.Some(blockHeight - n) @@ -341,6 +324,3 @@ func (txs *txStoreV2) UpdateHeight(now time.Time, blockHeight int64, blockTxs [] txs.recompute(inner) } } - -func (txs *txStoreV2) PendingBytes() uint64 { return txs.state.Load().pendingBytes } -func (txs *txStoreV2) PendingSize() int { return txs.state.Load().pendingCount } From 5a91fa628f4922dab0662fd84f21a248db441abe Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 18 May 2026 12:00:10 +0200 Subject: [PATCH 003/100] WIP --- sei-tendermint/internal/mempool/mempool.go | 79 +-------- sei-tendermint/internal/mempool/tx.go | 188 ++++++++++++--------- 2 files changed, 111 insertions(+), 156 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 46220558b0..278c3d1362 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -598,54 +598,7 @@ func (txmp *TxMempool) PopTxs(l ReapLimits) (types.Txs, int64) { // Update iterates over all the transactions provided by the block producer, // removes them from the cache (if applicable), and removes // the transactions from the main transaction store and associated indexes. -// If there are transactions remaining in the mempool, we initiate a -// re-CheckTx for them (if applicable), otherwise, we notify the caller more -// transactions are available. -// -// WARNING: callers should almost always pass recheck=false. recheck=true -// re-runs CheckTx on every tx still in the mempool after each block, and -// handleRecheckResult treats a "now pending" response as terminal: it -// evicts the tx and async-re-CheckTx-es it, which lands it back in -// pendingTxs. For chains whose antehandler returns pending for any -// ahead-of-nonce EVM tx (Sei), this evicts perfectly-valid queued txs. -// -// Example. txA (nonce 3), txB (nonce 2), txC (nonce 1) on the same sender. -// -// 1. txA, txB, txC are submitted in this order. -// 2. txA and txB enter pendingTxs (their nonce is ahead of the sender's -// expected nonce at CheckTx time so the EVM antehandler marks them -// pending). txC enters the priority index (its nonce matches expected). -// 3. Block 1 reaps and mines txC. The sender's expected nonce becomes 2. -// 4. handlePendingTransactions promotes txA and txB into the priority -// index. The per-sender evmQueue is now [txB (head), txA (tail)]. -// -// From step 5 onwards the recheck flag matters: -// -// recheck=false (correct): -// -// 5. updateReCheckTxs is skipped. The priority index keeps txB and txA. -// 6. Block 2 reaps the whole evmQueue. Both txB and txA mine. -// -// All 3 txs mine in 2 blocks, regardless of how out-of-order they arrived. -// -// recheck=true (broken): -// -// 5. updateReCheckTxs re-runs CheckTx on each tx in the priority index: -// - txB: nonce 2 == expected 2 → not pending → stays. -// - txA: nonce 3 > expected 2 → pending again. handleRecheckResult -// evicts it and async-re-CheckTx-es it, which lands it back in -// pendingTxs. -// 6. Block 2 reaps txB only (txA is no longer in the priority index). -// handlePendingTransactions re-promotes txA. txA's nonce now matches -// expected, so it survives the recheck this time. -// 7. Block 3 mines txA. -// -// All 3 txs take 3 blocks. With many out-of-order sequential nonces from -// one sender, this stalls the chain to 1-tx-per-block-per-sender throughput. -// -// CometBFT's default for ConsensusParams.ABCI.RecheckTx is false. Recheck -// primarily defended against state-dependent invalidation that modern -// chains catch in ProcessProposal/DeliverTx anyway. +// If recheck = true, CheckTx is called on all remaining transactions. // // NOTE: // - The caller must explicitly acquire a write-lock. @@ -661,10 +614,11 @@ func (txmp *TxMempool) Update( txmp.notifiedTxsAvailable.Store(false) txmp.txConstraintsFetcher = txConstraintsFetcher + txHashes := make([]types.TxHash,len(blockTxs)) for i, tx := range blockTxs { txHash := tx.Hash() + txHashes[i] = txHash // Remove transaction from the mempool, no matter if it succeeded, or not. - txmp.removeTx(txHash) if execTxResult[i].Code == abci.CodeTypeOK { // add the valid committed transaction to the cache (if missing) txmp.cache.Push(txHash) @@ -677,15 +631,7 @@ func (txmp *TxMempool) Update( // Subsequent failures: leave in cache to prevent infinite re-entry } } - txmp.txStore.UpdateHeight(blockHeight) - - // If there any uncommitted transactions left in the mempool, we either - // initiate re-CheckTx per remaining transaction or notify that remaining - // transactions are left. - if recheck { - txmp.updateReCheckTxs(ctx) - } - + txmp.txStore.UpdateHeight(time.Now(), blockHeight, txHashes, recheck) txmp.notifyTxsAvailable() txmp.metrics.Size.Set(float64(txmp.NumTxsNotPending())) txmp.metrics.TotalTxsSizeBytes.Set(float64(txmp.TotalTxsBytesSize())) @@ -701,16 +647,6 @@ func (txmp *TxMempool) Update( // NOTE: // - The caller must have a write-lock when executing updateReCheckTxs. func (txmp *TxMempool) updateReCheckTxs(ctx context.Context) { - // TODO(gprusak): this whole recheck thing is basically doing TxMempool.CheckTx for all remaining - // txs without restarting the gossip though. - logger.Debug( - "executing re-CheckTx for all remaining transactions", - "num_txs", txmp.Size(), - "height", txmp.height, - ) - - for e := txmp.txStore.readyTxs.Front(); e != nil; e = e.Next() { - wtx := e.Value() res, err := txmp.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{ Tx: wtx.Tx(), Type: abci.CheckTxTypeV2Recheck, @@ -718,12 +654,6 @@ func (txmp *TxMempool) updateReCheckTxs(ctx context.Context) { if err == nil { err = res.Err() } - if err != nil { - // TODO(gprusak): check if it would be safer to just remove the tx here, instead of waiting for retry. - // no need in retrying since the tx will be rechecked after the next block - logger.Debug("failed to execute CheckTx during recheck", "err", err, "hash", wtx.Hash()) - continue - } txmp.metrics.RecheckTimes.Add(1) // we will treat a transaction that turns pending in a recheck as invalid and evict it @@ -735,7 +665,6 @@ func (txmp *TxMempool) updateReCheckTxs(ctx context.Context) { "err", err, "code", res.Code, ) - txmp.removeTx(wtx.Hash()) } wtx.priority = res.Priority diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 8098edc53d..3e938f3e63 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -99,6 +99,11 @@ type txStoreState struct { total txCounter } +// Partial order. +func (c *txCounter) LessEqual(b *txCounter) bool { + return c.count <= b.count && c.bytes <= b.bytes +} + func (s txStoreState) PendingBytes() uint64 { return s.total.bytes - s.ready.bytes } func (s txStoreState) PendingCount() int { return s.total.count - s.ready.count } @@ -106,7 +111,9 @@ type txStoreInner struct { byHash map[types.TxHash]*WrappedTx byNonce map[evmAddrNonce]*WrappedTx accounts map[common.Address]*evmAccount - + + softLimit txCounter + hardLimit txCounter state utils.AtomicSend[txStoreState] } @@ -115,19 +122,22 @@ type txStore struct { app *proxy.Proxy inner utils.RWMutex[*txStoreInner] state utils.AtomicRecv[txStoreState] - // gossipIndex defines the gossiping index of valid transactions via a - // thread-safe linked-list. We also use the gossip index as a cursor for - // rechecking transactions already in the mempool. + // list of ready transactions that can be gossiped. readyTxs *clist.CList[*WrappedTx] } -func NewTxStore() *txStore { +func NewTxStore(config *Config) *txStore { + softLimit := txCounter{count:config.Size, bytes: utils.Clamp[uint64](config.MaxTxsBytes)} + hardLimit := txCounter{count:2*softLimit.count, bytes: 2*softLimit.bytes} inner := &txStoreInner{ byHash: map[types.TxHash]*WrappedTx{}, accounts: map[common.Address]*evmAccount{}, + softLimit: softLimit, + hardLimit: hardLimit, state: utils.NewAtomicSend(txStoreState{}), } return &txStore{ + config: config, inner: utils.NewRWMutex(inner), readyTxs: clist.New[*WrappedTx](), state: inner.state.Subscribe(), @@ -183,7 +193,7 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) { } an := evmAddrNonce{evm.address,evm.nonce} if old,ok := inner.byNonce[an]; ok { - // If the old tx is ready but the new tx is not, then reject new tx. + // If the old tx is ready but the new tx is not, then reject the new tx. if old.evm.OrPanic("non-evm tx").nonce < account.nextNonce && account.balance.Cmp(evm.requiredBalance) < 0 { return } @@ -207,120 +217,136 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) { account.nextNonce += 1 state.ready.Inc(wtx.Size()) } + } else { + // Non-evm txs are automatically ready + state.ready.Inc(wtx.Size()) } - // TODO: non-evm txs are ready state.total.Inc(wtx.Size()) inner.byHash[wtx.Hash()] = wtx if !wtx.readyEl.IsPresent() { wtx.readyEl = utils.Some(txs.readyTxs.PushBack(wtx)) } - inner.state.Store(state) - if highlimitExceeded { - txs.compact(inner) - } + inner.state.Store(state) } -func (txs *txStore) compact(inner *txStoreInner) { - // split into ready and not-ready txs - var notReady []*WrappedTx - var ready []*WrappedTx +// WARNING: works only if wtx has been already inserted. +func (inner *txStoreInner) isReady(wtx *WrappedTx) bool { + evm,ok := wtx.evm.Get() + return !ok || evm.nonce < inner.accounts[evm.address].nextNonce +} + +// Sorts transactions in inclusion order. Here we effectively simulate the following: +// * find account with the highest priority lowest nonce ready transaction and pop this transaction +// * repeat until no ready transactions are available +// * then repeat the same but for pending transactions (i.e. again in per-account nonce order, high priority first, just ignoring readiness) +// Cosmos transactions are all considered ready and from different accounts, so only priority is relevant. +func (inner *txStoreInner) inInclusionOrder() []*WrappedTx { + // Split txs into ready and pending. + // TODO(gprusak): we can precisely preallocate ready and pending in a single array, + // based on inner.state.total.count and inner.state.ready.count + var ready,pending []*WrappedTx for _,wtx := range inner.byHash { - // TODO: apply balance and monotone priority checks - // earlier nonce has too high requiredBalance => not-ready - // earlier nonce has low prio => prio - our prio is capped - // order by (inc prio, dec nonce) - if evm,ok := wtx.evm.Get(); ok && evm.nonce >= inner.accounts[evm.address].nextNonce { - notReady = append(notReady,wtx) - } else { + if inner.isReady(wtx) { ready = append(ready,wtx) + } else { + pending = append(pending,wtx) } } - cmpPrio := func(a,b *WrappedTx) int { return cmp.Compare(a.priority,b.priority) } - // remove not-ready by priority - slices.SortFunc(notReady, cmpPrio) - for _,wtx := range notReady { - if !lowLimitExceeded {} - delete(inner.byHash,wtx.Hash()) - } - // remove ready by priority - slices.SortFunc(notReady, cmpPrio) - for _,wtx := range ready { - if !lowLimitExceeded {} - delete(inner.byHash,wtx.Hash()) + for _,txs := range utils.Slice(ready,pending) { + // Sort by nonce. + slices.SortFunc(txs,func(a,b *WrappedTx) int { return cmp.Compare(a.EVMNonce(),b.EVMNonce()) }) + // Cap priority to obtain a linear order of txs per account by nonce. + // NOTE: this precisely emulates the heap behavior described in this functions docstring. + accPrio := make(map[common.Address]int64,len(inner.accounts)) + txPrio := make(map[*WrappedTx]int64,len(txs)) + for _,tx := range txs { + if evm,ok := tx.evm.Get(); ok { + if prio,ok := accPrio[evm.address]; !ok || prio > tx.priority { + accPrio[evm.address] = tx.priority + } + txPrio[tx] = accPrio[evm.address] + } else { + txPrio[tx] = tx.priority + } + } + // Stable sort by capped priority - it preserves the nonce ordering. + slices.SortStableFunc(txs,func(a,b *WrappedTx) int { return -cmp.Compare(txPrio[a],txPrio[b]) }) } - txs.recompute(inner) -} - -func (txs *txStore) ReapTxs(l ReapLimits, remove bool) (types.Txs, int64) { - // find ready and sort like in compact() - // reap until limits - // if remove { removeTxs(); recompute() } + return append(ready,pending...) } // SetTx stores a *WrappedTx by its hash. func (txs *txStore) Insert(wtx *WrappedTx) { for inner := range txs.inner.Lock() { - txs.insert(inner,wtx) + txs.insert(inner,wtx) + if total := inner.state.Load().total; !total.LessEqual(&inner.hardLimit) { + txs.compact(inner, false) + } } } -func (txs *txStore) recompute(inner *txStoreInner) { - byHash := inner.byHash +func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { + wtxs := inner.inInclusionOrder() + inner.state.Store(txStoreState{}) inner.byHash = map[types.TxHash]*WrappedTx{} inner.byNonce = map[evmAddrNonce]*WrappedTx{} + if clearAccounts { + inner.accounts = map[common.Address]*evmAccount{} + } for _, account := range inner.accounts { account.nextNonce = account.firstNonce } - // TODO: reset status - for _,wtx := range byHash { - txs.insert(inner,wtx) - } -} - -// RemoveTx removes a *WrappedTx from the transaction store. It deletes all -// indexes of the transaction. -func (txs *txStore) removeTxs(inner *txStoreInner, txHashes []types.TxHash) { - for _,txHash := range txHashes { - wtx, ok := inner.byHash[txHash] - if !ok { continue } - // TODO: update status - delete(inner.byHash,txHash) - if el,ok := wtx.readyEl.Get(); ok { - txs.readyTxs.Remove(el) + total := txCounter{} + for _,wtx := range wtxs { + total.Inc(wtx.Size()) + if total.LessEqual(&inner.softLimit) { + txs.insert(inner,wtx) + } else { + if el,ok := wtx.readyEl.Get(); ok { + txs.readyTxs.Remove(el) + } } } } -func (txs *txStore) UpdateHeight(now time.Time, blockHeight int64, blockTxs []types.TxHash) { +func (txs *txStore) UpdateHeight(now time.Time, blockHeight int64, blockTxs []types.TxHash, recheck bool) { minHeight := utils.None[int64]() - if n := txs.config.TTLNumBlocks; n > 0 && blockHeight > n { - minHeight = utils.Some(blockHeight - n) + if ttl,ok := txs.config.TTLNumBlocks.Get(); ok && blockHeight > ttl { + minHeight = utils.Some(blockHeight - ttl) } minTime := utils.None[time.Time]() - if d := txs.config.TTLDuration; d > 0 { + if d,ok := txs.config.TTLDuration.Get(); ok { minTime = utils.Some(now.Add(-d)) } + toPrune := func(wtx *WrappedTx) bool { + if t, ok := minTime.Get(); ok && wtx.timestamp.Before(t) { + return true + } + if h, ok := minHeight.Get(); ok && wtx.height < h { + return true + } + return false + } for inner := range txs.inner.Lock() { - // All account states need to be reevaluated. - inner.accounts = map[common.Address]*evmAccount{} - // Sequenced txs are pruned. - txs.removeTxs(inner, blockTxs) - // Old txs are pruned. - for _, wtx := range inner.byHash { - isOlder := func() bool { - if t, ok := minTime.Get(); ok && wtx.timestamp.Before(t) { - return true + // Remove included. + for _, txHash := range blockTxs { + if wtx,ok := inner.byHash[txHash]; ok { + delete(inner.byHash,txHash) + if el,ok := wtx.readyEl.Get(); ok { + txs.readyTxs.Remove(el) } - if h, ok := minHeight.Get(); ok && wtx.height < h { - return true + } + } + // Prune old. + for txHash, wtx := range inner.byHash { + if toPrune(wtx) && (!inner.isReady(wtx) || txs.config.RemoveExpiredTxsFromQueue) { + delete(inner.byHash,txHash) + if el,ok := wtx.readyEl.Get(); ok { + txs.readyTxs.Remove(el) } - return false - }() - if isOlder && (pending || txs.config.RemoveExpiredTxsFromQueue) { - // TODO: remove } } - // if recheck { ... } - txs.recompute(inner) + // TODO: if recheck { ... } + txs.compact(inner,true) } } From a6feb77da6b6c09714956c6c4d86ff63325675a6 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 18 May 2026 18:38:32 +0200 Subject: [PATCH 004/100] WIP --- sei-tendermint/internal/mempool/mempool.go | 236 +++++---------------- sei-tendermint/internal/mempool/tx.go | 166 +++++++++++---- sei-tendermint/internal/mempool/types.go | 6 - sei-tendermint/internal/state/execution.go | 2 +- sei-tendermint/internal/state/tx_filter.go | 18 +- 5 files changed, 189 insertions(+), 239 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 278c3d1362..7475024cdf 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -5,7 +5,6 @@ import ( "crypto/sha256" "errors" "fmt" - "math/big" "sync" "sync/atomic" "time" @@ -217,7 +216,7 @@ func NewTxMempool( txsAvailable: make(chan struct{}, 1), height: -1, metrics: metrics, - txStore: NewTxStore(), + txStore: NewTxStore(cfg), txConstraintsFetcher: txConstraintsFetcher, priorityReservoir: reservoir.New[int64](cfg.DropPriorityReservoirSize, cfg.DropPriorityThreshold, nil), // Use non-deterministic RNG cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), @@ -267,30 +266,6 @@ func (txmp *TxMempool) WaitForNextTx(ctx context.Context) (*clist.CElement[*Wrap // when transactions are available in the mempool. It is thread-safe. func (txmp *TxMempool) TxsAvailable() <-chan struct{} { return txmp.txsAvailable } -func (txmp *TxMempool) removeTx(txHash types.TxHash) { - if txmp.txStore.Remove(txHash) { - txmp.metrics.RemovedTxs.Add(1) - } -} - -func (txmp *TxMempool) checkTxConstraints(wtx *WrappedTx) error { - constraints, err := txmp.txConstraintsFetcher() - if err != nil { - return err - } - - if constraints.MaxGas == -1 { - return nil - } - if wtx.gasWanted < 0 { - return fmt.Errorf("negative gas wanted: %d", wtx.gasWanted) - } - if wtx.gasWanted > constraints.MaxGas { - return fmt.Errorf("gas wanted exceeds max gas: gas wanted %d is greater than max gas %d", wtx.gasWanted, constraints.MaxGas) - } - return nil -} - // CheckTx executes the ABCI CheckTx method for a given transaction. // It acquires a read-lock and attempts to execute the application's // CheckTx ABCI method synchronously. We return an error if any of @@ -312,7 +287,7 @@ func (txmp *TxMempool) checkTxConstraints(wtx *WrappedTx) error { // NOTE: // - The applications' CheckTx implementation may panic. // - The caller is not to explicitly require any locks for executing CheckTx. -func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) (*abci.ResponseCheckTx, error) { +func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.ResponseCheckTx, error) { txmp.mtx.RLock() defer txmp.mtx.RUnlock() @@ -336,11 +311,11 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) hint, err := txmp.app.GetTxPriorityHint(ctx, &abci.RequestGetTxPriorityHintV2{Tx: tx}) if err != nil { - txmp.metrics.observeCheckTxPriorityDistribution(0, true, txInfo.SenderNodeID, true) + txmp.metrics.observeCheckTxPriorityDistribution(0, true, "", true) logger.Error("failed to get tx priority hint", "err", err) return nil, err } - txmp.metrics.observeCheckTxPriorityDistribution(hint.Priority, true, txInfo.SenderNodeID, false) + txmp.metrics.observeCheckTxPriorityDistribution(hint.Priority, true, "", false) cutoff, found := txmp.priorityReservoir.Percentile() if found && hint.Priority <= cutoff { @@ -360,14 +335,10 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) if c, ok := txmp.duplicateTxsCache.Get(); ok { c.Increment(hTx.Hash()) } - - if len(txInfo.SenderNodeID) == 0 { - txmp.metrics.NumberOfLocalCheckTx.Add(1) - } res, err := txmp.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{Tx: tx}) if err != nil || !res.IsOK() { txmp.metrics.NumberOfFailedCheckTxs.Add(1) - txmp.metrics.observeCheckTxPriorityDistribution(0, false, txInfo.SenderNodeID, true) + txmp.metrics.observeCheckTxPriorityDistribution(0, false, "", true) txmp.cache.Remove(hTx.Hash()) } if err != nil { @@ -377,14 +348,19 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) return res.ResponseCheckTx, nil } txmp.metrics.NumberOfSuccessfulCheckTxs.Add(1) - txmp.metrics.observeCheckTxPriorityDistribution(res.Priority, false, txInfo.SenderNodeID, false) + txmp.metrics.observeCheckTxPriorityDistribution(res.Priority, false, "", false) + // if the tx doesn't have a gas estimate, fallback to gas wanted + estimatedGas := res.GasEstimated + if estimatedGas >= MinGasEVMTx && estimatedGas <= res.GasWanted { + estimatedGas = res.GasWanted + } wtx := &WrappedTx{ hashedTx: hTx, timestamp: time.Now().UTC(), height: txmp.height, priority: res.Priority, - estimatedGas: res.GasEstimated, + estimatedGas: estimatedGas, gasWanted: res.GasWanted, } if res.IsEVM { @@ -411,7 +387,7 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx, txInfo TxInfo) // most accurate. txmp.priorityReservoir.Add(wtx.priority) - if err := txmp.checkTxConstraints(wtx); err != nil { + if err := wtx.check(constraints); err != nil { // ignore bad transactions logger.Info("rejected bad transaction", "priority", wtx.priority, "tx", wtx.Hash(), "post_check_err", err) txmp.metrics.FailedTxs.Add(1) @@ -436,7 +412,7 @@ func (txmp *TxMempool) GetTxsForHashes(txHashes []types.TxHash) types.Txs { txs := make([]types.Tx, 0, len(txHashes)) for _, txHash := range txHashes { - wtx := txmp.txStore.GetTxByHash(txHash) + wtx := txmp.txStore.ByHash(txHash) txs = append(txs, wtx.Tx()) } return txs @@ -449,12 +425,11 @@ func (txmp *TxMempool) SafeGetTxsForHashes(txHashes []types.TxHash) (types.Txs, txs := make([]types.Tx, 0, len(txHashes)) missing := []types.TxHash{} for _, txHash := range txHashes { - wtx := txmp.txStore.GetTxByHash(txHash) - if wtx == nil { + if wtx := txmp.txStore.ByHash(txHash); wtx!=nil { + txs = append(txs, wtx.Tx()) + } else { missing = append(missing, txHash) - continue } - txs = append(txs, wtx.Tx()) } return txs, missing } @@ -468,9 +443,7 @@ func (txmp *TxMempool) SafeGetTxsForHashes(txHashes []types.TxHash) (types.Txs, func (txmp *TxMempool) Flush() { txmp.mtx.Lock() defer txmp.mtx.Unlock() - for _, wtx := range txmp.txStore.GetAllTxs() { - txmp.removeTx(wtx.Hash()) - } + txmp.txStore = NewTxStore(txmp.config) txmp.cache.Reset() } @@ -490,111 +463,15 @@ func (txmp *TxMempool) Flush() { func (txmp *TxMempool) ReapMaxBytesMaxGas(maxBytes, maxGasWanted, maxGasEstimated int64) types.Txs { txmp.mtx.Lock() defer txmp.mtx.Unlock() - txs, _ := txmp.reapTxs(ReapLimits{ + txs, _ := txmp.txStore.ReapTxs(ReapLimits{ MaxBytes: utils.Some(maxBytes), MaxGasWanted: utils.Some(maxGasWanted), MaxGasEstimated: utils.Some(maxGasEstimated), }) + // TODO: first evm txs, then non-evm txs return txs } -type ReapLimits struct { - MaxTxs utils.Option[uint64] - MaxBytes utils.Option[int64] - MaxGasWanted utils.Option[int64] - MaxGasEstimated utils.Option[int64] -} - -// ReapMaxTxsBytesMaxGas returns a list of transactions within the provided tx, -// byte, and gas constraints together with the total estimated gas for the -// returned transactions. -// -// NOTE: Gas limits are enforced using int64 running totals. If those totals -// overflow, gas limit enforcement no longer works correctly. This preserves the -// historical behavior for backward compatibility. -func (txmp *TxMempool) reapTxs(l ReapLimits) (types.Txs, int64) { - maxTxs := l.MaxTxs.Or(utils.Max[uint64]()) - maxBytes := l.MaxBytes.Or(utils.Max[int64]()) - maxGasWanted := l.MaxGasWanted.Or(utils.Max[int64]()) - maxGasEstimated := l.MaxGasEstimated.Or(utils.Max[int64]()) - if maxBytes < 0 { - maxBytes = utils.Max[int64]() - } - if maxGasWanted < 0 { - maxGasWanted = utils.Max[int64]() - } - if maxGasEstimated < 0 { - maxGasEstimated = utils.Max[int64]() - } - totalGasWanted := int64(0) - totalGasEstimated := int64(0) - totalSize := int64(0) - numTxs := uint64(0) - encounteredGasUnfit := false - - if uint64(txmp.NumTxsNotPending()) < txmp.config.TxNotifyThreshold { //nolint:gosec // NumTxsNotPending returns non-negative value - // do not reap anything if threshold is not met - return []types.Tx{}, 0 - } - var evmTxs []types.Tx - var nonEvmTxs []types.Tx - for wtx := range txmp.txStore.IterByPriority() { - // bytes limit is a hard stop - if wtx.protoSize > maxBytes-totalSize || numTxs >= maxTxs { - break - } - - // if the tx doesn't have a gas estimate, fallback to gas wanted - var txGasEstimate int64 - if wtx.estimatedGas >= MinGasEVMTx && wtx.estimatedGas <= wtx.gasWanted { - txGasEstimate = wtx.estimatedGas - } else { - wtx.estimatedGas = wtx.gasWanted - txGasEstimate = wtx.gasWanted - } - - limitExceeded := (maxGasWanted - totalGasWanted < wtx.gasWanted) || - (maxGasEstimated - totalGasEstimated < txGasEstimate) - - if limitExceeded { - // skip this unfit-by-gas tx once and attempt to pull up to 10 smaller ones - if !encounteredGasUnfit && numTxs < MinTxsToPeek { - encounteredGasUnfit = true - continue - } - break - } - - // include tx and update totals - numTxs += 1 - totalSize += wtx.protoSize - totalGasWanted += wtx.gasWanted - totalGasEstimated += txGasEstimate - - if wtx.evm.IsPresent() { - evmTxs = append(evmTxs, wtx.Tx()) - } else { - nonEvmTxs = append(nonEvmTxs, wtx.Tx()) - } - if encounteredGasUnfit && numTxs >= MinTxsToPeek { - break - } - } - - return append(evmTxs, nonEvmTxs...), totalGasEstimated -} - -// RemoveTxs removes the provided transactions from the mempool if present. -func (txmp *TxMempool) PopTxs(l ReapLimits) (types.Txs, int64) { - txmp.Lock() - defer txmp.Unlock() - txs, gasEstimated := txmp.reapTxs(l) - for _, tx := range txs { - txmp.removeTx(tx.Hash()) - } - return txs, gasEstimated -} - // Update iterates over all the transactions provided by the block producer, // removes them from the cache (if applicable), and removes // the transactions from the main transaction store and associated indexes. @@ -607,17 +484,19 @@ func (txmp *TxMempool) Update( blockHeight int64, blockTxs types.Txs, execTxResult []*abci.ExecTxResult, - txConstraintsFetcher TxConstraintsFetcher, + txConstraints TxConstraints, recheck bool, ) error { txmp.height = blockHeight txmp.notifiedTxsAvailable.Store(false) - txmp.txConstraintsFetcher = txConstraintsFetcher + txmp.txConstraintsFetcher = func() (TxConstraints,error) { + return txConstraints,nil + } - txHashes := make([]types.TxHash,len(blockTxs)) + txHashes := map[types.TxHash]struct{}{} for i, tx := range blockTxs { txHash := tx.Hash() - txHashes[i] = txHash + txHashes[txHash] = struct{}{} // Remove transaction from the mempool, no matter if it succeeded, or not. if execTxResult[i].Code == abci.CodeTypeOK { // add the valid committed transaction to the cache (if missing) @@ -631,7 +510,32 @@ func (txmp *TxMempool) Update( // Subsequent failures: leave in cache to prevent infinite re-entry } } - txmp.txStore.UpdateHeight(time.Now(), blockHeight, txHashes, recheck) + newPriority := map[types.TxHash]int64{} + if recheck { + for _, wtx := range txmp.txStore.AllReady() { + if _,ok := txHashes[wtx.Hash()]; ok { + continue + } + txmp.metrics.RecheckTimes.Add(1) + res, err := txmp.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{ + Tx: wtx.Tx(), + Type: abci.CheckTxTypeV2Recheck, + }) + // If recheck fails, just remove the tx. + if err!=nil || res.IsOK() { + txHashes[wtx.Hash()] = struct{}{} + } else { + newPriority[wtx.Hash()] = res.Priority + } + } + } + txmp.txStore.Update(updateSpec { + Now: time.Now(), + Height: blockHeight, + TxsToRemove: txHashes, + NewPriorities: newPriority, + Constraints: txConstraints, + }) txmp.notifyTxsAvailable() txmp.metrics.Size.Set(float64(txmp.NumTxsNotPending())) txmp.metrics.TotalTxsSizeBytes.Set(float64(txmp.TotalTxsBytesSize())) @@ -639,42 +543,6 @@ func (txmp *TxMempool) Update( return nil } -// updateReCheckTxs updates the recheck cursors using the gossipIndex. For -// each transaction, it executes CheckTx. The global callback defined on -// the app will be executed for each transaction after CheckTx is -// executed. -// -// NOTE: -// - The caller must have a write-lock when executing updateReCheckTxs. -func (txmp *TxMempool) updateReCheckTxs(ctx context.Context) { - res, err := txmp.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{ - Tx: wtx.Tx(), - Type: abci.CheckTxTypeV2Recheck, - }) - if err == nil { - err = res.Err() - } - txmp.metrics.RecheckTimes.Add(1) - - // we will treat a transaction that turns pending in a recheck as invalid and evict it - if err := txmp.checkTxConstraints(wtx); err != nil || res.Code != abci.CodeTypeOK { - logger.Debug( - "existing transaction no longer valid; failed re-CheckTx callback", - "priority", wtx.priority, - "tx", wtx.Hash(), - "err", err, - "code", res.Code, - ) - } - - wtx.priority = res.Priority - if evm, ok := wtx.evm.Get(); ok { - evm.requiredBalance = new(big.Int).Set(res.EVMRequiredBalance) - wtx.evm = utils.Some(evm) - } - } -} - func (txmp *TxMempool) notifyTxsAvailable() { if txmp.NumTxsNotPending() == 0 || txmp.notifiedTxsAvailable.Swap(true) { return diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 3e938f3e63..184062de2a 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -7,6 +7,7 @@ import ( "math/big" "time" "cmp" + "fmt" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/ethereum/go-ethereum/common" @@ -15,18 +16,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) -// TxInfo are parameters that get passed when attempting to add a tx to the -// mempool. -type TxInfo struct { - // SenderID is the internal peer ID used in the mempool to identify the - // sender, storing two bytes with each transaction instead of 20 bytes for - // the types.NodeID. - SenderID uint16 - - // SenderNodeID is the actual types.NodeID of the sender. - SenderNodeID types.NodeID -} - type hashedTx struct { tx types.Tx hash types.TxHash @@ -56,11 +45,21 @@ type WrappedTx struct { evm utils.Option[evmTx] // evm transaction info } +func (wtx *WrappedTx) check(c TxConstraints) error { + if wtx.gasWanted < 0 { + return fmt.Errorf("negative gas wanted: %d", wtx.gasWanted) + } + if c.MaxGas >= 0 && wtx.gasWanted > c.MaxGas { + return fmt.Errorf("gas wanted exceeds max gas: gas wanted %d is greater than max gas %d", wtx.gasWanted, c.MaxGas) + } + return nil +} + type evmTx struct { address common.Address seiAddress []byte nonce uint64 - // evmRequiredBalance is the sender balance threshold for this EVM tx to become ready. + // requiredBalance is the sender balance threshold for this EVM tx to become ready. requiredBalance *big.Int } @@ -170,8 +169,20 @@ func (txs *txStore) GetAllTxs() []*WrappedTx { panic("unreachable") } +func (txs *txStore) AllReady() []*WrappedTx { + var ready []*WrappedTx + for inner := range txs.inner.RLock() { + for _,wtx := range inner.byHash { + if inner.isReady(wtx) { + ready = append(ready,wtx) + } + } + } + return ready +} + // GetTxByHash returns a *WrappedTx by the transaction's hash. -func (txs *txStore) GetTxByHash(key types.TxHash) *WrappedTx { +func (txs *txStore) ByHash(key types.TxHash) *WrappedTx { for inner := range txs.inner.RLock() { return inner.byHash[key] } @@ -191,6 +202,10 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) { account = &evmAccount{b,n,n} inner.accounts[evm.address] = account } + // Reject transactions with old nonces. + if evm.nonce < account.firstNonce { + return + } an := evmAddrNonce{evm.address,evm.nonce} if old,ok := inner.byNonce[an]; ok { // If the old tx is ready but the new tx is not, then reject the new tx. @@ -296,8 +311,8 @@ func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { for _, account := range inner.accounts { account.nextNonce = account.firstNonce } - total := txCounter{} for _,wtx := range wtxs { + total := inner.state.Load().total total.Inc(wtx.Size()) if total.LessEqual(&inner.softLimit) { txs.insert(inner,wtx) @@ -309,44 +324,119 @@ func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { } } -func (txs *txStore) UpdateHeight(now time.Time, blockHeight int64, blockTxs []types.TxHash, recheck bool) { +type updateSpec struct { + Now time.Time + Height int64 + TxsToRemove map[types.TxHash]struct{} + Constraints TxConstraints + NewPriorities map[types.TxHash]int64 +} + +func (txs *txStore) Update(spec updateSpec) { minHeight := utils.None[int64]() - if ttl,ok := txs.config.TTLNumBlocks.Get(); ok && blockHeight > ttl { - minHeight = utils.Some(blockHeight - ttl) + if ttl,ok := txs.config.TTLNumBlocks.Get(); ok && spec.Height > ttl { + minHeight = utils.Some(spec.Height - ttl) } minTime := utils.None[time.Time]() if d,ok := txs.config.TTLDuration.Get(); ok { - minTime = utils.Some(now.Add(-d)) - } - toPrune := func(wtx *WrappedTx) bool { - if t, ok := minTime.Get(); ok && wtx.timestamp.Before(t) { - return true - } - if h, ok := minHeight.Get(); ok && wtx.height < h { - return true - } - return false + minTime = utils.Some(spec.Now.Add(-d)) } for inner := range txs.inner.Lock() { - // Remove included. - for _, txHash := range blockTxs { - if wtx,ok := inner.byHash[txHash]; ok { - delete(inner.byHash,txHash) - if el,ok := wtx.readyEl.Get(); ok { - txs.readyTxs.Remove(el) - } + toRemove := func(wtx *WrappedTx) bool { + if _,ok := spec.TxsToRemove[wtx.Hash()]; ok { + return true } + if wtx.check(spec.Constraints) != nil { + return true + } + // Consider expiration. + if inner.isReady(wtx) && !txs.config.RemoveExpiredTxsFromQueue { + return false + } + if t, ok := minTime.Get(); ok && wtx.timestamp.Before(t) { + return true + } + if h, ok := minHeight.Get(); ok && wtx.height < h { + return true + } + return false } - // Prune old. for txHash, wtx := range inner.byHash { - if toPrune(wtx) && (!inner.isReady(wtx) || txs.config.RemoveExpiredTxsFromQueue) { + if toRemove(wtx) { delete(inner.byHash,txHash) if el,ok := wtx.readyEl.Get(); ok { txs.readyTxs.Remove(el) } + } else if newPriority,ok := spec.NewPriorities[wtx.Hash()]; ok { + wtx.priority = newPriority } } - // TODO: if recheck { ... } txs.compact(inner,true) } } + +type ReapLimits struct { + MaxTxs utils.Option[uint64] + MaxBytes utils.Option[int64] // Max total bytes in proto representation. + MaxGasWanted utils.Option[int64] + MaxGasEstimated utils.Option[int64] +} + +// ReapTxs returns a list of transactions within the provided tx, +// byte, and gas constraints together with the total estimated gas for the +// returned transactions. +func (txs *txStore) ReapTxs(l ReapLimits) (types.Txs, int64) { + maxTxs := l.MaxTxs.Or(utils.Max[uint64]()) + maxBytes := l.MaxBytes.Or(utils.Max[int64]()) + maxGasWanted := l.MaxGasWanted.Or(utils.Max[int64]()) + maxGasEstimated := l.MaxGasEstimated.Or(utils.Max[int64]()) + if maxBytes < 0 { + maxBytes = utils.Max[int64]() + } + if maxGasWanted < 0 { + maxGasWanted = utils.Max[int64]() + } + if maxGasEstimated < 0 { + maxGasEstimated = utils.Max[int64]() + } + totalGasWanted := int64(0) + totalGasEstimated := int64(0) + totalSize := int64(0) + + for inner := range txs.inner.Lock() { + if uint64(inner.state.Load().ready.count) < txs.config.TxNotifyThreshold { //nolint:gosec + // do not reap anything if threshold is not met + return types.Txs{}, 0 + } + var txs []types.Tx + encounteredGasUnfit := false + for _,wtx := range inner.inInclusionOrder() { + // bytes limit is a hard stop + if wtx.protoSize > maxBytes-totalSize || uint64(len(txs)) >= maxTxs { + break + } + limitExceeded := (maxGasWanted - totalGasWanted < wtx.gasWanted) || + (maxGasEstimated - totalGasEstimated < wtx.estimatedGas) + + if limitExceeded { + // skip this unfit-by-gas tx once and attempt to pull up to 10 smaller ones + if !encounteredGasUnfit && len(txs) < MinTxsToPeek { + encounteredGasUnfit = true + continue + } + break + } + + // include tx and update totals + totalSize += wtx.protoSize + totalGasWanted += wtx.gasWanted + totalGasEstimated += wtx.estimatedGas + txs = append(txs, wtx.Tx()) + if encounteredGasUnfit && len(txs) >= MinTxsToPeek { + break + } + } + return txs, totalGasEstimated + } + panic("unreachable") +} diff --git a/sei-tendermint/internal/mempool/types.go b/sei-tendermint/internal/mempool/types.go index d3343beb37..97d0b4951e 100644 --- a/sei-tendermint/internal/mempool/types.go +++ b/sei-tendermint/internal/mempool/types.go @@ -2,12 +2,6 @@ package mempool import "math" -const ( - // UnknownPeerID is the peer ID to use when running CheckTx when there is - // no peer (e.g. RPC) - UnknownPeerID uint16 = 0 -) - // TxConstraints contains the precomputed consensus-derived mempool limits for // the current state snapshot. type TxConstraints struct { diff --git a/sei-tendermint/internal/state/execution.go b/sei-tendermint/internal/state/execution.go index 03147cc3e2..8f1c8bcce8 100644 --- a/sei-tendermint/internal/state/execution.go +++ b/sei-tendermint/internal/state/execution.go @@ -484,7 +484,7 @@ func (blockExec *BlockExecutor) Commit( block.Height, block.Txs, txResults, - TxConstraintsFetcherForState(state), + TxConstraintsForState(state), state.ConsensusParams.ABCI.RecheckTx, ) blockExec.metrics.UpdateMempoolTime.Observe(float64(time.Since(start))) diff --git a/sei-tendermint/internal/state/tx_filter.go b/sei-tendermint/internal/state/tx_filter.go index 3ee20dc02e..809c1b2be7 100644 --- a/sei-tendermint/internal/state/tx_filter.go +++ b/sei-tendermint/internal/state/tx_filter.go @@ -48,18 +48,16 @@ func TxConstraintsFetcherFromStore(store Store) mempool.TxConstraintsFetcher { return mempool.TxConstraints{}, err } - return TxConstraintsFetcherForState(state)() + return TxConstraintsForState(state),nil } } -func TxConstraintsFetcherForState(state State) mempool.TxConstraintsFetcher { - return func() (mempool.TxConstraints, error) { - return mempool.TxConstraints{ - MaxDataBytes: types.MaxDataBytesNoEvidence( - state.ConsensusParams.Block.MaxBytes, - state.Validators.Size(), - ), - MaxGas: state.ConsensusParams.Block.MaxGas, - }, nil +func TxConstraintsForState(state State) mempool.TxConstraints { + return mempool.TxConstraints{ + MaxDataBytes: types.MaxDataBytesNoEvidence( + state.ConsensusParams.Block.MaxBytes, + state.Validators.Size(), + ), + MaxGas: state.ConsensusParams.Block.MaxGas, } } From df95cdefa2965959eaac3de0288c9f8f9d08b2e9 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 18 May 2026 18:51:56 +0200 Subject: [PATCH 005/100] removed priority queue --- sei-tendermint/internal/mempool/mempool.go | 10 +- .../internal/mempool/priority_queue.go | 374 -------------- .../internal/mempool/priority_queue_test.go | 489 ------------------ .../internal/mempool/reactor/reactor.go | 15 +- sei-tendermint/internal/mempool/tx.go | 4 +- 5 files changed, 13 insertions(+), 879 deletions(-) delete mode 100644 sei-tendermint/internal/mempool/priority_queue.go delete mode 100644 sei-tendermint/internal/mempool/priority_queue_test.go diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 7475024cdf..c13e14da64 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -258,7 +258,7 @@ func (txmp *TxMempool) utilisation() float64 { // WaitForNextTx waits until the next transaction is available for gossip. // Returns the next valid transaction to gossip. -func (txmp *TxMempool) WaitForNextTx(ctx context.Context) (*clist.CElement[*WrappedTx], error) { +func (txmp *TxMempool) WaitForReadyTx(ctx context.Context) (*clist.CElement[*WrappedTx], error) { return txmp.txStore.readyTxs.WaitFront(ctx) } @@ -510,7 +510,7 @@ func (txmp *TxMempool) Update( // Subsequent failures: leave in cache to prevent infinite re-entry } } - newPriority := map[types.TxHash]int64{} + newPriorities := map[types.TxHash]int64{} if recheck { for _, wtx := range txmp.txStore.AllReady() { if _,ok := txHashes[wtx.Hash()]; ok { @@ -525,15 +525,15 @@ func (txmp *TxMempool) Update( if err!=nil || res.IsOK() { txHashes[wtx.Hash()] = struct{}{} } else { - newPriority[wtx.Hash()] = res.Priority + newPriorities[wtx.Hash()] = res.Priority } } } txmp.txStore.Update(updateSpec { Now: time.Now(), Height: blockHeight, - TxsToRemove: txHashes, - NewPriorities: newPriority, + ToRemove: txHashes, + NewPriorities: newPriorities, Constraints: txConstraints, }) txmp.notifyTxsAvailable() diff --git a/sei-tendermint/internal/mempool/priority_queue.go b/sei-tendermint/internal/mempool/priority_queue.go deleted file mode 100644 index 1e2a721ac3..0000000000 --- a/sei-tendermint/internal/mempool/priority_queue.go +++ /dev/null @@ -1,374 +0,0 @@ -package mempool - -import ( - "cmp" - "container/heap" - "slices" - "sort" - "sync" - - "github.com/ethereum/go-ethereum/common" - tmmath "github.com/sei-protocol/sei-chain/sei-tendermint/libs/math" -) - -var _ heap.Interface = (*TxPriorityQueue)(nil) - -// TxPriorityQueue defines a thread-safe priority queue for valid transactions. -type TxPriorityQueue struct { - mtx sync.RWMutex - txs []*WrappedTx // priority heap - // invariant 1: no duplicate nonce in the same queue - // invariant 2: no nonce gap in the same queue - // invariant 3: head of the queue must be in heap - evmQueue map[common.Address][]*WrappedTx // indexed by sender address, sorted by nonce -} - -func insertToEVMQueue(queue []*WrappedTx, tx *WrappedTx, i int) []*WrappedTx { - // Make room for new value and add it - queue = append(queue, nil) - copy(queue[i+1:], queue[i:]) - queue[i] = tx - return queue -} - -// binarySearch finds the index at which nonce should be inserted in queue and -// whether an exact nonce match already exists. -func binarySearch(queue []*WrappedTx, nonce uint64) (int, bool) { - return slices.BinarySearchFunc(queue, nonce, func(tx *WrappedTx, target uint64) int { - return cmp.Compare(tx.EVMNonce(), target) - }) -} - -func NewTxPriorityQueue() *TxPriorityQueue { - pq := &TxPriorityQueue{ - txs: nil, - evmQueue: map[common.Address][]*WrappedTx{}, - } - heap.Init(pq) - return pq -} - -func (pq *TxPriorityQueue) TxByAddrNonce(addr common.Address, nonce uint64) (*WrappedTx, int) { - pq.mtx.RLock() - defer pq.mtx.RUnlock() - return pq.txByAddrNonceUnsafe(addr, nonce) -} - -func (pq *TxPriorityQueue) txByAddrNonceUnsafe(addr common.Address, nonce uint64) (*WrappedTx, int) { - queue := pq.evmQueue[addr] - if idx, found := binarySearch(queue, nonce); found { - return queue[idx], idx - } - return nil, -1 -} - -// GetEvictableTxs attempts to find and return a list of *WrappedTx than can be -// evicted to make room for another *WrappedTx with higher priority. If no such -// list of *WrappedTx exists, nil will be returned. The returned list of *WrappedTx -// indicate that these transactions can be removed due to them being of lower -// priority and that their total sum in size allows room for the incoming -// transaction according to the mempool's configured limits. -func (pq *TxPriorityQueue) GetEvictableTxs(priority, txSize, totalSize, cap int64) []*WrappedTx { - pq.mtx.RLock() - defer pq.mtx.RUnlock() - txs := append([]*WrappedTx{}, pq.txs...) - for _, queue := range pq.evmQueue { - txs = append(txs, queue[1:]...) - } - - sort.Slice(txs, func(i, j int) bool { - return txs[i].priority < txs[j].priority - }) - - var ( - toEvict []*WrappedTx - i int - ) - - currSize := totalSize - - // Loop over all transactions in ascending priority order evaluating those - // that are only of less priority than the provided argument. We continue - // evaluating transactions until there is sufficient capacity for the new - // transaction (size) as defined by txSize. - for i < len(txs) && txs[i].priority < priority { - toEvict = append(toEvict, txs[i]) - currSize -= int64(txs[i].Size()) - - if currSize+txSize <= cap { - return toEvict - } - - i++ - } - - return nil -} - -// requires read lock -func (pq *TxPriorityQueue) numQueuedUnsafe() int { - var result int - for _, queue := range pq.evmQueue { - result += len(queue) - } - // first items in queue are also in heap, subtract one - return result - len(pq.evmQueue) -} - -// NumTxs returns the number of transactions in the priority queue. It is -// thread safe. -func (pq *TxPriorityQueue) NumTxs() int { - pq.mtx.RLock() - defer pq.mtx.RUnlock() - - return len(pq.txs) + pq.numQueuedUnsafe() -} - -func (pq *TxPriorityQueue) removeQueuedEvmTxUnsafe(tx *WrappedTx) (removedIdx int) { - evm, ok := tx.evm.Get() - if !ok { - return -1 - } - if queue, ok := pq.evmQueue[evm.address]; ok { - for i, t := range queue { - if t.Hash() == tx.Hash() { - pq.evmQueue[evm.address] = append(queue[:i], queue[i+1:]...) - if len(pq.evmQueue[evm.address]) == 0 { - delete(pq.evmQueue, evm.address) - } - return i - } - } - } - return -1 -} - -func (pq *TxPriorityQueue) findTxIndexUnsafe(tx *WrappedTx) (int, bool) { - // safety check for race situation where heapIndex is out of range of txs - if tx.heapIndex >= 0 && tx.heapIndex < len(pq.txs) && pq.txs[tx.heapIndex].Hash() == tx.Hash() { - return tx.heapIndex, true - } - - // heap index isn't trustable here, so attempt to find it - for i, t := range pq.txs { - if t.Hash() == tx.Hash() { - return i, true - } - } - return 0, false -} - -// RemoveTx removes a specific transaction from the priority queue. -func (pq *TxPriorityQueue) RemoveTx(tx *WrappedTx, shouldReenqueue bool) (toBeReenqueued []*WrappedTx) { - pq.mtx.Lock() - defer pq.mtx.Unlock() - - var removedIdx int - - if idx, ok := pq.findTxIndexUnsafe(tx); ok { - heap.Remove(pq, idx) - if evm, ok := tx.evm.Get(); ok { - removedIdx = pq.removeQueuedEvmTxUnsafe(tx) - if !shouldReenqueue && len(pq.evmQueue[evm.address]) > 0 { - heap.Push(pq, pq.evmQueue[evm.address][0]) - } - } - } else if tx.evm.IsPresent() { - removedIdx = pq.removeQueuedEvmTxUnsafe(tx) - } - if evm, ok := tx.evm.Get(); ok && shouldReenqueue && len(pq.evmQueue[evm.address]) > 0 && removedIdx >= 0 { - toBeReenqueued = pq.evmQueue[evm.address][removedIdx:] - } - return -} - -func (pq *TxPriorityQueue) pushTxUnsafe(tx *WrappedTx) { - evm, ok := tx.evm.Get() - if !ok { - heap.Push(pq, tx) - return - } - - // if there aren't other waiting txs, init and return - queue, exists := pq.evmQueue[evm.address] - if !exists { - pq.evmQueue[evm.address] = []*WrappedTx{tx} - heap.Push(pq, tx) - return - } - - // this item is on the heap at the moment - first := queue[0] - - // the queue's first item (and ONLY the first item) must be on the heap - // if this tx is before the first item, then we need to remove the first - // item from the heap - if evm.nonce < first.EVMNonce() { - if idx, ok := pq.findTxIndexUnsafe(first); ok { - heap.Remove(pq, idx) - } - heap.Push(pq, tx) - } - idx, _ := binarySearch(queue, evm.nonce) - pq.evmQueue[evm.address] = insertToEVMQueue(queue, tx, idx) -} - -func (pq *TxPriorityQueue) popTxUnsafe() *WrappedTx { - if len(pq.txs) == 0 { - return nil - } - - // remove the first item from the heap - x := heap.Pop(pq) - if x == nil { - return nil - } - tx := x.(*WrappedTx) - - // this situation is primarily for a test case that inserts nils - if tx == nil { - return nil - } - - // non-evm transactions do not have txs waiting on a nonce - evm, ok := tx.evm.Get() - if !ok { - return tx - } - - // evm transactions can have txs waiting on this nonce - // if there are any, we should replace the heap with the next nonce - // for the address - - // remove the first item from the evmQueue - pq.removeQueuedEvmTxUnsafe(tx) - - // if there is a next item, now it can be added to the heap - if len(pq.evmQueue[evm.address]) > 0 { - heap.Push(pq, pq.evmQueue[evm.address][0]) - } - - return tx -} - -// PopTx removes the top priority transaction from the queue. It is thread safe. -func (pq *TxPriorityQueue) PopTx() *WrappedTx { - pq.mtx.Lock() - defer pq.mtx.Unlock() - - return pq.popTxUnsafe() -} - -// dequeue up to `max` transactions and reenqueue while locked -func (pq *TxPriorityQueue) ForEachTx(handler func(wtx *WrappedTx) bool) { - pq.mtx.Lock() - defer pq.mtx.Unlock() - - numTxs := len(pq.txs) + pq.numQueuedUnsafe() - - txs := make([]*WrappedTx, 0, numTxs) - - defer func() { - for _, tx := range txs { - pq.pushTxUnsafe(tx) - } - }() - - for range numTxs { - popped := pq.popTxUnsafe() - if popped == nil { - break - } - txs = append(txs, popped) - if !handler(popped) { - return - } - } -} - -// dequeue up to `max` transactions and reenqueue while locked -// TODO: use ForEachTx instead -func (pq *TxPriorityQueue) PeekTxs(max int) []*WrappedTx { - pq.mtx.Lock() - defer pq.mtx.Unlock() - - numTxs := len(pq.txs) + pq.numQueuedUnsafe() - if max < 0 { - max = numTxs - } - - cap := tmmath.MinInt(numTxs, max) - res := make([]*WrappedTx, 0, cap) - for i := 0; i < cap; i++ { - popped := pq.popTxUnsafe() - if popped == nil { - break - } - - res = append(res, popped) - } - - for _, tx := range res { - pq.pushTxUnsafe(tx) - } - return res -} - -// Push implements the Heap interface. -// -// NOTE: A caller should never call Push. Use PushTx instead. -func (pq *TxPriorityQueue) Push(x interface{}) { - n := len(pq.txs) - item := x.(*WrappedTx) - item.heapIndex = n - pq.txs = append(pq.txs, item) -} - -// Pop implements the Heap interface. -// -// NOTE: A caller should never call Pop. Use PopTx instead. -func (pq *TxPriorityQueue) Pop() interface{} { - old := pq.txs - n := len(old) - item := old[n-1] - old[n-1] = nil // avoid memory leak - setHeapIndex(item, -1) // for safety - pq.txs = old[0 : n-1] - return item -} - -// Len implements the Heap interface. -// -// NOTE: A caller should never call Len. Use NumTxs instead. -func (pq *TxPriorityQueue) Len() int { - return len(pq.txs) -} - -// Less implements the Heap interface. It returns true if the transaction at -// position i in the queue is of less priority than the transaction at position j. -func (pq *TxPriorityQueue) Less(i, j int) bool { - // If there exists two transactions with the same priority, consider the one - // that we saw the earliest as the higher priority transaction. - if pq.txs[i].priority == pq.txs[j].priority { - return pq.txs[i].timestamp.Before(pq.txs[j].timestamp) - } - - // We want Pop to give us the highest, not lowest, priority so we use greater - // than here. - return pq.txs[i].priority > pq.txs[j].priority -} - -// Swap implements the Heap interface. It swaps two transactions in the queue. -func (pq *TxPriorityQueue) Swap(i, j int) { - pq.txs[i], pq.txs[j] = pq.txs[j], pq.txs[i] - setHeapIndex(pq.txs[i], i) - setHeapIndex(pq.txs[j], j) -} - -func setHeapIndex(tx *WrappedTx, i int) { - // a removed tx can be nil - if tx == nil { - return - } - tx.heapIndex = i -} diff --git a/sei-tendermint/internal/mempool/priority_queue_test.go b/sei-tendermint/internal/mempool/priority_queue_test.go deleted file mode 100644 index 4b885c4e21..0000000000 --- a/sei-tendermint/internal/mempool/priority_queue_test.go +++ /dev/null @@ -1,489 +0,0 @@ -package mempool - -import ( - "fmt" - "math/rand" - "sort" - "sync" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" - "github.com/sei-protocol/sei-chain/sei-tendermint/types" - "github.com/stretchr/testify/require" -) - -func wrappedEVMTx(tx types.Tx, address string, nonce uint64, priority int64) *WrappedTx { - return &WrappedTx{ - hashedTx: newHashedTx(tx), - priority: priority, - evm: utils.Some(evmTx{ - address: common.HexToAddress(address), - nonce: nonce, - }), - } -} - -// TxTestCase represents a single test case for the TxPriorityQueue -type TxTestCase struct { - name string - inputTxs []*WrappedTx // Input transactions - expectedOutput []int64 // Expected order of transaction IDs -} - -func TestTxPriorityQueue_ReapHalf(t *testing.T) { - pq := NewTxPriorityQueue() - - // Generate transactions with different priorities and nonces - txs := make([]*WrappedTx, 100) - for i := range txs { - txs[i] = &WrappedTx{ - hashedTx: newHashedTx(types.Tx(fmt.Sprintf("tx-%d", i))), - priority: int64(i), - } - - // Push the transaction - pq.PushTx(txs[i]) - } - - //reverse sort txs by priority - sort.Slice(txs, func(i, j int) bool { - return txs[i].priority > txs[j].priority - }) - - // Reap half of the transactions - reapedTxs := pq.PeekTxs(len(txs) / 2) - - // Check if the reaped transactions are in the correct order of their priorities and nonces - for i, reapedTx := range reapedTxs { - require.Equal(t, txs[i].priority, reapedTx.priority) - } -} - -func TestAvoidPanicIfTransactionIsNil(t *testing.T) { - pq := NewTxPriorityQueue() - pq.Push(wrappedEVMTx(nil, "0xabc", 1, 10)) - pq.txs = append(pq.txs, nil) - - var count int - pq.ForEachTx(func(tx *WrappedTx) bool { - count++ - return true - }) - - require.Equal(t, 1, count) -} - -func TestTxPriorityQueue_PriorityAndNonceOrdering(t *testing.T) { - testCases := []TxTestCase{ - { - name: "PriorityWithEVMAndNonEVMDuplicateNonce", - inputTxs: []*WrappedTx{ - wrappedEVMTx(types.Tx("1"), "0xabc", 1, 10), - wrappedEVMTx(types.Tx("3"), "0xabc", 3, 9), - wrappedEVMTx(types.Tx("2"), "0xabc", 1, 7), - }, - expectedOutput: []int64{1, 3}, - }, - { - name: "PriorityWithEVMAndNonEVMDuplicateNonce", - inputTxs: []*WrappedTx{ - wrappedEVMTx(types.Tx("1"), "0xabc", 1, 10), - {hashedTx: newHashedTx(types.Tx("2")), priority: 9}, - wrappedEVMTx(types.Tx("4"), "0xabc", 0, 9), // Same EVM address as first, lower nonce - wrappedEVMTx(types.Tx("5"), "0xdef", 1, 7), - wrappedEVMTx(types.Tx("3"), "0xdef", 0, 8), - {hashedTx: newHashedTx(types.Tx("6")), priority: 6}, - wrappedEVMTx(types.Tx("7"), "0xghi", 2, 5), - }, - expectedOutput: []int64{2, 4, 1, 3, 5, 6, 7}, - }, - { - name: "PriorityWithEVMAndNonEVM", - inputTxs: []*WrappedTx{ - wrappedEVMTx(types.Tx("1"), "0xabc", 1, 10), - {hashedTx: newHashedTx(types.Tx("2")), priority: 9}, - wrappedEVMTx(types.Tx("4"), "0xabc", 0, 9), // Same EVM address as first, lower nonce - wrappedEVMTx(types.Tx("5"), "0xdef", 1, 7), - wrappedEVMTx(types.Tx("3"), "0xdef", 0, 8), - {hashedTx: newHashedTx(types.Tx("6")), priority: 6}, - wrappedEVMTx(types.Tx("7"), "0xghi", 2, 5), - }, - expectedOutput: []int64{2, 4, 1, 3, 5, 6, 7}, - }, - { - name: "IdenticalPrioritiesAndNoncesDifferentAddresses", - inputTxs: []*WrappedTx{ - wrappedEVMTx(types.Tx("1"), "0xabc", 2, 5), - wrappedEVMTx(types.Tx("2"), "0xdef", 2, 5), - wrappedEVMTx(types.Tx("3"), "0xghi", 2, 5), - }, - expectedOutput: []int64{1, 2, 3}, - }, - { - name: "InterleavedEVAndNonEVMTransactions", - inputTxs: []*WrappedTx{ - {hashedTx: newHashedTx(types.Tx("7")), priority: 15}, - wrappedEVMTx(types.Tx("8"), "0xabc", 1, 20), - {hashedTx: newHashedTx(types.Tx("9")), priority: 10}, - wrappedEVMTx(types.Tx("10"), "0xdef", 2, 20), - }, - expectedOutput: []int64{8, 10, 7, 9}, - }, - { - name: "SameAddressPriorityDifferentNonces", - inputTxs: []*WrappedTx{ - wrappedEVMTx(types.Tx("11"), "0xabc", 3, 10), - wrappedEVMTx(types.Tx("12"), "0xabc", 1, 10), - wrappedEVMTx(types.Tx("13"), "0xabc", 2, 10), - }, - expectedOutput: []int64{12, 13, 11}, - }, - { - name: "OneItem", - inputTxs: []*WrappedTx{ - wrappedEVMTx(types.Tx("14"), "0xabc", 1, 10), - }, - expectedOutput: []int64{14}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - pq := NewTxPriorityQueue() - now := time.Now() - - // Add input transactions to the queue and set timestamp to order inserted - for i, tx := range tc.inputTxs { - tx.timestamp = now.Add(time.Duration(i) * time.Second) - pq.PushTx(tx) - } - - results := pq.PeekTxs(len(tc.inputTxs)) - // Validate the order of transactions - require.Len(t, results, len(tc.expectedOutput)) - for i, expectedTxID := range tc.expectedOutput { - tx := results[i] - require.Equal(t, fmt.Sprintf("%d", expectedTxID), string(tx.Tx())) - } - }) - } -} - -func TestTxPriorityQueue_SameAddressDifferentNonces(t *testing.T) { - pq := NewTxPriorityQueue() - address := "0x123" - - // Insert transactions with the same address but different nonces and priorities - pq.PushTx(wrappedEVMTx(types.Tx("tx1"), address, 2, 10)) - pq.PushTx(wrappedEVMTx(types.Tx("tx2"), address, 1, 5)) - pq.PushTx(wrappedEVMTx(types.Tx("tx3"), address, 3, 15)) - - // Pop transactions and verify they are in the correct order of nonce - tx1 := pq.PopTx() - evm, ok := tx1.evm.Get() - require.True(t, ok) - require.Equal(t, uint64(1), evm.nonce) - tx2 := pq.PopTx() - evm, ok = tx2.evm.Get() - require.True(t, ok) - require.Equal(t, uint64(2), evm.nonce) - tx3 := pq.PopTx() - evm, ok = tx3.evm.Get() - require.True(t, ok) - require.Equal(t, uint64(3), evm.nonce) -} - -func TestTxPriorityQueue(t *testing.T) { - pq := NewTxPriorityQueue() - numTxs := 1000 - - priorities := make([]int, numTxs) - - var wg sync.WaitGroup - for i := 1; i <= numTxs; i++ { - priorities[i-1] = i - wg.Add(1) - - go func(i int) { - pq.PushTx(&WrappedTx{ - hashedTx: newHashedTx(types.Tx(fmt.Sprintf("%d", i))), - priority: int64(i), - timestamp: time.Now(), - }) - - wg.Done() - }(i) - } - - sort.Sort(sort.Reverse(sort.IntSlice(priorities))) - - wg.Wait() - require.Equal(t, numTxs, pq.NumTxs()) - - // Wait a second and push a tx with a duplicate priority - time.Sleep(time.Second) - now := time.Now() - pq.PushTx(&WrappedTx{ - hashedTx: newHashedTx(types.Tx(fmt.Sprintf("%d", time.Now().UnixNano()))), - priority: 1000, - timestamp: now, - }) - require.Equal(t, 1001, pq.NumTxs()) - - tx := pq.PopTx() - require.Equal(t, 1000, pq.NumTxs()) - require.Equal(t, int64(1000), tx.priority) - require.NotEqual(t, now, tx.timestamp) - - gotPriorities := make([]int, 0) - for pq.NumTxs() > 0 { - gotPriorities = append(gotPriorities, int(pq.PopTx().priority)) - } - - require.Equal(t, priorities, gotPriorities) -} - -func TestTxPriorityQueue_GetEvictableTxs(t *testing.T) { - pq := NewTxPriorityQueue() - rng := rand.New(rand.NewSource(time.Now().UnixNano())) - - values := make([]int, 1000) - - for i := 0; i < 1000; i++ { - tx := make([]byte, 5) // each tx is 5 bytes - _, err := rng.Read(tx) - require.NoError(t, err) - - x := rng.Intn(100000) - pq.PushTx(&WrappedTx{ - hashedTx: newHashedTx(tx), - priority: int64(x), - }) - - values[i] = x - } - - sort.Ints(values) - - max := values[len(values)-1] - min := values[0] - totalSize := int64(len(values) * 5) - - testCases := []struct { - name string - priority, txSize, totalSize, cap int64 - expectedLen int - }{ - { - name: "larest priority; single tx", - priority: int64(max + 1), - txSize: 5, - totalSize: totalSize, - cap: totalSize, - expectedLen: 1, - }, - { - name: "larest priority; multi tx", - priority: int64(max + 1), - txSize: 17, - totalSize: totalSize, - cap: totalSize, - expectedLen: 4, - }, - { - name: "larest priority; out of capacity", - priority: int64(max + 1), - txSize: totalSize + 1, - totalSize: totalSize, - cap: totalSize, - expectedLen: 0, - }, - { - name: "smallest priority; no tx", - priority: int64(min - 1), - txSize: 5, - totalSize: totalSize, - cap: totalSize, - expectedLen: 0, - }, - { - name: "small priority; no tx", - priority: int64(min), - txSize: 5, - totalSize: totalSize, - cap: totalSize, - expectedLen: 0, - }, - } - - for _, tc := range testCases { - - t.Run(tc.name, func(t *testing.T) { - evictTxs := pq.GetEvictableTxs(tc.priority, tc.txSize, tc.totalSize, tc.cap) - require.Len(t, evictTxs, tc.expectedLen) - }) - } -} - -func TestTxPriorityQueue_RemoveTxEvm(t *testing.T) { - pq := NewTxPriorityQueue() - - tx1 := &WrappedTx{ - hashedTx: newHashedTx(types.Tx("tx1")), - priority: 1, - evm: utils.Some(evmTx{ - address: common.HexToAddress("0xabc"), - nonce: 1, - }), - } - tx2 := &WrappedTx{ - hashedTx: newHashedTx(types.Tx("tx2")), - priority: 1, - evm: utils.Some(evmTx{ - address: common.HexToAddress("0xabc"), - nonce: 2, - }), - } - - pq.PushTx(tx1) - pq.PushTx(tx2) - - pq.RemoveTx(tx1, false) - - result := pq.PopTx() - require.Equal(t, tx2, result) -} - -func TestTxPriorityQueue_RemoveTx(t *testing.T) { - pq := NewTxPriorityQueue() - rng := rand.New(rand.NewSource(time.Now().UnixNano())) - numTxs := 1000 - - values := make([]int, numTxs) - - for i := 0; i < numTxs; i++ { - x := rng.Intn(100000) - pq.PushTx(&WrappedTx{ - hashedTx: newHashedTx(types.Tx(fmt.Sprintf("%d", i))), - priority: int64(x), - }) - - values[i] = x - } - - require.Equal(t, numTxs, pq.NumTxs()) - - sort.Ints(values) - max := values[len(values)-1] - - wtx := pq.txs[pq.NumTxs()/2] - pq.RemoveTx(wtx, false) - require.Equal(t, numTxs-1, pq.NumTxs()) - require.Equal(t, int64(max), pq.PopTx().priority) - require.Equal(t, numTxs-2, pq.NumTxs()) - - require.NotPanics(t, func() { - pq.RemoveTx(&WrappedTx{heapIndex: numTxs}, false) - pq.RemoveTx(&WrappedTx{heapIndex: numTxs + 1}, false) - }) - require.Equal(t, numTxs-2, pq.NumTxs()) -} - -func TestTxPriorityQueue_TryReplacement(t *testing.T) { - for _, test := range []struct { - tx *WrappedTx - existing []*WrappedTx - expectedReplaced bool - expectedDropped bool - expectedQueue []*WrappedTx - expectedHeap []*WrappedTx - }{ - // non-evm transaction is inserted into empty queue - {&WrappedTx{}, []*WrappedTx{}, false, false, []*WrappedTx{{}}, []*WrappedTx{{}}}, - // evm transaction is inserted into empty queue - {wrappedEVMTx(nil, "addr1", 0, 0), []*WrappedTx{}, false, false, []*WrappedTx{wrappedEVMTx(nil, "addr1", 0, 0)}, []*WrappedTx{wrappedEVMTx(nil, "addr1", 0, 0)}}, - // evm transaction (new nonce) is inserted into queue with existing tx (lower nonce) - { - wrappedEVMTx(types.Tx("abc"), "addr1", 1, 100), []*WrappedTx{ - wrappedEVMTx(types.Tx("def"), "addr1", 0, 100), - }, false, false, []*WrappedTx{ - wrappedEVMTx(types.Tx("def"), "addr1", 0, 100), - wrappedEVMTx(types.Tx("abc"), "addr1", 1, 100), - }, []*WrappedTx{ - wrappedEVMTx(types.Tx("def"), "addr1", 0, 100), - wrappedEVMTx(types.Tx("abc"), "addr1", 1, 100), - }, - }, - // evm transaction (new nonce) is not inserted because it's a duplicate nonce and same priority - { - wrappedEVMTx(types.Tx("abc"), "addr1", 0, 100), []*WrappedTx{ - wrappedEVMTx(types.Tx("def"), "addr1", 0, 100), - }, false, true, []*WrappedTx{ - wrappedEVMTx(types.Tx("def"), "addr1", 0, 100), - }, []*WrappedTx{ - wrappedEVMTx(types.Tx("def"), "addr1", 0, 100), - }, - }, - // evm transaction (new nonce) replaces the existing nonce transaction because its priority is higher - { - wrappedEVMTx(types.Tx("abc"), "addr1", 0, 101), []*WrappedTx{ - wrappedEVMTx(types.Tx("def"), "addr1", 0, 100), - }, true, false, []*WrappedTx{ - wrappedEVMTx(types.Tx("abc"), "addr1", 0, 101), - }, []*WrappedTx{ - wrappedEVMTx(types.Tx("abc"), "addr1", 0, 101), - }, - }, - { - wrappedEVMTx(types.Tx("abc"), "addr1", 1, 100), []*WrappedTx{ - wrappedEVMTx(types.Tx("def"), "addr1", 0, 100), - wrappedEVMTx(types.Tx("ghi"), "addr1", 1, 99), - }, true, false, []*WrappedTx{ - wrappedEVMTx(types.Tx("def"), "addr1", 0, 100), - wrappedEVMTx(types.Tx("abc"), "addr1", 1, 100), - }, []*WrappedTx{ - wrappedEVMTx(types.Tx("def"), "addr1", 0, 100), - }, - }, - } { - pq := NewTxPriorityQueue() - for _, e := range test.existing { - pq.PushTx(e) - } - replaced, inserted := pq.PushTx(test.tx) - if test.expectedReplaced { - require.NotNil(t, replaced) - } else { - require.Nil(t, replaced) - } - require.Equal(t, test.expectedDropped, !inserted) - txEVM, ok := test.tx.evm.Get() - if !ok { - require.Empty(t, pq.evmQueue) - continue - } - for i, q := range pq.evmQueue[txEVM.address] { - require.Equal(t, test.expectedQueue[i].Hash(), q.Hash()) - require.Equal(t, test.expectedQueue[i].priority, q.priority) - expectedEVM, ok := test.expectedQueue[i].evm.Get() - require.True(t, ok) - queueEVM, ok := q.evm.Get() - require.True(t, ok) - require.Equal(t, expectedEVM.nonce, queueEVM.nonce) - } - for i, q := range pq.txs { - require.Equal(t, test.expectedHeap[i].Hash(), q.Hash()) - require.Equal(t, test.expectedHeap[i].priority, q.priority) - expectedEVM, ok := test.expectedHeap[i].evm.Get() - if ok { - queueEVM, ok := q.evm.Get() - require.True(t, ok) - require.Equal(t, expectedEVM.nonce, queueEVM.nonce) - } else { - require.False(t, q.evm.IsPresent()) - } - } - } -} diff --git a/sei-tendermint/internal/mempool/reactor/reactor.go b/sei-tendermint/internal/mempool/reactor/reactor.go index ecb23cad98..172a59cef8 100644 --- a/sei-tendermint/internal/mempool/reactor/reactor.go +++ b/sei-tendermint/internal/mempool/reactor/reactor.go @@ -242,21 +242,18 @@ func (r *Reactor) processPeerUpdates(ctx context.Context) error { } func (r *Reactor) broadcastTxRoutine(ctx context.Context, peerID types.NodeID) { - peerMempoolID := r.ids.GetForPeer(peerID) for { - next, err := r.mempool.WaitForNextTx(ctx) + next, err := r.mempool.WaitForReadyTx(ctx) if err != nil { return } for { memTx := next.Value() - if ok := r.mempool.TxStore().TxHasPeer(memTx.Hash(), peerMempoolID); !ok { - r.channel.Send(&pb.Message{ - Sum: &pb.Message_Txs{ - Txs: &pb.Txs{Txs: [][]byte{memTx.Tx()}}, - }, - }, peerID) - } + r.channel.Send(&pb.Message{ + Sum: &pb.Message_Txs{ + Txs: &pb.Txs{Txs: [][]byte{memTx.Tx()}}, + }, + }, peerID) next, err = next.NextWait(ctx) if err != nil { diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 184062de2a..cd7a343c92 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -327,7 +327,7 @@ func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { type updateSpec struct { Now time.Time Height int64 - TxsToRemove map[types.TxHash]struct{} + ToRemove map[types.TxHash]struct{} Constraints TxConstraints NewPriorities map[types.TxHash]int64 } @@ -343,7 +343,7 @@ func (txs *txStore) Update(spec updateSpec) { } for inner := range txs.inner.Lock() { toRemove := func(wtx *WrappedTx) bool { - if _,ok := spec.TxsToRemove[wtx.Hash()]; ok { + if _,ok := spec.ToRemove[wtx.Hash()]; ok { return true } if wtx.check(spec.Constraints) != nil { From 4635fcad8797356c87b4943ebb8a6a980758b162 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 19 May 2026 15:53:40 +0200 Subject: [PATCH 006/100] clist --- sei-tendermint/internal/evidence/pool.go | 1 - .../internal/libs/clist/bench_test.go | 18 ----- sei-tendermint/internal/libs/clist/clist.go | 46 ++--------- sei-tendermint/internal/mempool/mempool.go | 1 - sei-tendermint/internal/mempool/tx.go | 81 +++++++++---------- 5 files changed, 45 insertions(+), 102 deletions(-) diff --git a/sei-tendermint/internal/evidence/pool.go b/sei-tendermint/internal/evidence/pool.go index 550b0a420f..fcf98d185d 100644 --- a/sei-tendermint/internal/evidence/pool.go +++ b/sei-tendermint/internal/evidence/pool.go @@ -534,7 +534,6 @@ func (evpool *Pool) removeEvidenceFromList( ev := e.Value() if _, ok := blockEvidenceMap[evMapKey(ev)]; ok { evpool.evidenceList.Remove(e) - e.DetachPrev() } } } diff --git a/sei-tendermint/internal/libs/clist/bench_test.go b/sei-tendermint/internal/libs/clist/bench_test.go index 8a5ab3699f..54f8ac64f1 100644 --- a/sei-tendermint/internal/libs/clist/bench_test.go +++ b/sei-tendermint/internal/libs/clist/bench_test.go @@ -2,24 +2,6 @@ package clist import "testing" -func BenchmarkDetaching(b *testing.B) { - lst := New[int]() - for i := 0; i < b.N+1; i++ { - lst.PushBack(i) - } - start := lst.Front() - nxt := start.Next() - b.ResetTimer() - for i := 0; i < b.N; i++ { - start.removed = true - start.detachNext() - start.DetachPrev() - tmp := nxt - nxt = nxt.Next() - start = tmp - } -} - // This is used to benchmark the time of RMutex. func BenchmarkRemoved(b *testing.B) { lst := New[int]() diff --git a/sei-tendermint/internal/libs/clist/clist.go b/sei-tendermint/internal/libs/clist/clist.go index 7c026077e4..7307c0b497 100644 --- a/sei-tendermint/internal/libs/clist/clist.go +++ b/sei-tendermint/internal/libs/clist/clist.go @@ -77,63 +77,34 @@ func (e *CElement[T]) NextWait(ctx context.Context) (*CElement[T], error) { // Nonblocking, may return nil if at the end. func (e *CElement[T]) Next() *CElement[T] { e.mtx.RLock() - val := e.next - e.mtx.RUnlock() - return val + defer e.mtx.RUnlock() + return e.next } // Nonblocking, may return nil if at the end. func (e *CElement[T]) Prev() *CElement[T] { e.mtx.RLock() - prev := e.prev - e.mtx.RUnlock() - return prev + defer e.mtx.RUnlock() + return e.prev } func (e *CElement[T]) Removed() bool { e.mtx.RLock() - isRemoved := e.removed - e.mtx.RUnlock() - return isRemoved + defer e.mtx.RUnlock() + return e.removed } func (e *CElement[T]) Value() T { return e.value } -func (e *CElement[T]) detachNext() { - e.mtx.Lock() - if !e.removed { - e.mtx.Unlock() - panic("DetachNext() must be called after Remove(e)") - } - e.next = nil - e.mtx.Unlock() -} - -func (e *CElement[T]) DetachPrev() { - e.mtx.Lock() - if !e.removed { - e.mtx.Unlock() - panic("DetachPrev() must be called after Remove(e)") - } - e.prev = nil - e.mtx.Unlock() -} - // NOTE: This function needs to be safe for // concurrent goroutines waiting on nextWg. func (e *CElement[T]) setNext(newNext *CElement[T]) { e.mtx.Lock() - oldNext := e.next e.next = newNext if oldNext != nil && newNext == nil { - // See https://golang.org/pkg/sync/: - // - // If a WaitGroup is reused to wait for several independent sets of - // events, new Add calls must happen after all previous Wait calls have - // returned. e.nextWaitCh = make(chan struct{}) } if oldNext == nil && newNext != nil { @@ -154,13 +125,12 @@ func (e *CElement[T]) setPrev(newPrev *CElement[T]) { func (e *CElement[T]) setRemoved() { e.mtx.Lock() defer e.mtx.Unlock() - - e.removed = true - // This wakes up anyone waiting. if e.next == nil { close(e.nextWaitCh) } + e.prev = nil + e.removed = true } //-------------------------------------------------------------------------------- diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index c13e14da64..433b49eaa0 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -468,7 +468,6 @@ func (txmp *TxMempool) ReapMaxBytesMaxGas(maxBytes, maxGasWanted, maxGasEstimate MaxGasWanted: utils.Some(maxGasWanted), MaxGasEstimated: utils.Some(maxGasEstimated), }) - // TODO: first evm txs, then non-evm txs return txs } diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index cd7a343c92..899c1951bb 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -41,7 +41,6 @@ type WrappedTx struct { estimatedGas int64 // estimatedGas defines the amount of gas that the transaction is estimated to use priority int64 // ResponseCheckTx.priority timestamp time.Time // time at which the transaction was received - readyEl utils.Option[*clist.CElement[*WrappedTx]] // linked-list element in the gossip index evm utils.Option[evmTx] // evm transaction info } @@ -122,7 +121,7 @@ type txStore struct { inner utils.RWMutex[*txStoreInner] state utils.AtomicRecv[txStoreState] // list of ready transactions that can be gossiped. - readyTxs *clist.CList[*WrappedTx] + readyTxs *clist.CList[types.Tx] } func NewTxStore(config *Config) *txStore { @@ -138,7 +137,7 @@ func NewTxStore(config *Config) *txStore { return &txStore{ config: config, inner: utils.NewRWMutex(inner), - readyTxs: clist.New[*WrappedTx](), + readyTxs: clist.New[types.Tx](), state: inner.state.Subscribe(), } } @@ -189,8 +188,8 @@ func (txs *txStore) ByHash(key types.TxHash) *WrappedTx { panic("unreachable") } -func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) { - if _,ok := inner.byHash[wtx.Hash()]; ok { return } +func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) bool { + if _,ok := inner.byHash[wtx.Hash()]; ok { return false } state := inner.state.Load() if evm,ok := wtx.evm.Get(); ok { // Fetch the evm account state. @@ -204,21 +203,19 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) { } // Reject transactions with old nonces. if evm.nonce < account.firstNonce { - return + return false } an := evmAddrNonce{evm.address,evm.nonce} if old,ok := inner.byNonce[an]; ok { // If the old tx is ready but the new tx is not, then reject the new tx. if old.evm.OrPanic("non-evm tx").nonce < account.nextNonce && account.balance.Cmp(evm.requiredBalance) < 0 { - return + return false } // If the old tx has >= priority, then reject new tx. - if old.priority >= wtx.priority { return } + if old.priority >= wtx.priority { return false } // Remove the old transaction. delete(inner.byHash,old.Hash()) - if el,ok := wtx.readyEl.Get(); ok { - txs.readyTxs.Remove(el) - } + txs.readyTxs.Remove(wtx) state.ready.Dec(old.Size()) state.total.Dec(old.Size()) state.ready.Inc(wtx.Size()) @@ -239,9 +236,10 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) { state.total.Inc(wtx.Size()) inner.byHash[wtx.Hash()] = wtx if !wtx.readyEl.IsPresent() { - wtx.readyEl = utils.Some(txs.readyTxs.PushBack(wtx)) + wtx.readyEl = utils.Some(txs.readyTxs.PushBack(wtx.Tx())) } - inner.state.Store(state) + inner.state.Store(state) + return true } // WARNING: works only if wtx has been already inserted. @@ -403,40 +401,35 @@ func (txs *txStore) ReapTxs(l ReapLimits) (types.Txs, int64) { totalGasEstimated := int64(0) totalSize := int64(0) + var wtxs []*WrappedTx for inner := range txs.inner.Lock() { - if uint64(inner.state.Load().ready.count) < txs.config.TxNotifyThreshold { //nolint:gosec - // do not reap anything if threshold is not met - return types.Txs{}, 0 - } - var txs []types.Tx - encounteredGasUnfit := false - for _,wtx := range inner.inInclusionOrder() { - // bytes limit is a hard stop - if wtx.protoSize > maxBytes-totalSize || uint64(len(txs)) >= maxTxs { - break - } - limitExceeded := (maxGasWanted - totalGasWanted < wtx.gasWanted) || - (maxGasEstimated - totalGasEstimated < wtx.estimatedGas) - - if limitExceeded { - // skip this unfit-by-gas tx once and attempt to pull up to 10 smaller ones - if !encounteredGasUnfit && len(txs) < MinTxsToPeek { - encounteredGasUnfit = true - continue + if uint64(inner.state.Load().ready.count) >= txs.config.TxNotifyThreshold { + for _,wtx := range inner.inInclusionOrder() { + if wtx.protoSize > maxBytes-totalSize || uint64(len(wtxs)) >= maxTxs { + break } - break - } - - // include tx and update totals - totalSize += wtx.protoSize - totalGasWanted += wtx.gasWanted - totalGasEstimated += wtx.estimatedGas - txs = append(txs, wtx.Tx()) - if encounteredGasUnfit && len(txs) >= MinTxsToPeek { - break + if maxGasWanted - totalGasWanted < wtx.gasWanted { + break + } + if maxGasEstimated - totalGasEstimated < wtx.estimatedGas { + break + } + // include tx and update totals + totalSize += wtx.protoSize + totalGasWanted += wtx.gasWanted + totalGasEstimated += wtx.estimatedGas + wtxs = append(wtxs, wtx) } } - return txs, totalGasEstimated } - panic("unreachable") + // EVM txs go first. + var evmTxs,nonEvmTxs types.Txs + for _,wtx := range wtxs { + if wtx.evm.IsPresent() { + evmTxs = append(evmTxs, wtx.Tx()) + } else { + nonEvmTxs = append(nonEvmTxs, wtx.Tx()) + } + } + return append(evmTxs,nonEvmTxs...), totalGasEstimated } From 13254e5186c7b9af5a60222f98b1ced2e63d2a35 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 19 May 2026 16:06:01 +0200 Subject: [PATCH 007/100] before implementing PopTxs --- sei-tendermint/internal/mempool/mempool.go | 2 +- sei-tendermint/internal/mempool/reactor/reactor.go | 12 +++--------- sei-tendermint/internal/mempool/testonly.go | 10 ++++++++-- sei-tendermint/internal/mempool/tx.go | 6 ++++-- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 433b49eaa0..83bbec1795 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -258,7 +258,7 @@ func (txmp *TxMempool) utilisation() float64 { // WaitForNextTx waits until the next transaction is available for gossip. // Returns the next valid transaction to gossip. -func (txmp *TxMempool) WaitForReadyTx(ctx context.Context) (*clist.CElement[*WrappedTx], error) { +func (txmp *TxMempool) WaitForReadyTx(ctx context.Context) (*clist.CElement[types.Tx], error) { return txmp.txStore.readyTxs.WaitFront(ctx) } diff --git a/sei-tendermint/internal/mempool/reactor/reactor.go b/sei-tendermint/internal/mempool/reactor/reactor.go index 172a59cef8..0f1b2e9c1a 100644 --- a/sei-tendermint/internal/mempool/reactor/reactor.go +++ b/sei-tendermint/internal/mempool/reactor/reactor.go @@ -113,14 +113,8 @@ func (r *Reactor) handleMempoolMessage(ctx context.Context, m p2p.RecvMsg[*pb.Me return err } protoTxs := msg.Txs.GetTxs() - - txInfo := mempool.TxInfo{SenderID: r.ids.GetForPeer(m.From)} - if len(m.From) != 0 { - txInfo.SenderNodeID = m.From - } - for _, tx := range protoTxs { - if _, err := r.mempool.CheckTx(ctx, tx, txInfo); err != nil { + if _, err := r.mempool.CheckTx(ctx, tx); err != nil { r.accountFailedCheckTx(m.From, err) if errors.Is(err, mempool.ErrTxInCache) { // If the tx is in the cache, then we've been gossiped a tx @@ -248,10 +242,10 @@ func (r *Reactor) broadcastTxRoutine(ctx context.Context, peerID types.NodeID) { return } for { - memTx := next.Value() + tx := next.Value() r.channel.Send(&pb.Message{ Sum: &pb.Message_Txs{ - Txs: &pb.Txs{Txs: [][]byte{memTx.Tx()}}, + Txs: &pb.Txs{Txs: [][]byte{tx}}, }, }, peerID) diff --git a/sei-tendermint/internal/mempool/testonly.go b/sei-tendermint/internal/mempool/testonly.go index b351390813..30132013c8 100644 --- a/sei-tendermint/internal/mempool/testonly.go +++ b/sei-tendermint/internal/mempool/testonly.go @@ -1,11 +1,17 @@ package mempool +import ( + "time" + + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" +) + func TestConfig() *Config { cfg := DefaultConfig() cfg.CacheSize = 1000 cfg.DropUtilisationThreshold = 0.0 // Disable TTL purging in tests. - cfg.TTLNumBlocks = 0 - cfg.TTLDuration = 0 + cfg.TTLNumBlocks = utils.None[int64]() + cfg.TTLDuration = utils.None[time.Duration]() return cfg } diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 899c1951bb..16648e3500 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -42,6 +42,7 @@ type WrappedTx struct { priority int64 // ResponseCheckTx.priority timestamp time.Time // time at which the transaction was received evm utils.Option[evmTx] // evm transaction info + readyEl utils.Option[*clist.CElement[types.Tx]] } func (wtx *WrappedTx) check(c TxConstraints) error { @@ -120,7 +121,6 @@ type txStore struct { app *proxy.Proxy inner utils.RWMutex[*txStoreInner] state utils.AtomicRecv[txStoreState] - // list of ready transactions that can be gossiped. readyTxs *clist.CList[types.Tx] } @@ -215,7 +215,9 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) bool { if old.priority >= wtx.priority { return false } // Remove the old transaction. delete(inner.byHash,old.Hash()) - txs.readyTxs.Remove(wtx) + if el,ok := wtx.readyEl.Get(); ok { + txs.readyTxs.Remove(el) + } state.ready.Dec(old.Size()) state.total.Dec(old.Size()) state.ready.Inc(wtx.Size()) From 750c82a71b4cdfb33cb08eb5b477d4cee0462bf8 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 19 May 2026 17:04:48 +0200 Subject: [PATCH 008/100] reap marker --- .../internal/autobahn/producer/state.go | 7 +-- sei-tendermint/internal/consensus/state.go | 2 +- sei-tendermint/internal/mempool/mempool.go | 24 ++++----- sei-tendermint/internal/mempool/tx.go | 50 +++++++++++++++---- sei-tendermint/internal/state/execution.go | 16 +----- 5 files changed, 54 insertions(+), 45 deletions(-) diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index f5c7ee088b..a0b17ef827 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -73,7 +73,7 @@ func (s *State) makePayload(ctx context.Context) (*types.Payload, error) { return nil, err } - txs, gasEstimated := s.txMempool.PopTxs(mempool.ReapLimits{ + txs, gasEstimated := s.txMempool.ReapTxsAndMark(mempool.ReapLimits{ MaxTxs: utils.Some(min(types.MaxTxsPerBlock, s.cfg.maxTxsPerBlock())), MaxBytes: utils.Some(utils.Clamp[int64](types.MaxTxsBytesPerBlock)), MaxGasWanted: utils.Some(s.cfg.MaxGasPerBlockI64()), @@ -85,10 +85,7 @@ func (s *State) makePayload(ctx context.Context) (*types.Payload, error) { } payload, err := types.PayloadBuilder{ CreatedAt: time.Now(), - // TODO: ReapMaxTxsBytesMaxGas does not handle corner cases correctly rn, which actually - // can produce negative total gas. Fixing it right away might be backward incompatible afaict, - // so we leave it as is for now. - TotalGas: uint64(gasEstimated), // nolint:gosec + TotalGas: uint64(gasEstimated), // nolint:gosec // always non-negative Txs: payloadTxs, }.Build() // This should never happen: we construct the payload from correctly sized data. diff --git a/sei-tendermint/internal/consensus/state.go b/sei-tendermint/internal/consensus/state.go index 825dd9b949..ae03eb74e7 100644 --- a/sei-tendermint/internal/consensus/state.go +++ b/sei-tendermint/internal/consensus/state.go @@ -2275,7 +2275,7 @@ func (cs *State) buildProposalBlock(proposal *types.Proposal) *types.Block { txs, missingTxs := cs.blockExec.SafeGetTxsByHashes(proposal.TxHashes) if len(missingTxs) > 0 { cs.metrics.ProposalMissingTxs.Set(float64(len(missingTxs))) - logger.Debug("Missing txs when trying to build block", "missing_txs", cs.blockExec.GetMissingTxs(proposal.TxHashes)) + logger.Debug("Missing txs when trying to build block", "missing_txs", missingTxs) return nil } block := cs.state.MakeBlock(proposal.Height, txs, proposal.LastCommit, proposal.Evidence, proposal.ProposerAddress) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 83bbec1795..c269deebe1 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -406,18 +406,6 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response return res.ResponseCheckTx, nil } -func (txmp *TxMempool) GetTxsForHashes(txHashes []types.TxHash) types.Txs { - txmp.mtx.RLock() - defer txmp.mtx.RUnlock() - - txs := make([]types.Tx, 0, len(txHashes)) - for _, txHash := range txHashes { - wtx := txmp.txStore.ByHash(txHash) - txs = append(txs, wtx.Tx()) - } - return txs -} - func (txmp *TxMempool) SafeGetTxsForHashes(txHashes []types.TxHash) (types.Txs, []types.TxHash) { txmp.mtx.RLock() defer txmp.mtx.RUnlock() @@ -460,17 +448,23 @@ func (txmp *TxMempool) Flush() { // NOTE: // - Transactions returned are not removed from the mempool transaction // store or indexes. -func (txmp *TxMempool) ReapMaxBytesMaxGas(maxBytes, maxGasWanted, maxGasEstimated int64) types.Txs { +func (txmp *TxMempool) ReapMaxBytesMaxGas(height int64, maxBytes, maxGasWanted, maxGasEstimated int64) types.Txs { txmp.mtx.Lock() defer txmp.mtx.Unlock() - txs, _ := txmp.txStore.ReapTxs(ReapLimits{ + txs, _ := txmp.txStore.Reap(ReapLimits{ MaxBytes: utils.Some(maxBytes), MaxGasWanted: utils.Some(maxGasWanted), MaxGasEstimated: utils.Some(maxGasEstimated), - }) + }, false) return txs } +func (txmp *TxMempool) ReapTxsAndMark(limits ReapLimits) (types.Txs,int64) { + txmp.mtx.Lock() + defer txmp.mtx.Unlock() + return txmp.txStore.Reap(limits, true) +} + // Update iterates over all the transactions provided by the block producer, // removes them from the cache (if applicable), and removes // the transactions from the main transaction store and associated indexes. diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 16648e3500..e0b227405b 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -42,7 +42,9 @@ type WrappedTx struct { priority int64 // ResponseCheckTx.priority timestamp time.Time // time at which the transaction was received evm utils.Option[evmTx] // evm transaction info + readyEl utils.Option[*clist.CElement[types.Tx]] + reaped bool } func (wtx *WrappedTx) check(c TxConstraints) error { @@ -222,6 +224,7 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) bool { state.total.Dec(old.Size()) state.ready.Inc(wtx.Size()) } + state.total.Inc(wtx.Size()) inner.byNonce[an] = wtx // Update account ready txs. for { @@ -230,16 +233,19 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) bool { if !ok || account.balance.Cmp(wtx.evm.OrPanic("non-evm tx").requiredBalance) < 0 { break } account.nextNonce += 1 state.ready.Inc(wtx.Size()) + if !wtx.readyEl.IsPresent() { + wtx.readyEl = utils.Some(txs.readyTxs.PushBack(wtx.Tx())) + } } } else { // Non-evm txs are automatically ready + state.total.Inc(wtx.Size()) state.ready.Inc(wtx.Size()) + if !wtx.readyEl.IsPresent() { + wtx.readyEl = utils.Some(txs.readyTxs.PushBack(wtx.Tx())) + } } - state.total.Inc(wtx.Size()) inner.byHash[wtx.Hash()] = wtx - if !wtx.readyEl.IsPresent() { - wtx.readyEl = utils.Some(txs.readyTxs.PushBack(wtx.Tx())) - } inner.state.Store(state) return true } @@ -300,7 +306,9 @@ func (txs *txStore) Insert(wtx *WrappedTx) { } } +// O(m log m) func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { + // Order all txs by priority. wtxs := inner.inInclusionOrder() inner.state.Store(txStoreState{}) inner.byHash = map[types.TxHash]*WrappedTx{} @@ -346,6 +354,10 @@ func (txs *txStore) Update(spec updateSpec) { if _,ok := spec.ToRemove[wtx.Hash()]; ok { return true } + if wtx.reaped { + // If we already reaped the transaction, we shouldn't lose track of it. + return false + } if wtx.check(spec.Constraints) != nil { return true } @@ -367,8 +379,10 @@ func (txs *txStore) Update(spec updateSpec) { if el,ok := wtx.readyEl.Get(); ok { txs.readyTxs.Remove(el) } - } else if newPriority,ok := spec.NewPriorities[wtx.Hash()]; ok { - wtx.priority = newPriority + } else { + if newPriority,ok := spec.NewPriorities[wtx.Hash()]; ok { + wtx.priority = newPriority + } } } txs.compact(inner,true) @@ -382,10 +396,11 @@ type ReapLimits struct { MaxGasEstimated utils.Option[int64] } -// ReapTxs returns a list of transactions within the provided tx, +// Reap returns a list of transactions within the provided tx, // byte, and gas constraints together with the total estimated gas for the // returned transactions. -func (txs *txStore) ReapTxs(l ReapLimits) (types.Txs, int64) { +// O(m log m) where m is the size of the txStore. +func (txs *txStore) Reap(l ReapLimits, markReaped bool) (types.Txs, int64) { maxTxs := l.MaxTxs.Or(utils.Max[uint64]()) maxBytes := l.MaxBytes.Or(utils.Max[int64]()) maxGasWanted := l.MaxGasWanted.Or(utils.Max[int64]()) @@ -407,7 +422,23 @@ func (txs *txStore) ReapTxs(l ReapLimits) (types.Txs, int64) { for inner := range txs.inner.Lock() { if uint64(inner.state.Load().ready.count) >= txs.config.TxNotifyThreshold { for _,wtx := range inner.inInclusionOrder() { - if wtx.protoSize > maxBytes-totalSize || uint64(len(wtxs)) >= maxTxs { + // Transactions are reaped to be included in a block at a particular height. + // In case of tendermint, txs are not reaped "in advance" - before the next block is proposed, + // the previous one needs to be finalized. + // In case of autobahn Reap and Update are called asynchronously, because execution is async. + // Consecutive calls to Reap should NOT return the same txs. + // Also in autobahn we have a guarantee that reaped transactions will be included, because + // every producer builds their blocks unanonimously, therefore reaped transactions will be eventually + // removed (once sequenced). + // TODO(gprusak): this is a weak constract between autobahn and mempool and may lead to mempool capacity + // leakage if violated. Redesign later. + if wtx.reaped { + continue + } + if uint64(len(wtxs)) >= maxTxs || !inner.isReady(wtx) { + break + } + if maxBytes - totalSize < wtx.protoSize { break } if maxGasWanted - totalGasWanted < wtx.gasWanted { @@ -417,6 +448,7 @@ func (txs *txStore) ReapTxs(l ReapLimits) (types.Txs, int64) { break } // include tx and update totals + wtx.reaped = markReaped totalSize += wtx.protoSize totalGasWanted += wtx.gasWanted totalGasEstimated += wtx.estimatedGas diff --git a/sei-tendermint/internal/state/execution.go b/sei-tendermint/internal/state/execution.go index 8f1c8bcce8..f799141386 100644 --- a/sei-tendermint/internal/state/execution.go +++ b/sei-tendermint/internal/state/execution.go @@ -122,15 +122,11 @@ func (blockExec *BlockExecutor) CreateProposalBlock( // Fetch a limited amount of valid txs maxDataBytes := types.MaxDataBytes(maxBytes, evSize, state.Validators.Size()) - txs := blockExec.mempool.ReapMaxBytesMaxGas(maxDataBytes, maxGasWanted, maxGas) + txs := blockExec.mempool.ReapMaxBytesMaxGas(height, maxDataBytes, maxGasWanted, maxGas) block = state.MakeBlock(height, txs, lastCommit, evidence, proposerAddr) return block, nil } -func (blockExec *BlockExecutor) GetTxsForHashes(txHashes []types.TxHash) types.Txs { - return blockExec.mempool.GetTxsForHashes(txHashes) -} - func (blockExec *BlockExecutor) ProcessProposal( ctx context.Context, block *types.Block, @@ -492,16 +488,6 @@ func (blockExec *BlockExecutor) Commit( return res.RetainHeight, err } -func (blockExec *BlockExecutor) GetMissingTxs(txHashes []types.TxHash) []types.TxHash { - var missingTxHashes []types.TxHash - for _, txHash := range txHashes { - if !blockExec.mempool.HasTx(txHash) { - missingTxHashes = append(missingTxHashes, txHash) - } - } - return missingTxHashes -} - func (blockExec *BlockExecutor) SafeGetTxsByHashes(txHashes []types.TxHash) (types.Txs, []types.TxHash) { return blockExec.mempool.SafeGetTxsForHashes(txHashes) } From 6bf90a752a370dad1a047b27e9f61069ddc957e4 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 19 May 2026 17:12:13 +0200 Subject: [PATCH 009/100] prod code compiles --- sei-tendermint/config/config.go | 18 +- sei-tendermint/internal/mempool/mempool.go | 69 +++---- .../internal/mempool/reactor/ids.go | 87 -------- .../internal/mempool/reactor/ids_test.go | 89 -------- .../internal/mempool/reactor/reactor.go | 4 - sei-tendermint/internal/mempool/tx.go | 192 +++++++++--------- sei-tendermint/internal/mempool/types.go | 8 +- sei-tendermint/internal/p2p/giga_router.go | 2 +- 8 files changed, 153 insertions(+), 316 deletions(-) delete mode 100644 sei-tendermint/internal/mempool/reactor/ids.go delete mode 100644 sei-tendermint/internal/mempool/reactor/ids_test.go diff --git a/sei-tendermint/config/config.go b/sei-tendermint/config/config.go index 99b3b72667..593d009a5e 100644 --- a/sei-tendermint/config/config.go +++ b/sei-tendermint/config/config.go @@ -14,6 +14,7 @@ import ( mempoolcfg "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" tmos "github.com/sei-protocol/sei-chain/sei-tendermint/libs/os" "github.com/sei-protocol/sei-chain/sei-tendermint/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" ) const ( @@ -859,16 +860,14 @@ type MempoolConfig struct { DropPriorityReservoirSize int `mapstructure:"drop-priority-reservoir-size"` } -func (cfg *MempoolConfig) ToMempoolConfig() *mempoolcfg.Config { - return &mempoolcfg.Config{ +func (cfg *MempoolConfig) ToMempoolConfig() *mempoolcfg.Config { + mcfg := &mempoolcfg.Config{ Size: cfg.Size, MaxTxsBytes: cfg.MaxTxsBytes, CacheSize: cfg.CacheSize, DuplicateTxsCacheSize: cfg.DuplicateTxsCacheSize, KeepInvalidTxsInCache: cfg.KeepInvalidTxsInCache, MaxTxBytes: cfg.MaxTxBytes, - TTLDuration: cfg.TTLDuration, - TTLNumBlocks: cfg.TTLNumBlocks, TxNotifyThreshold: cfg.TxNotifyThreshold, PendingSize: cfg.PendingSize, MaxPendingTxsBytes: cfg.MaxPendingTxsBytes, @@ -877,6 +876,13 @@ func (cfg *MempoolConfig) ToMempoolConfig() *mempoolcfg.Config { DropUtilisationThreshold: cfg.DropUtilisationThreshold, DropPriorityReservoirSize: cfg.DropPriorityReservoirSize, } + if cfg.TTLDuration != 0 { + mcfg.TTLDuration = utils.Some(cfg.TTLDuration) + } + if cfg.TTLNumBlocks != 0 { + mcfg.TTLNumBlocks = utils.Some(cfg.TTLNumBlocks) + } + return mcfg } // DefaultMempoolConfig returns a default configuration for the Tendermint mempool. @@ -891,8 +897,8 @@ func DefaultMempoolConfig() *MempoolConfig { KeepInvalidTxsInCache: cfg.KeepInvalidTxsInCache, MaxTxBytes: cfg.MaxTxBytes, MaxBatchBytes: 0, - TTLDuration: cfg.TTLDuration, - TTLNumBlocks: cfg.TTLNumBlocks, + TTLDuration: cfg.TTLDuration.Or(0), + TTLNumBlocks: cfg.TTLNumBlocks.Or(0), TxNotifyThreshold: cfg.TxNotifyThreshold, CheckTxErrorBlacklistEnabled: true, CheckTxErrorThreshold: 50, diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index c269deebe1..6353c8c8de 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -70,9 +70,9 @@ type Config struct { MaxTxBytes int // time after which transaction is removed from mempool. - TTLDuration utils.Option[time.Duration] - - // number of blocks after which a transaction is removed from mempool. + TTLDuration utils.Option[time.Duration] + + // number of blocks after which a transaction is removed from mempool. TTLNumBlocks utils.Option[int64] // TxNotifyThreshold, if non-zero, defines the minimum number of transactions @@ -136,9 +136,9 @@ func DefaultConfig() *Config { MaxTxsBytes: 1024 * 1024 * 1024, // 1GB CacheSize: 10000, DuplicateTxsCacheSize: 100000, - MaxTxBytes: 1024 * 1024, // 1MB + MaxTxBytes: 1024 * 1024, // 1MB TTLDuration: utils.Some(5 * time.Second), // prevent stale txs from filling mempool - TTLNumBlocks: utils.Some(int64(10)), // remove txs after 10 blocks + TTLNumBlocks: utils.Some(int64(10)), // remove txs after 10 blocks TxNotifyThreshold: 0, PendingSize: 5000, MaxPendingTxsBytes: 1024 * 1024 * 1024, // 1GB @@ -177,7 +177,7 @@ type TxMempool struct { // blockFailedTxs tracks tx hashes that have previously failed during // block execution. Used to prevent infinite re-entry of txs that // consistently fail before fee charging in DeliverTx. - blockFailedTxs *LRUTxCache + blockFailedTxs *LRUTxCache // A TTL cache which keeps all txs that we have seen before over the TTL window. // Currently, this can be used for tracking whether checkTx is always serving the same tx or not. @@ -197,12 +197,12 @@ type TxMempool struct { priorityReservoir *reservoir.Sampler[int64] } -func (txmp *TxMempool) SizeBytes() uint64 { return txmp.txStore.State().total.bytes } -func (txmp *TxMempool) NumTxsNotPending() int { return txmp.txStore.State().ready.count } -func (txmp *TxMempool) BytesNotPending() uint64 { return txmp.txStore.State().ready.bytes } +func (txmp *TxMempool) SizeBytes() uint64 { return txmp.txStore.State().total.bytes } +func (txmp *TxMempool) NumTxsNotPending() int { return txmp.txStore.State().ready.count } +func (txmp *TxMempool) BytesNotPending() uint64 { return txmp.txStore.State().ready.bytes } func (txmp *TxMempool) TotalTxsBytesSize() uint64 { return txmp.txStore.State().total.bytes } -func (txmp *TxMempool) PendingSize() int { return txmp.txStore.State().PendingCount() } -func (txmp *TxMempool) PendingSizeBytes() uint64 { return txmp.txStore.State().PendingBytes() } +func (txmp *TxMempool) PendingSize() int { return txmp.txStore.State().PendingCount() } +func (txmp *TxMempool) PendingSizeBytes() uint64 { return txmp.txStore.State().PendingBytes() } func NewTxMempool( cfg *Config, @@ -219,8 +219,8 @@ func NewTxMempool( txStore: NewTxStore(cfg), txConstraintsFetcher: txConstraintsFetcher, priorityReservoir: reservoir.New[int64](cfg.DropPriorityReservoirSize, cfg.DropPriorityThreshold, nil), // Use non-deterministic RNG - cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), - blockFailedTxs: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), + cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), + blockFailedTxs: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), } if cfg.DuplicateTxsCacheSize > 0 { @@ -230,7 +230,7 @@ func NewTxMempool( return txmp } -func (txmp *TxMempool) Config() *Config { return txmp.config } +func (txmp *TxMempool) Config() *Config { return txmp.config } func (txmp *TxMempool) App() *proxy.Proxy { return txmp.app } func (txmp *TxMempool) EvmNextPendingNonce(addr common.Address) uint64 { return txmp.txStore.NextNonce(addr) @@ -255,7 +255,6 @@ func (txmp *TxMempool) utilisation() float64 { return float64(txmp.NumTxsNotPending()) / float64(txmp.config.Size) } - // WaitForNextTx waits until the next transaction is available for gossip. // Returns the next valid transaction to gossip. func (txmp *TxMempool) WaitForReadyTx(ctx context.Context) (*clist.CElement[types.Tx], error) { @@ -327,7 +326,9 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response // We add the transaction to the mempool's cache and if the // transaction is already present in the cache, i.e. false is returned, then we // check if we've seen this transaction and error if we have. - if !txmp.cache.Push(hTx.Hash()) { return nil, ErrTxInCache } + if !txmp.cache.Push(hTx.Hash()) { + return nil, ErrTxInCache + } txmp.metrics.CacheSize.Set(float64(txmp.cache.Size())) // Check TTL cache to see if we've recently processed this transaction @@ -386,22 +387,22 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response // inaccurate. The true priority as determined by the application is the // most accurate. txmp.priorityReservoir.Add(wtx.priority) - + if err := wtx.check(constraints); err != nil { // ignore bad transactions logger.Info("rejected bad transaction", "priority", wtx.priority, "tx", wtx.Hash(), "post_check_err", err) txmp.metrics.FailedTxs.Add(1) return nil, err } - + txmp.txStore.Insert(wtx) - - txmp.metrics.InsertedTxs.Add(1) + + txmp.metrics.InsertedTxs.Add(1) txmp.metrics.TxSizeBytes.Add(float64(wtx.Size())) txmp.metrics.Size.Set(float64(txmp.NumTxsNotPending())) txmp.metrics.PendingSize.Set(float64(txmp.PendingSize())) txmp.metrics.TotalTxsSizeBytes.Set(float64(txmp.TotalTxsBytesSize())) - + txmp.notifyTxsAvailable() return res.ResponseCheckTx, nil } @@ -413,7 +414,7 @@ func (txmp *TxMempool) SafeGetTxsForHashes(txHashes []types.TxHash) (types.Txs, txs := make([]types.Tx, 0, len(txHashes)) missing := []types.TxHash{} for _, txHash := range txHashes { - if wtx := txmp.txStore.ByHash(txHash); wtx!=nil { + if wtx := txmp.txStore.ByHash(txHash); wtx != nil { txs = append(txs, wtx.Tx()) } else { missing = append(missing, txHash) @@ -459,10 +460,10 @@ func (txmp *TxMempool) ReapMaxBytesMaxGas(height int64, maxBytes, maxGasWanted, return txs } -func (txmp *TxMempool) ReapTxsAndMark(limits ReapLimits) (types.Txs,int64) { +func (txmp *TxMempool) ReapTxsAndMark(limits ReapLimits) (types.Txs, int64) { txmp.mtx.Lock() defer txmp.mtx.Unlock() - return txmp.txStore.Reap(limits, true) + return txmp.txStore.Reap(limits, true) } // Update iterates over all the transactions provided by the block producer, @@ -482,8 +483,8 @@ func (txmp *TxMempool) Update( ) error { txmp.height = blockHeight txmp.notifiedTxsAvailable.Store(false) - txmp.txConstraintsFetcher = func() (TxConstraints,error) { - return txConstraints,nil + txmp.txConstraintsFetcher = func() (TxConstraints, error) { + return txConstraints, nil } txHashes := map[types.TxHash]struct{}{} @@ -506,7 +507,7 @@ func (txmp *TxMempool) Update( newPriorities := map[types.TxHash]int64{} if recheck { for _, wtx := range txmp.txStore.AllReady() { - if _,ok := txHashes[wtx.Hash()]; ok { + if _, ok := txHashes[wtx.Hash()]; ok { continue } txmp.metrics.RecheckTimes.Add(1) @@ -515,19 +516,19 @@ func (txmp *TxMempool) Update( Type: abci.CheckTxTypeV2Recheck, }) // If recheck fails, just remove the tx. - if err!=nil || res.IsOK() { - txHashes[wtx.Hash()] = struct{}{} + if err != nil || res.IsOK() { + txHashes[wtx.Hash()] = struct{}{} } else { newPriorities[wtx.Hash()] = res.Priority } } } - txmp.txStore.Update(updateSpec { - Now: time.Now(), - Height: blockHeight, - ToRemove: txHashes, + txmp.txStore.Update(updateSpec{ + Now: time.Now(), + Height: blockHeight, + ToRemove: txHashes, NewPriorities: newPriorities, - Constraints: txConstraints, + Constraints: txConstraints, }) txmp.notifyTxsAvailable() txmp.metrics.Size.Set(float64(txmp.NumTxsNotPending())) diff --git a/sei-tendermint/internal/mempool/reactor/ids.go b/sei-tendermint/internal/mempool/reactor/ids.go deleted file mode 100644 index b089526fe9..0000000000 --- a/sei-tendermint/internal/mempool/reactor/ids.go +++ /dev/null @@ -1,87 +0,0 @@ -package reactor - -import ( - "fmt" - "math" - "sync" - - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" - "github.com/sei-protocol/sei-chain/sei-tendermint/types" -) - -const MaxActiveIDs = math.MaxUint16 - -type IDs struct { - mtx sync.RWMutex - peerMap map[types.NodeID]uint16 - nextID uint16 // assumes that a node will never have over 65536 active peers - activeIDs map[uint16]struct{} // used to check if a given peerID key is used -} - -func NewMempoolIDs() *IDs { - return &IDs{ - peerMap: make(map[types.NodeID]uint16), - - // reserve UnknownPeerID for mempoolReactor.BroadcastTx - activeIDs: map[uint16]struct{}{mempool.UnknownPeerID: {}}, - nextID: 1, - } -} - -// ReserveForPeer searches for the next unused ID and assigns it to the provided -// peer. -func (ids *IDs) ReserveForPeer(peerID types.NodeID) { - ids.mtx.Lock() - defer ids.mtx.Unlock() - - if _, ok := ids.peerMap[peerID]; ok { - // the peer has been reserved - return - } - - curID := ids.nextPeerID() - ids.peerMap[peerID] = curID - ids.activeIDs[curID] = struct{}{} -} - -// Reclaim returns the ID reserved for the peer back to unused pool. -func (ids *IDs) Reclaim(peerID types.NodeID) { - ids.mtx.Lock() - defer ids.mtx.Unlock() - - removedID, ok := ids.peerMap[peerID] - if ok { - delete(ids.activeIDs, removedID) - delete(ids.peerMap, peerID) - if removedID < ids.nextID { - ids.nextID = removedID - } - } -} - -// GetForPeer returns an ID reserved for the peer. -func (ids *IDs) GetForPeer(peerID types.NodeID) uint16 { - ids.mtx.RLock() - defer ids.mtx.RUnlock() - - return ids.peerMap[peerID] -} - -// nextPeerID returns the next unused peer ID to use. We assume that the mutex -// is already held. -func (ids *IDs) nextPeerID() uint16 { - if len(ids.activeIDs) == MaxActiveIDs { - panic(fmt.Sprintf("node has maximum %d active IDs and wanted to get one more", MaxActiveIDs)) - } - - _, idExists := ids.activeIDs[ids.nextID] - for idExists { - ids.nextID++ - _, idExists = ids.activeIDs[ids.nextID] - } - - curID := ids.nextID - ids.nextID++ - - return curID -} diff --git a/sei-tendermint/internal/mempool/reactor/ids_test.go b/sei-tendermint/internal/mempool/reactor/ids_test.go deleted file mode 100644 index 53ad039b07..0000000000 --- a/sei-tendermint/internal/mempool/reactor/ids_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package reactor - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/sei-protocol/sei-chain/sei-tendermint/types" -) - -func TestMempoolIDsBasic(t *testing.T) { - ids := NewMempoolIDs() - - peerID, err := types.NewNodeID("0011223344556677889900112233445566778899") - require.NoError(t, err) - require.EqualValues(t, 0, ids.GetForPeer(peerID)) - - ids.ReserveForPeer(peerID) - require.EqualValues(t, 1, ids.GetForPeer(peerID)) - - ids.Reclaim(peerID) - require.EqualValues(t, 0, ids.GetForPeer(peerID)) - - ids.ReserveForPeer(peerID) - require.EqualValues(t, 1, ids.GetForPeer(peerID)) -} - -func TestMempoolIDsPeerDupReserve(t *testing.T) { - ids := NewMempoolIDs() - - peerID, err := types.NewNodeID("0011223344556677889900112233445566778899") - require.NoError(t, err) - require.EqualValues(t, 0, ids.GetForPeer(peerID)) - - ids.ReserveForPeer(peerID) - require.EqualValues(t, 1, ids.GetForPeer(peerID)) - - ids.ReserveForPeer(peerID) - require.EqualValues(t, 1, ids.GetForPeer(peerID)) -} - -func TestMempoolIDs2Peers(t *testing.T) { - ids := NewMempoolIDs() - - peer1ID, _ := types.NewNodeID("0011223344556677889900112233445566778899") - require.EqualValues(t, 0, ids.GetForPeer(peer1ID)) - - ids.ReserveForPeer(peer1ID) - require.EqualValues(t, 1, ids.GetForPeer(peer1ID)) - - ids.Reclaim(peer1ID) - require.EqualValues(t, 0, ids.GetForPeer(peer1ID)) - - peer2ID, _ := types.NewNodeID("1011223344556677889900112233445566778899") - - ids.ReserveForPeer(peer2ID) - require.EqualValues(t, 1, ids.GetForPeer(peer2ID)) - - ids.ReserveForPeer(peer1ID) - require.EqualValues(t, 2, ids.GetForPeer(peer1ID)) -} - -func TestMempoolIDsNextExistID(t *testing.T) { - ids := NewMempoolIDs() - - peer1ID, _ := types.NewNodeID("0011223344556677889900112233445566778899") - ids.ReserveForPeer(peer1ID) - require.EqualValues(t, 1, ids.GetForPeer(peer1ID)) - - peer2ID, _ := types.NewNodeID("1011223344556677889900112233445566778899") - ids.ReserveForPeer(peer2ID) - require.EqualValues(t, 2, ids.GetForPeer(peer2ID)) - - peer3ID, _ := types.NewNodeID("2011223344556677889900112233445566778899") - ids.ReserveForPeer(peer3ID) - require.EqualValues(t, 3, ids.GetForPeer(peer3ID)) - - ids.Reclaim(peer1ID) - require.EqualValues(t, 0, ids.GetForPeer(peer1ID)) - - ids.Reclaim(peer3ID) - require.EqualValues(t, 0, ids.GetForPeer(peer3ID)) - - ids.ReserveForPeer(peer1ID) - require.EqualValues(t, 1, ids.GetForPeer(peer1ID)) - - ids.ReserveForPeer(peer3ID) - require.EqualValues(t, 3, ids.GetForPeer(peer3ID)) -} diff --git a/sei-tendermint/internal/mempool/reactor/reactor.go b/sei-tendermint/internal/mempool/reactor/reactor.go index 0f1b2e9c1a..d5aae51173 100644 --- a/sei-tendermint/internal/mempool/reactor/reactor.go +++ b/sei-tendermint/internal/mempool/reactor/reactor.go @@ -34,7 +34,6 @@ type Reactor struct { cfg *config.MempoolConfig mempool *mempool.TxMempool - ids *IDs router *p2p.Router @@ -53,7 +52,6 @@ func NewReactor(cfg *config.MempoolConfig, txmp *mempool.TxMempool, router *p2p. r := &Reactor{ cfg: cfg, mempool: txmp, - ids: NewMempoolIDs(), router: router, channel: channel, failedCheckTxCounts: utils.NewMutex(map[types.NodeID]int{}), @@ -213,7 +211,6 @@ func (r *Reactor) processPeerUpdates(ctx context.Context) error { } pctx, pcancel := context.WithCancel(ctx) peerRoutines[update.NodeID] = pcancel - r.ids.ReserveForPeer(update.NodeID) // We keep peer management even when broadcasting is disabled, // so that failedCheckTxCounts WAI. if r.cfg.Broadcast { @@ -224,7 +221,6 @@ func (r *Reactor) processPeerUpdates(ctx context.Context) error { } case p2p.PeerStatusDown: - r.ids.Reclaim(update.NodeID) for counts := range r.failedCheckTxCounts.Lock() { delete(counts, update.NodeID) } diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index e0b227405b..7a6503731f 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -1,24 +1,24 @@ package mempool import ( + "cmp" "context" - "slices" + "fmt" "maps" "math/big" + "slices" "time" - "cmp" - "fmt" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/ethereum/go-ethereum/common" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/libs/clist" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) type hashedTx struct { - tx types.Tx - hash types.TxHash + tx types.Tx + hash types.TxHash protoSize int64 } @@ -30,21 +30,21 @@ func newHashedTx(tx types.Tx) hashedTx { func (ktx *hashedTx) Tx() types.Tx { return ktx.tx } func (ktx *hashedTx) Hash() types.TxHash { return ktx.hash } -func (ktx *hashedTx) Size() uint64 { return uint64(len(ktx.tx)) } +func (ktx *hashedTx) Size() uint64 { return uint64(len(ktx.tx)) } // WrappedTx defines a wrapper around a raw transaction with additional metadata // that is used for indexing. type WrappedTx struct { hashedTx - height int64 // height defines the height at which the transaction was validated at - gasWanted int64 // gasWanted defines the amount of gas the transaction sender requires - estimatedGas int64 // estimatedGas defines the amount of gas that the transaction is estimated to use - priority int64 // ResponseCheckTx.priority - timestamp time.Time // time at which the transaction was received - evm utils.Option[evmTx] // evm transaction info - + height int64 // height defines the height at which the transaction was validated at + gasWanted int64 // gasWanted defines the amount of gas the transaction sender requires + estimatedGas int64 // estimatedGas defines the amount of gas that the transaction is estimated to use + priority int64 // ResponseCheckTx.priority + timestamp time.Time // time at which the transaction was received + evm utils.Option[evmTx] // evm transaction info + readyEl utils.Option[*clist.CElement[types.Tx]] - reaped bool + reaped bool } func (wtx *WrappedTx) check(c TxConstraints) error { @@ -75,9 +75,9 @@ func (wtx *WrappedTx) EVMNonce() uint64 { } type evmAccount struct { - balance *big.Int + balance *big.Int firstNonce uint64 - nextNonce uint64 + nextNonce uint64 } type txCounter struct { @@ -106,41 +106,41 @@ func (c *txCounter) LessEqual(b *txCounter) bool { } func (s txStoreState) PendingBytes() uint64 { return s.total.bytes - s.ready.bytes } -func (s txStoreState) PendingCount() int { return s.total.count - s.ready.count } +func (s txStoreState) PendingCount() int { return s.total.count - s.ready.count } type txStoreInner struct { - byHash map[types.TxHash]*WrappedTx - byNonce map[evmAddrNonce]*WrappedTx + byHash map[types.TxHash]*WrappedTx + byNonce map[evmAddrNonce]*WrappedTx accounts map[common.Address]*evmAccount softLimit txCounter hardLimit txCounter - state utils.AtomicSend[txStoreState] + state utils.AtomicSend[txStoreState] } type txStore struct { - config *Config - app *proxy.Proxy - inner utils.RWMutex[*txStoreInner] - state utils.AtomicRecv[txStoreState] + config *Config + app *proxy.Proxy + inner utils.RWMutex[*txStoreInner] + state utils.AtomicRecv[txStoreState] readyTxs *clist.CList[types.Tx] } func NewTxStore(config *Config) *txStore { - softLimit := txCounter{count:config.Size, bytes: utils.Clamp[uint64](config.MaxTxsBytes)} - hardLimit := txCounter{count:2*softLimit.count, bytes: 2*softLimit.bytes} + softLimit := txCounter{count: config.Size, bytes: utils.Clamp[uint64](config.MaxTxsBytes)} + hardLimit := txCounter{count: 2 * softLimit.count, bytes: 2 * softLimit.bytes} inner := &txStoreInner{ - byHash: map[types.TxHash]*WrappedTx{}, - accounts: map[common.Address]*evmAccount{}, + byHash: map[types.TxHash]*WrappedTx{}, + accounts: map[common.Address]*evmAccount{}, softLimit: softLimit, hardLimit: hardLimit, - state: utils.NewAtomicSend(txStoreState{}), + state: utils.NewAtomicSend(txStoreState{}), } return &txStore{ - config: config, - inner: utils.NewRWMutex(inner), + config: config, + inner: utils.NewRWMutex(inner), readyTxs: clist.New[types.Tx](), - state: inner.state.Subscribe(), + state: inner.state.Subscribe(), } } @@ -155,11 +155,11 @@ func (txs *txStore) WaitForTxs(ctx context.Context) error { func (txs *txStore) NextNonce(addr common.Address) uint64 { for inner := range txs.inner.RLock() { - if acc,ok := inner.accounts[addr]; ok { + if acc, ok := inner.accounts[addr]; ok { return acc.nextNonce } } - return txs.app.EvmNonce(addr) + return txs.app.EvmNonce(addr) } // GetAllTxs returns all the transactions currently in the store. @@ -173,9 +173,9 @@ func (txs *txStore) GetAllTxs() []*WrappedTx { func (txs *txStore) AllReady() []*WrappedTx { var ready []*WrappedTx for inner := range txs.inner.RLock() { - for _,wtx := range inner.byHash { + for _, wtx := range inner.byHash { if inner.isReady(wtx) { - ready = append(ready,wtx) + ready = append(ready, wtx) } } } @@ -191,33 +191,37 @@ func (txs *txStore) ByHash(key types.TxHash) *WrappedTx { } func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) bool { - if _,ok := inner.byHash[wtx.Hash()]; ok { return false } + if _, ok := inner.byHash[wtx.Hash()]; ok { + return false + } state := inner.state.Load() - if evm,ok := wtx.evm.Get(); ok { + if evm, ok := wtx.evm.Get(); ok { // Fetch the evm account state. - account,ok := inner.accounts[evm.address] + account, ok := inner.accounts[evm.address] if !ok { // TODO(gprusak): consider whether we should move these queries out of the mutex. - b := txs.app.EvmBalance(evm.address,evm.seiAddress) + b := txs.app.EvmBalance(evm.address, evm.seiAddress) n := txs.app.EvmNonce(evm.address) - account = &evmAccount{b,n,n} - inner.accounts[evm.address] = account + account = &evmAccount{b, n, n} + inner.accounts[evm.address] = account } // Reject transactions with old nonces. if evm.nonce < account.firstNonce { return false } - an := evmAddrNonce{evm.address,evm.nonce} - if old,ok := inner.byNonce[an]; ok { + an := evmAddrNonce{evm.address, evm.nonce} + if old, ok := inner.byNonce[an]; ok { // If the old tx is ready but the new tx is not, then reject the new tx. if old.evm.OrPanic("non-evm tx").nonce < account.nextNonce && account.balance.Cmp(evm.requiredBalance) < 0 { return false } // If the old tx has >= priority, then reject new tx. - if old.priority >= wtx.priority { return false } + if old.priority >= wtx.priority { + return false + } // Remove the old transaction. - delete(inner.byHash,old.Hash()) - if el,ok := wtx.readyEl.Get(); ok { + delete(inner.byHash, old.Hash()) + if el, ok := wtx.readyEl.Get(); ok { txs.readyTxs.Remove(el) } state.ready.Dec(old.Size()) @@ -226,11 +230,13 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) bool { } state.total.Inc(wtx.Size()) inner.byNonce[an] = wtx - // Update account ready txs. + // Update account ready txs. for { an.Nonce = account.nextNonce - wtx,ok := inner.byNonce[an] - if !ok || account.balance.Cmp(wtx.evm.OrPanic("non-evm tx").requiredBalance) < 0 { break } + wtx, ok := inner.byNonce[an] + if !ok || account.balance.Cmp(wtx.evm.OrPanic("non-evm tx").requiredBalance) < 0 { + break + } account.nextNonce += 1 state.ready.Inc(wtx.Size()) if !wtx.readyEl.IsPresent() { @@ -250,9 +256,9 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) bool { return true } -// WARNING: works only if wtx has been already inserted. +// WARNING: works only if wtx has been already inserted. func (inner *txStoreInner) isReady(wtx *WrappedTx) bool { - evm,ok := wtx.evm.Get() + evm, ok := wtx.evm.Get() return !ok || evm.nonce < inner.accounts[evm.address].nextNonce } @@ -265,41 +271,41 @@ func (inner *txStoreInner) inInclusionOrder() []*WrappedTx { // Split txs into ready and pending. // TODO(gprusak): we can precisely preallocate ready and pending in a single array, // based on inner.state.total.count and inner.state.ready.count - var ready,pending []*WrappedTx - for _,wtx := range inner.byHash { + var ready, pending []*WrappedTx + for _, wtx := range inner.byHash { if inner.isReady(wtx) { - ready = append(ready,wtx) + ready = append(ready, wtx) } else { - pending = append(pending,wtx) + pending = append(pending, wtx) } } - for _,txs := range utils.Slice(ready,pending) { + for _, txs := range utils.Slice(ready, pending) { // Sort by nonce. - slices.SortFunc(txs,func(a,b *WrappedTx) int { return cmp.Compare(a.EVMNonce(),b.EVMNonce()) }) + slices.SortFunc(txs, func(a, b *WrappedTx) int { return cmp.Compare(a.EVMNonce(), b.EVMNonce()) }) // Cap priority to obtain a linear order of txs per account by nonce. // NOTE: this precisely emulates the heap behavior described in this functions docstring. - accPrio := make(map[common.Address]int64,len(inner.accounts)) - txPrio := make(map[*WrappedTx]int64,len(txs)) - for _,tx := range txs { - if evm,ok := tx.evm.Get(); ok { - if prio,ok := accPrio[evm.address]; !ok || prio > tx.priority { + accPrio := make(map[common.Address]int64, len(inner.accounts)) + txPrio := make(map[*WrappedTx]int64, len(txs)) + for _, tx := range txs { + if evm, ok := tx.evm.Get(); ok { + if prio, ok := accPrio[evm.address]; !ok || prio > tx.priority { accPrio[evm.address] = tx.priority } txPrio[tx] = accPrio[evm.address] - } else { + } else { txPrio[tx] = tx.priority } } // Stable sort by capped priority - it preserves the nonce ordering. - slices.SortStableFunc(txs,func(a,b *WrappedTx) int { return -cmp.Compare(txPrio[a],txPrio[b]) }) + slices.SortStableFunc(txs, func(a, b *WrappedTx) int { return -cmp.Compare(txPrio[a], txPrio[b]) }) } - return append(ready,pending...) + return append(ready, pending...) } // SetTx stores a *WrappedTx by its hash. func (txs *txStore) Insert(wtx *WrappedTx) { for inner := range txs.inner.Lock() { - txs.insert(inner,wtx) + txs.insert(inner, wtx) if total := inner.state.Load().total; !total.LessEqual(&inner.hardLimit) { txs.compact(inner, false) } @@ -310,7 +316,7 @@ func (txs *txStore) Insert(wtx *WrappedTx) { func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { // Order all txs by priority. wtxs := inner.inInclusionOrder() - inner.state.Store(txStoreState{}) + inner.state.Store(txStoreState{}) inner.byHash = map[types.TxHash]*WrappedTx{} inner.byNonce = map[evmAddrNonce]*WrappedTx{} if clearAccounts { @@ -319,13 +325,13 @@ func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { for _, account := range inner.accounts { account.nextNonce = account.firstNonce } - for _,wtx := range wtxs { + for _, wtx := range wtxs { total := inner.state.Load().total total.Inc(wtx.Size()) if total.LessEqual(&inner.softLimit) { - txs.insert(inner,wtx) + txs.insert(inner, wtx) } else { - if el,ok := wtx.readyEl.Get(); ok { + if el, ok := wtx.readyEl.Get(); ok { txs.readyTxs.Remove(el) } } @@ -333,30 +339,30 @@ func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { } type updateSpec struct { - Now time.Time - Height int64 - ToRemove map[types.TxHash]struct{} - Constraints TxConstraints + Now time.Time + Height int64 + ToRemove map[types.TxHash]struct{} + Constraints TxConstraints NewPriorities map[types.TxHash]int64 } func (txs *txStore) Update(spec updateSpec) { minHeight := utils.None[int64]() - if ttl,ok := txs.config.TTLNumBlocks.Get(); ok && spec.Height > ttl { + if ttl, ok := txs.config.TTLNumBlocks.Get(); ok && spec.Height > ttl { minHeight = utils.Some(spec.Height - ttl) } minTime := utils.None[time.Time]() - if d,ok := txs.config.TTLDuration.Get(); ok { + if d, ok := txs.config.TTLDuration.Get(); ok { minTime = utils.Some(spec.Now.Add(-d)) } for inner := range txs.inner.Lock() { toRemove := func(wtx *WrappedTx) bool { - if _,ok := spec.ToRemove[wtx.Hash()]; ok { + if _, ok := spec.ToRemove[wtx.Hash()]; ok { return true } if wtx.reaped { // If we already reaped the transaction, we shouldn't lose track of it. - return false + return false } if wtx.check(spec.Constraints) != nil { return true @@ -375,17 +381,17 @@ func (txs *txStore) Update(spec updateSpec) { } for txHash, wtx := range inner.byHash { if toRemove(wtx) { - delete(inner.byHash,txHash) - if el,ok := wtx.readyEl.Get(); ok { + delete(inner.byHash, txHash) + if el, ok := wtx.readyEl.Get(); ok { txs.readyTxs.Remove(el) } } else { - if newPriority,ok := spec.NewPriorities[wtx.Hash()]; ok { + if newPriority, ok := spec.NewPriorities[wtx.Hash()]; ok { wtx.priority = newPriority } } } - txs.compact(inner,true) + txs.compact(inner, true) } } @@ -421,7 +427,7 @@ func (txs *txStore) Reap(l ReapLimits, markReaped bool) (types.Txs, int64) { var wtxs []*WrappedTx for inner := range txs.inner.Lock() { if uint64(inner.state.Load().ready.count) >= txs.config.TxNotifyThreshold { - for _,wtx := range inner.inInclusionOrder() { + for _, wtx := range inner.inInclusionOrder() { // Transactions are reaped to be included in a block at a particular height. // In case of tendermint, txs are not reaped "in advance" - before the next block is proposed, // the previous one needs to be finalized. @@ -438,17 +444,17 @@ func (txs *txStore) Reap(l ReapLimits, markReaped bool) (types.Txs, int64) { if uint64(len(wtxs)) >= maxTxs || !inner.isReady(wtx) { break } - if maxBytes - totalSize < wtx.protoSize { - break + if maxBytes-totalSize < wtx.protoSize { + break } - if maxGasWanted - totalGasWanted < wtx.gasWanted { + if maxGasWanted-totalGasWanted < wtx.gasWanted { break } - if maxGasEstimated - totalGasEstimated < wtx.estimatedGas { + if maxGasEstimated-totalGasEstimated < wtx.estimatedGas { break } // include tx and update totals - wtx.reaped = markReaped + wtx.reaped = markReaped totalSize += wtx.protoSize totalGasWanted += wtx.gasWanted totalGasEstimated += wtx.estimatedGas @@ -457,13 +463,13 @@ func (txs *txStore) Reap(l ReapLimits, markReaped bool) (types.Txs, int64) { } } // EVM txs go first. - var evmTxs,nonEvmTxs types.Txs - for _,wtx := range wtxs { + var evmTxs, nonEvmTxs types.Txs + for _, wtx := range wtxs { if wtx.evm.IsPresent() { evmTxs = append(evmTxs, wtx.Tx()) } else { nonEvmTxs = append(nonEvmTxs, wtx.Tx()) } } - return append(evmTxs,nonEvmTxs...), totalGasEstimated + return append(evmTxs, nonEvmTxs...), totalGasEstimated } diff --git a/sei-tendermint/internal/mempool/types.go b/sei-tendermint/internal/mempool/types.go index 97d0b4951e..8cc85d6dcb 100644 --- a/sei-tendermint/internal/mempool/types.go +++ b/sei-tendermint/internal/mempool/types.go @@ -13,9 +13,13 @@ type TxConstraints struct { // state snapshot. type TxConstraintsFetcher func() (TxConstraints, error) -func NopTxConstraintsFetcher() (TxConstraints, error) { +func NopTxConstraints() TxConstraints { return TxConstraints{ MaxDataBytes: math.MaxInt64, MaxGas: -1, - }, nil + } +} + +func NopTxConstraintsFetcher() (TxConstraints, error) { + return NopTxConstraints(), nil } diff --git a/sei-tendermint/internal/p2p/giga_router.go b/sei-tendermint/internal/p2p/giga_router.go index 5434979ab0..6211cbec47 100644 --- a/sei-tendermint/internal/p2p/giga_router.go +++ b/sei-tendermint/internal/p2p/giga_router.go @@ -299,7 +299,7 @@ func (r *GigaRouter) executeBlock(ctx context.Context, b *atypes.GlobalBlock) (* // TODO: We need the constraints to be fixed per epoch, because we don't know where the lane blocks will be sequenced. // Therefore we disable constraints for now, until epochs are supported AND // chain state understands that consensus parameters can change only at the epoch boundary. - mempool.NopTxConstraintsFetcher, + mempool.NopTxConstraints(), // recheck=false; see TxMempool.Update doc for why. false, ) From cd616cea8346d7b5c3024fdfb50728b8da2b08d6 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 19 May 2026 17:18:23 +0200 Subject: [PATCH 010/100] fixing tests WIP --- sei-tendermint/internal/mempool/cache.go | 22 ------- sei-tendermint/internal/mempool/cache_test.go | 38 ----------- sei-tendermint/internal/mempool/mempool.go | 8 +-- .../internal/mempool/mempool_bench_test.go | 3 +- .../internal/mempool/mempool_test.go | 9 ++- .../internal/mempool/reactor/reactor_test.go | 64 ++++--------------- sei-tendermint/internal/state/execution.go | 2 +- 7 files changed, 19 insertions(+), 127 deletions(-) diff --git a/sei-tendermint/internal/mempool/cache.go b/sei-tendermint/internal/mempool/cache.go index 9555283cdc..76d23c9d49 100644 --- a/sei-tendermint/internal/mempool/cache.go +++ b/sei-tendermint/internal/mempool/cache.go @@ -11,28 +11,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) -// TxCache defines an interface for raw transaction caching in a mempool. -// Currently, a TxCache does not allow direct reading or getting of transaction -// values. A TxCache is used primarily to push transactions and removing -// transactions. Pushing via Push returns a boolean telling the caller if the -// transaction already exists in the cache or not. -type TxCache interface { - // Reset resets the cache to an empty state. - Reset() - - // Push adds the given transaction key to the cache and returns true if it was - // newly added. Otherwise, it returns false. - Push(tx types.TxHash) bool - - // Remove removes the given transaction key from the cache. - Remove(tx types.TxHash) - - // Size returns the current size of the cache - Size() int -} - -var _ TxCache = (*LRUTxCache)(nil) - // LRUTxCache maintains a thread-safe LRU cache of raw transactions. The cache // only stores the hash of the raw transaction. type LRUTxCache struct { diff --git a/sei-tendermint/internal/mempool/cache_test.go b/sei-tendermint/internal/mempool/cache_test.go index 6b2475f37e..246185b31c 100644 --- a/sei-tendermint/internal/mempool/cache_test.go +++ b/sei-tendermint/internal/mempool/cache_test.go @@ -108,32 +108,6 @@ func TestLRUTxCache(t *testing.T) { }) } -func TestNopTxCache(t *testing.T) { - cache := NopTxCache{} - - t.Run("Reset", func(t *testing.T) { - // Should not panic - cache.Reset() - }) - - t.Run("Push", func(t *testing.T) { - tx := types.Tx("test").Hash() - result := cache.Push(tx) - assert.True(t, result) - }) - - t.Run("Remove", func(t *testing.T) { - tx := types.Tx("test").Hash() - // Should not panic - cache.Remove(tx) - }) - - t.Run("Size", func(t *testing.T) { - size := cache.Size() - assert.Equal(t, 0, size) - }) -} - func TestDuplicateTxCache(t *testing.T) { t.Run("NewDuplicateTxCache_WithExpiration", func(t *testing.T) { cache := NewDuplicateTxCache(100, 100*time.Millisecond, 0) @@ -491,18 +465,6 @@ func TestDuplicateTxCache_EdgeCases(t *testing.T) { }) } -func TestCache_InterfaceCompliance(t *testing.T) { - // Test that all implementations properly implement their interfaces - - t.Run("LRUTxCache_Implements_TxCache", func(t *testing.T) { - var _ TxCache = (*LRUTxCache)(nil) - }) - - t.Run("NopTxCache_Implements_TxCache", func(t *testing.T) { - var _ TxCache = (*NopTxCache)(nil) - }) -} - // createTestTxHash creates a test TxHash from a string by hashing it func createTestTxHash(input string) types.TxHash { // Create a simple hash-like key for testing diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 6353c8c8de..92354888fe 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -449,14 +449,10 @@ func (txmp *TxMempool) Flush() { // NOTE: // - Transactions returned are not removed from the mempool transaction // store or indexes. -func (txmp *TxMempool) ReapMaxBytesMaxGas(height int64, maxBytes, maxGasWanted, maxGasEstimated int64) types.Txs { +func (txmp *TxMempool) ReapTxs(limits ReapLimits) types.Txs { txmp.mtx.Lock() defer txmp.mtx.Unlock() - txs, _ := txmp.txStore.Reap(ReapLimits{ - MaxBytes: utils.Some(maxBytes), - MaxGasWanted: utils.Some(maxGasWanted), - MaxGasEstimated: utils.Some(maxGasEstimated), - }, false) + txs, _ := txmp.txStore.Reap(limits, false) return txs } diff --git a/sei-tendermint/internal/mempool/mempool_bench_test.go b/sei-tendermint/internal/mempool/mempool_bench_test.go index 67b1f35a7a..84ce7f5b53 100644 --- a/sei-tendermint/internal/mempool/mempool_bench_test.go +++ b/sei-tendermint/internal/mempool/mempool_bench_test.go @@ -36,11 +36,10 @@ func BenchmarkTxMempool_CheckTx(b *testing.B) { priority := int64(rng.Intn(9999-1000) + 1000) tx := []byte(fmt.Sprintf("sender-%d-%d=%X=%d", n, peerID, prefix, priority)) - txInfo := TxInfo{SenderID: uint16(peerID)} b.StartTimer() - _, err = txmp.CheckTx(ctx, tx, txInfo) + _, err = txmp.CheckTx(ctx, tx) require.NoError(b, err) } } diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index 0b7c419daa..d1ae9fd490 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -146,7 +146,6 @@ func checkTxs(ctx context.Context, t *testing.T, txmp *TxMempool, numTxs int, pe t.Helper() txs := make([]testTx, numTxs) - txInfo := TxInfo{SenderID: peerID} rng := rand.New(rand.NewSource(time.Now().UnixNano())) @@ -161,7 +160,7 @@ func checkTxs(ctx context.Context, t *testing.T, txmp *TxMempool, numTxs int, pe tx: []byte(fmt.Sprintf("sender-%d-%d=%X=%d", i, peerID, prefix, priority)), priority: priority, } - _, err = txmp.CheckTx(ctx, txs[i].tx, txInfo) + _, err = txmp.CheckTx(ctx, txs[i].tx) require.NoError(t, err) } @@ -224,7 +223,7 @@ func TestTxMempool_TxsAvailable(t *testing.T) { // commit half the transactions and ensure we fire an event txmp.Lock() - require.NoError(t, txmp.Update(ctx, 1, rawTxs[:50], responses, txmp.txConstraintsFetcher, true)) + require.NoError(t, txmp.Update(ctx, 1, rawTxs[:50], responses, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) txmp.Unlock() ensureTxFire() ensureNoTxFire() @@ -257,7 +256,7 @@ func TestTxMempool_Size(t *testing.T) { } txmp.Lock() - require.NoError(t, txmp.Update(ctx, 1, rawTxs[:50], responses, txmp.txConstraintsFetcher, true)) + require.NoError(t, txmp.Update(ctx, 1, rawTxs[:50], responses, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) txmp.Unlock() require.Equal(t, len(rawTxs)/2, txmp.Size()) @@ -285,7 +284,7 @@ func TestTxMempool_Flush(t *testing.T) { } txmp.Lock() - require.NoError(t, txmp.Update(ctx, 1, rawTxs[:50], responses, txmp.txConstraintsFetcher, true)) + require.NoError(t, txmp.Update(ctx, 1, rawTxs[:50], responses, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) txmp.Unlock() txmp.Flush() diff --git a/sei-tendermint/internal/mempool/reactor/reactor_test.go b/sei-tendermint/internal/mempool/reactor/reactor_test.go index 1b65130ea1..f937ac6bea 100644 --- a/sei-tendermint/internal/mempool/reactor/reactor_test.go +++ b/sei-tendermint/internal/mempool/reactor/reactor_test.go @@ -12,7 +12,6 @@ import ( "time" "github.com/fortytw2/leaktest" - "github.com/sei-protocol/sei-chain/sei-tendermint/crypto/ed25519" "github.com/sei-protocol/sei-chain/sei-tendermint/abci/example/kvstore" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" @@ -54,11 +53,10 @@ func setupMempool(t testing.TB, app *proxy.Proxy, cacheSize int, txConstraintsFe return mempool.NewTxMempool(cfg.Mempool.ToMempoolConfig(), app, mempool.NopMetrics(), txConstraintsFetcher) } -func checkTxs(ctx context.Context, t *testing.T, txmp *mempool.TxMempool, numTxs int, peerID uint16) []testTx { +func checkTxs(ctx context.Context, t *testing.T, txmp *mempool.TxMempool, numTxs int) []testTx { t.Helper() txs := make([]testTx, numTxs) - txInfo := mempool.TxInfo{SenderID: peerID} rng := rand.New(rand.NewSource(time.Now().UnixNano())) for i := range numTxs { @@ -67,9 +65,9 @@ func checkTxs(ctx context.Context, t *testing.T, txmp *mempool.TxMempool, numTxs require.NoError(t, err) txs[i] = testTx{ - tx: []byte(fmt.Sprintf("sender-%d-%d=%X=%d", i, peerID, prefix, i+1000)), + tx: []byte(fmt.Sprintf("sender-%d-%d=%X=%d", i, prefix, i+1000)), } - _, err = txmp.CheckTx(ctx, txs[i].tx, txInfo) + _, err = txmp.CheckTx(ctx, txs[i].tx) require.NoError(t, err) } @@ -208,7 +206,7 @@ func TestReactorBroadcastTxs(t *testing.T) { primary := rts.nodes[0] secondaries := rts.nodes[1:] - txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs, mempool.UnknownPeerID) + txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs) require.Equal(t, numTxs, rts.reactors[primary].mempool.Size()) @@ -305,7 +303,6 @@ func TestReactorPeerDownClearsFailedCheckTxCount(t *testing.T) { require.NoError(t, reactor.handleMempoolMessage(t.Context(), msg)) require.Equal(t, utils.Some(1), peerFailedCheckTxCount(reactor, "sender")) - reactor.ids.Reclaim("sender") for counts := range reactor.failedCheckTxCounts.Lock() { delete(counts, "sender") } @@ -338,7 +335,6 @@ func TestReactorMissingFailedCheckTxCountIsNotRecreated(t *testing.T) { counts["sender"] = 0 delete(counts, "sender") } - reactor.ids.Reclaim("sender") require.NoError(t, reactor.handleMempoolMessage(t.Context(), msg)) require.Equal(t, utils.None[int](), peerFailedCheckTxCount(reactor, "sender")) @@ -362,7 +358,7 @@ func TestReactorConcurrency(t *testing.T) { for range runtime.NumCPU() * 2 { wg.Add(2) - txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs, mempool.UnknownPeerID) + txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs) go func() { defer wg.Done() @@ -376,10 +372,10 @@ func TestReactorConcurrency(t *testing.T) { deliverTxResponses[i] = &abci.ExecTxResult{Code: 0} } - require.NoError(t, txmp.Update(ctx, 1, convertTex(txs), deliverTxResponses, mempool.NopTxConstraintsFetcher, true)) + require.NoError(t, txmp.Update(ctx, 1, convertTex(txs), deliverTxResponses, mempool.NopTxConstraints(), true)) }() - _ = checkTxs(ctx, t, rts.reactors[secondary].mempool, numTxs, mempool.UnknownPeerID) + _ = checkTxs(ctx, t, rts.reactors[secondary].mempool, numTxs) go func() { defer wg.Done() @@ -388,7 +384,7 @@ func TestReactorConcurrency(t *testing.T) { txmp.Lock() defer txmp.Unlock() - err := txmp.Update(ctx, 1, []types.Tx{}, make([]*abci.ExecTxResult, 0), mempool.NopTxConstraintsFetcher, true) + err := txmp.Update(ctx, 1, []types.Tx{}, make([]*abci.ExecTxResult, 0), mempool.NopTxConstraints(), true) require.NoError(t, err) }() } @@ -407,8 +403,7 @@ func TestReactorNoBroadcastToSender(t *testing.T) { primary := rts.nodes[0] secondary := rts.nodes[1] - peerID := uint16(1) - _ = checkTxs(ctx, t, rts.mempools[primary], numTxs, peerID) + _ = checkTxs(ctx, t, rts.mempools[primary], numTxs) rts.start(t) time.Sleep(100 * time.Millisecond) @@ -433,7 +428,6 @@ func TestReactor_MaxTxBytes(t *testing.T) { _, err := rts.reactors[primary].mempool.CheckTx( ctx, tx1, - mempool.TxInfo{SenderID: mempool.UnknownPeerID}, ) require.NoError(t, err) @@ -443,46 +437,10 @@ func TestReactor_MaxTxBytes(t *testing.T) { rts.reactors[secondary].mempool.Flush() tx2 := tmrand.Bytes(cfg.Mempool.MaxTxBytes + 1) - _, err = rts.mempools[primary].CheckTx(ctx, tx2, mempool.TxInfo{SenderID: mempool.UnknownPeerID}) + _, err = rts.mempools[primary].CheckTx(ctx, tx2) require.Error(t, err) } -func TestDontExhaustMaxActiveIDs(t *testing.T) { - t.Skip("this test fails, but the property it tests is not very useful") - - ctx := t.Context() - rts := setupReactors(ctx, t, 1) - t.Cleanup(leaktest.Check(t)) - - nodeID := rts.nodes[0] - - for range MaxActiveIDs + 1 { - privKey := ed25519.GenerateSecretKey() - peerID := types.NodeIDFromPubKey(privKey.Public()) - rts.reactors[nodeID].ids.ReserveForPeer(peerID) - } -} - -func TestMempoolIDsPanicsIfNodeRequestsOvermaxActiveIDs(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode") - } - - ids := NewMempoolIDs() - - for i := range MaxActiveIDs - 1 { - peerID, err := types.NewNodeID(fmt.Sprintf("%040d", i)) - require.NoError(t, err) - ids.ReserveForPeer(peerID) - } - - peerID, err := types.NewNodeID(fmt.Sprintf("%040d", MaxActiveIDs-1)) - require.NoError(t, err) - require.Panics(t, func() { - ids.ReserveForPeer(peerID) - }) -} - func TestBroadcastTxForPeerStopsWhenPeerStops(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") @@ -499,7 +457,7 @@ func TestBroadcastTxForPeerStopsWhenPeerStops(t *testing.T) { rts.start(t) rts.network.Remove(t, secondary) - txs := checkTxs(ctx, t, rts.reactors[primary].mempool, 4, mempool.UnknownPeerID) + txs := checkTxs(ctx, t, rts.reactors[primary].mempool, 4) require.Equal(t, 4, len(txs)) require.Equal(t, 4, rts.mempools[primary].Size()) require.Equal(t, 0, rts.mempools[secondary].Size()) diff --git a/sei-tendermint/internal/state/execution.go b/sei-tendermint/internal/state/execution.go index f799141386..b4186b0c84 100644 --- a/sei-tendermint/internal/state/execution.go +++ b/sei-tendermint/internal/state/execution.go @@ -122,7 +122,7 @@ func (blockExec *BlockExecutor) CreateProposalBlock( // Fetch a limited amount of valid txs maxDataBytes := types.MaxDataBytes(maxBytes, evSize, state.Validators.Size()) - txs := blockExec.mempool.ReapMaxBytesMaxGas(height, maxDataBytes, maxGasWanted, maxGas) + txs := blockExec.mempool.ReapMaxBytesMaxGas(maxDataBytes, maxGasWanted, maxGas) block = state.MakeBlock(height, txs, lastCommit, evidence, proposerAddr) return block, nil } From c860fe95478ee0866ba02334e05ed404091f8696 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 19 May 2026 17:27:05 +0200 Subject: [PATCH 011/100] WIP --- .../internal/consensus/mempool_test.go | 8 +- .../internal/consensus/reactor_test.go | 3 +- .../internal/consensus/replay_test.go | 14 +- .../internal/consensus/state_test.go | 5 +- .../internal/mempool/mempool_test.go | 137 ++++++++---------- .../internal/mempool/recheck_drain_test.go | 8 +- .../internal/p2p/giga_router_test.go | 2 +- sei-tendermint/internal/rpc/core/mempool.go | 5 +- sei-tendermint/internal/state/execution.go | 7 +- sei-tendermint/node/node_test.go | 8 +- sei-tendermint/rpc/client/rpc_test.go | 6 +- .../test/fuzz/tests/mempool_test.go | 2 +- 12 files changed, 100 insertions(+), 105 deletions(-) diff --git a/sei-tendermint/internal/consensus/mempool_test.go b/sei-tendermint/internal/consensus/mempool_test.go index b97b9e793d..15e2411b4e 100644 --- a/sei-tendermint/internal/consensus/mempool_test.go +++ b/sei-tendermint/internal/consensus/mempool_test.go @@ -148,7 +148,7 @@ func checkTxsRange(ctx context.Context, t *testing.T, cs *testState, start, end for i := start; i < end; i++ { txBytes := make([]byte, 8) binary.BigEndian.PutUint64(txBytes, uint64(i)) - res, err := cs.txMempool.CheckTx(ctx, txBytes, mempool.TxInfo{}) + res, err := cs.txMempool.CheckTx(ctx, txBytes) require.NoError(t, err, "error after checkTx") require.Equal(t, code.CodeTypeOK, res.Code, "checkTx code is error, txBytes %X, index=%d", txBytes, i) } @@ -182,7 +182,7 @@ func TestMempoolTxConcurrentWithCommit(t *testing.T) { for i := range int(numTxs) { txBytes := make([]byte, 8) binary.BigEndian.PutUint64(txBytes, uint64(i)) - res, err := cs.txMempool.CheckTx(ctx, txBytes, mempool.TxInfo{}) + res, err := cs.txMempool.CheckTx(ctx, txBytes) require.NoError(t, err, "error after checkTx") require.Equal(t, code.CodeTypeOK, res.Code, "checkTx code is error, txBytes %X, index=%d", txBytes, i) } @@ -234,7 +234,7 @@ func TestMempoolRmBadTx(t *testing.T) { // Try to send the tx through the mempool. // CheckTx should not err, but the app should return a bad abci code // and the tx should get removed from the pool - res, err := cs.txMempool.CheckTx(ctx, txBytes, mempool.TxInfo{}) + res, err := cs.txMempool.CheckTx(ctx, txBytes) if err != nil { t.Errorf("error after CheckTx: %v", err) return @@ -247,7 +247,7 @@ func TestMempoolRmBadTx(t *testing.T) { // check for the tx for { - txs := cs.txMempool.ReapMaxBytesMaxGas(int64(len(txBytes)), utils.Max[int64](), utils.Max[int64]()) + txs := cs.txMempool.ReapTxs(mempool.ReapLimits{MaxBytes: utils.Some(int64(len(txBytes)))}) if len(txs) == 0 { emptyMempoolCh <- struct{}{} return diff --git a/sei-tendermint/internal/consensus/reactor_test.go b/sei-tendermint/internal/consensus/reactor_test.go index 1d56c57132..21c9306700 100644 --- a/sei-tendermint/internal/consensus/reactor_test.go +++ b/sei-tendermint/internal/consensus/reactor_test.go @@ -152,7 +152,7 @@ func finalizeTx( return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { for i, sub := range blocksSubs { s.Spawn(func() error { - if _, err := states[i].txMempool.CheckTx(ctx, tx, mempool.TxInfo{}); err != nil { + if _, err := states[i].txMempool.CheckTx(ctx, tx); err != nil { return fmt.Errorf("CheckTx(): %w", err) } for { @@ -367,7 +367,6 @@ func TestReactorCreatesBlockWhenEmptyBlocksFalse(t *testing.T) { _, err := states[1].txMempool.CheckTx( ctx, []byte{1, 2, 3}, - mempool.TxInfo{}, ) require.NoError(t, err) diff --git a/sei-tendermint/internal/consensus/replay_test.go b/sei-tendermint/internal/consensus/replay_test.go index c835b8757c..a9eb4e6a56 100644 --- a/sei-tendermint/internal/consensus/replay_test.go +++ b/sei-tendermint/internal/consensus/replay_test.go @@ -154,7 +154,7 @@ func sendTxs(ctx context.Context, cs *testState) error { return nil } tx := []byte{byte(i)} - if _, err := cs.txMempool.CheckTx(ctx, tx, mempool.TxInfo{}); err != nil { + if _, err := cs.txMempool.CheckTx(ctx, tx); err != nil { return fmt.Errorf("cs.mempool.CheckTx(): %w", err) } } @@ -389,7 +389,7 @@ func setupSimulator(ctx context.Context, t *testing.T) *simulatorTestSuite { require.NoError(t, err) valPubKey1ABCI := crypto.PubKeyToProto(newValidatorPubKey1) newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower) - _, err = css[0].txMempool.CheckTx(ctx, newValidatorTx1, mempool.TxInfo{}) + _, err = css[0].txMempool.CheckTx(ctx, newValidatorTx1) assert.NoError(t, err) css[0].signAddVotes(ctx, t, tmproto.PrecommitType, sim.Config.ChainID(), @@ -408,7 +408,7 @@ func setupSimulator(ctx context.Context, t *testing.T) *simulatorTestSuite { require.NoError(t, err) updatePubKey1ABCI := crypto.PubKeyToProto(updateValidatorPubKey1) updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25) - _, err = css[0].txMempool.CheckTx(ctx, updateValidatorTx1, mempool.TxInfo{}) + _, err = css[0].txMempool.CheckTx(ctx, updateValidatorTx1) assert.NoError(t, err) css[0].signAddVotes(ctx, t, tmproto.PrecommitType, sim.Config.ChainID(), types.BlockID{Hash: rs.ProposalBlock.Hash(), PartSetHeader: rs.ProposalBlockParts.Header()}, @@ -426,14 +426,14 @@ func setupSimulator(ctx context.Context, t *testing.T) *simulatorTestSuite { require.NoError(t, err) newVal2ABCI := crypto.PubKeyToProto(newValidatorPubKey2) newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower) - _, err = css[0].txMempool.CheckTx(ctx, newValidatorTx2, mempool.TxInfo{}) + _, err = css[0].txMempool.CheckTx(ctx, newValidatorTx2) assert.NoError(t, err) pv, _ = css[nVals+2].privValidator.Get() newValidatorPubKey3, err := pv.GetPubKey(ctx) require.NoError(t, err) newVal3ABCI := crypto.PubKeyToProto(newValidatorPubKey3) newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower) - _, err = css[0].txMempool.CheckTx(ctx, newValidatorTx3, mempool.TxInfo{}) + _, err = css[0].txMempool.CheckTx(ctx, newValidatorTx3) assert.NoError(t, err) css[0].signAddVotes(ctx, t, tmproto.PrecommitType, sim.Config.ChainID(), types.BlockID{Hash: rs.ProposalBlock.Hash(), PartSetHeader: rs.ProposalBlockParts.Header()}, @@ -469,7 +469,7 @@ func setupSimulator(ctx context.Context, t *testing.T) *simulatorTestSuite { ensureProposalFromCurrentLeader(height, round) rs = css[0].GetRoundState() removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0) - _, err = css[0].txMempool.CheckTx(ctx, removeValidatorTx2, mempool.TxInfo{}) + _, err = css[0].txMempool.CheckTx(ctx, removeValidatorTx2) assert.NoError(t, err) for i := 0; i < nVals+1; i++ { @@ -498,7 +498,7 @@ func setupSimulator(ctx context.Context, t *testing.T) *simulatorTestSuite { rs = css[0].GetRoundState() removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0) - _, err = css[0].txMempool.CheckTx(ctx, removeValidatorTx3, mempool.TxInfo{}) + _, err = css[0].txMempool.CheckTx(ctx, removeValidatorTx3) assert.NoError(t, err) for i := 0; i < nVals+1; i++ { if i == selfIndex { diff --git a/sei-tendermint/internal/consensus/state_test.go b/sei-tendermint/internal/consensus/state_test.go index f3ec67aa1b..6a2531631a 100644 --- a/sei-tendermint/internal/consensus/state_test.go +++ b/sei-tendermint/internal/consensus/state_test.go @@ -18,7 +18,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/crypto/ed25519" cstypes "github.com/sei-protocol/sei-chain/sei-tendermint/internal/consensus/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/eventbus" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" tmpubsub "github.com/sei-protocol/sei-chain/sei-tendermint/internal/pubsub" tmquery "github.com/sei-protocol/sei-chain/sei-tendermint/internal/pubsub/query" @@ -2208,7 +2207,7 @@ func TestStartNextHeightCorrectlyAfterTimeout(t *testing.T) { ensureNewBlockHeader(t, newBlockHeader, height, blockID.Hash) - _, err := cs1.txMempool.CheckTx(ctx, types.Tx("test-key=test-value"), mempool.TxInfo{}) + _, err := cs1.txMempool.CheckTx(ctx, types.Tx("test-key=test-value")) require.NoError(t, err, "failed to seed the mempool with a transaction") ensureNewTimeout(t, timeoutProposeCh, height+1, 0) @@ -2592,7 +2591,7 @@ func TestTryCreateProposalBlock_PartsMismatch(t *testing.T) { incrementRound(vss[1:]...) cs.startTestRound(ctx, height, round) - _, err := cs.txMempool.CheckTx(ctx, types.Tx("test-key=test-value"), mempool.TxInfo{}) + _, err := cs.txMempool.CheckTx(ctx, types.Tx("test-key=test-value")) require.NoError(t, err, "failed to seed the mempool with a transaction") proposal, block := cs.decideProposal(ctx, t, vss[1], height, round) diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index d1ae9fd490..7eda5fd43d 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -142,7 +142,7 @@ func setup(t testing.TB, app *proxy.Proxy, cacheSize int, txConstraintsFetcher T return NewTxMempool(cfg, app, NopMetrics(), txConstraintsFetcher) } -func checkTxs(ctx context.Context, t *testing.T, txmp *TxMempool, numTxs int, peerID uint16) []testTx { +func checkTxs(ctx context.Context, t *testing.T, txmp *TxMempool, numTxs int) []testTx { t.Helper() txs := make([]testTx, numTxs) @@ -157,7 +157,7 @@ func checkTxs(ctx context.Context, t *testing.T, txmp *TxMempool, numTxs int, pe priority := int64(rng.Intn(9999-1000) + 1000) txs[i] = testTx{ - tx: []byte(fmt.Sprintf("sender-%d-%d=%X=%d", i, peerID, prefix, priority)), + tx: []byte(fmt.Sprintf("sender-%d-%d=%X=%d", i, prefix, priority)), priority: priority, } _, err = txmp.CheckTx(ctx, txs[i].tx) @@ -207,7 +207,7 @@ func TestTxMempool_TxsAvailable(t *testing.T) { // Execute CheckTx for some transactions and ensure TxsAvailable only fires // once. - txs := checkTxs(ctx, t, txmp, 100, 0) + txs := checkTxs(ctx, t, txmp, 100) ensureTxFire() ensureNoTxFire() @@ -230,7 +230,7 @@ func TestTxMempool_TxsAvailable(t *testing.T) { // Execute CheckTx for more transactions and ensure we do not fire another // event as we're still on the same height (1). - _ = checkTxs(ctx, t, txmp, 100, 0) + _ = checkTxs(ctx, t, txmp, 100) ensureNoTxFire() } @@ -240,7 +240,7 @@ func TestTxMempool_Size(t *testing.T) { client := &application{Application: kvstore.NewApplication()} txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - txs := checkTxs(ctx, t, txmp, 100, 0) + txs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(txs), txmp.Size()) require.Equal(t, 0, txmp.PendingSize()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -269,7 +269,7 @@ func TestTxMempool_Flush(t *testing.T) { client := &application{Application: kvstore.NewApplication()} txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - txs := checkTxs(ctx, t, txmp, 100, 0) + txs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(txs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -299,7 +299,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { client := &application{Application: kvstore.NewApplication(), gasEstimated: &gasEstimated} txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - tTxs := checkTxs(ctx, t, txmp, 100, 0) // all txs request 1 gas unit + tTxs := checkTxs(ctx, t, txmp, 100) // all txs request 1 gas unit require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -330,7 +330,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapMaxBytesMaxGas(utils.Max[int64](), 50, utils.Max[int64]()) + reapedTxs := txmp.ReapTxs(ReapLimits{MaxGasWanted: utils.Some(int64(50))}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -341,7 +341,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapMaxBytesMaxGas(1000, utils.Max[int64](), utils.Max[int64]()) + reapedTxs := txmp.ReapTxs(ReapLimits{MaxBytes: utils.Some(int64(1000))}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -353,7 +353,10 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapMaxBytesMaxGas(1500, 30, utils.Max[int64]()) + reapedTxs := txmp.ReapTxs(ReapLimits{ + MaxBytes: utils.Some(int64(1500)), + MaxGasWanted: utils.Some(int64(30)), + }) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -364,7 +367,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapMaxBytesMaxGas(utils.Max[int64](), 2, utils.Max[int64]()) + reapedTxs := txmp.ReapTxs(ReapLimits{MaxGasWanted: utils.Some(int64(2))}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Len(t, reapedTxs, 2) @@ -374,7 +377,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapMaxBytesMaxGas(utils.Max[int64](), utils.Max[int64](), 50) + reapedTxs := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(50))}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Len(t, reapedTxs, 50) @@ -390,7 +393,7 @@ func TestTxMempool_ReapMaxBytesMaxGas_FallbackToGasWanted(t *testing.T) { client := &application{Application: kvstore.NewApplication(), gasEstimated: &gasEstimated} txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - tTxs := checkTxs(ctx, t, txmp, 100, 0) + tTxs := checkTxs(ctx, t, txmp, 100) txMap := make(map[types.TxHash]testTx) priorities := make([]int64, len(tTxs)) @@ -418,7 +421,7 @@ func TestTxMempool_ReapMaxBytesMaxGas_FallbackToGasWanted(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapMaxBytesMaxGas(utils.Max[int64](), utils.Max[int64](), 50) + reapedTxs := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(50))}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Len(t, reapedTxs, 50) @@ -433,7 +436,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { client := &application{Application: kvstore.NewApplication()} txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - tTxs := checkTxs(ctx, t, txmp, 100, 0) + tTxs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -464,7 +467,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapMaxTxs(utils.Max[int]()) + reapedTxs := txmp.ReapTxs(ReapLimits{}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -475,7 +478,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapMaxTxs(1) + reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(1))}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -486,7 +489,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapMaxTxs(len(tTxs) / 2) + reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(len(tTxs) / 2))}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -505,17 +508,16 @@ func TestTxMempool_ReapMaxBytesMaxGas_MinGasEVMTxThreshold(t *testing.T) { client := &application{Application: kvstore.NewApplication(), gasEstimated: &gasEstimated, gasWanted: &gasWanted} txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - peerID := uint16(1) address := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" // Insert a single EVM tx (format: evm-sender=account=priority=nonce) - _, err := txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address, 100, 0)), TxInfo{SenderID: peerID}) + _, err := txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address, 100, 0))) require.NoError(t, err) require.Equal(t, 1, txmp.Size()) // With MinGasEVMTx=21000, estimatedGas (10000) is ignored and we fallback to gasWanted (50000). // Setting maxGasEstimated below gasWanted should therefore result in 0 reaped txs. - reaped := txmp.ReapMaxBytesMaxGas(utils.Max[int64](), utils.Max[int64](), 40000) + reaped := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(40000))}) require.Len(t, reaped, 0) // Note: If MinGasEVMTx is changed to 0, the same scenario would use estimatedGas (10000) @@ -533,14 +535,14 @@ func TestTxMempool_CheckTxExceedsMaxSize(t *testing.T) { _, err := rng.Read(tx) require.NoError(t, err) - _, err = txmp.CheckTx(ctx, tx, TxInfo{SenderID: 0}) + _, err = txmp.CheckTx(ctx, tx) require.Error(t, err) tx = make([]byte, txmp.config.MaxTxBytes-1) _, err = rng.Read(tx) require.NoError(t, err) - _, err = txmp.CheckTx(ctx, tx, TxInfo{SenderID: 0}) + _, err = txmp.CheckTx(ctx, tx) require.NoError(t, err) } @@ -551,7 +553,6 @@ func TestTxMempool_Reap_SkipGasUnfitAndCollectMinTxs(t *testing.T) { client := app txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - peerID := uint16(1) // Insert one high-priority tx that is unfit by gas (exceeds maxGasEstimated) gwBig := int64(100) @@ -559,7 +560,7 @@ func TestTxMempool_Reap_SkipGasUnfitAndCollectMinTxs(t *testing.T) { app.gasWanted = &gwBig app.gasEstimated = &geBig bigTx := []byte(fmt.Sprintf("sender-big=key=%d", 1000000)) - _, err := txmp.CheckTx(ctx, bigTx, TxInfo{SenderID: peerID}) + _, err := txmp.CheckTx(ctx, bigTx) require.NoError(t, err) // Now insert many small, lower-priority txs that fit well under the gas limit @@ -569,12 +570,12 @@ func TestTxMempool_Reap_SkipGasUnfitAndCollectMinTxs(t *testing.T) { app.gasEstimated = &geSmall for i := 0; i < 50; i++ { tx := []byte(fmt.Sprintf("sender-%d=key=%d", i, 1000-i)) - _, err := txmp.CheckTx(ctx, tx, TxInfo{SenderID: peerID}) + _, err := txmp.CheckTx(ctx, tx) require.NoError(t, err) } // Reap with a maxGasEstimated that makes the first tx unfit but allows many small txs - reaped := txmp.ReapMaxBytesMaxGas(utils.Max[int64](), utils.Max[int64](), 50) + reaped := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(50))}) require.Len(t, reaped, MinTxsToPeek) // Ensure all reaped small txs are under gas constraint @@ -590,7 +591,6 @@ func TestTxMempool_Reap_SkipGasUnfitStopsAtMinEvenWithCapacity(t *testing.T) { client := app txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - peerID := uint16(1) // First tx: unfit by gas (bigger than limit), highest priority gwBig := int64(100) @@ -598,7 +598,7 @@ func TestTxMempool_Reap_SkipGasUnfitStopsAtMinEvenWithCapacity(t *testing.T) { app.gasWanted = &gwBig app.gasEstimated = &geBig bigTx := []byte(fmt.Sprintf("sender-big=key=%d", 1000000)) - _, err := txmp.CheckTx(ctx, bigTx, TxInfo{SenderID: peerID}) + _, err := txmp.CheckTx(ctx, bigTx) require.NoError(t, err) // Insert many small txs that fit; plenty of capacity for more than 10 @@ -608,12 +608,12 @@ func TestTxMempool_Reap_SkipGasUnfitStopsAtMinEvenWithCapacity(t *testing.T) { app.gasEstimated = &geSmall for i := 0; i < 100; i++ { tx := []byte(fmt.Sprintf("sender-sm-%d=key=%d", i, 2000-i)) - _, err := txmp.CheckTx(ctx, tx, TxInfo{SenderID: peerID}) + _, err := txmp.CheckTx(ctx, tx) require.NoError(t, err) } // Make the gas limit very small so the first (big) tx is unfit and we only collect MinTxsPerBlock - reaped := txmp.ReapMaxBytesMaxGas(utils.Max[int64](), utils.Max[int64](), 10) + reaped := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(10))}) require.Len(t, reaped, MinTxsToPeek) } @@ -623,7 +623,6 @@ func TestTxMempool_Prioritization(t *testing.T) { client := &application{Application: kvstore.NewApplication()} txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 100, NopTxConstraintsFetcher) - peerID := uint16(1) address1 := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" address2 := "0xfD23B3A9DE15e92B9ef9540E587B3661E15A12fA" @@ -663,12 +662,12 @@ func TestTxMempool_Prioritization(t *testing.T) { txsCopy = append(txsCopy, evmTxs...) for i := range txsCopy { - _, err := txmp.CheckTx(ctx, txsCopy[i], TxInfo{SenderID: peerID}) + _, err := txmp.CheckTx(ctx, txsCopy[i]) require.NoError(t, err) } // Reap the transactions - reapedTxs := txmp.ReapMaxTxs(len(txs)) + reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(len(txs)))}) // Check if the reaped transactions are in the correct order of their priorities for _, tx := range txs { fmt.Printf("expected: %s\n", string(tx)) @@ -689,13 +688,12 @@ func TestTxMempool_PendingStoreSize(t *testing.T) { txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 100, NopTxConstraintsFetcher) txmp.config.PendingSize = 1 - peerID := uint16(1) address1 := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" - _, err := txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 1)), TxInfo{SenderID: peerID}) + _, err := txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 1))) require.NoError(t, err) - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 2)), TxInfo{SenderID: peerID}) + _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 2))) require.Error(t, err) require.Contains(t, err.Error(), "mempool pending set is full") } @@ -707,15 +705,13 @@ func TestTxMempool_RemoveCacheWhenPendingTxIsFull(t *testing.T) { txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 10, NopTxConstraintsFetcher) txmp.config.PendingSize = 1 - peerID := uint16(1) address1 := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" - _, err := txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 1)), TxInfo{SenderID: peerID}) + _, err := txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 1))) require.NoError(t, err) - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 2)), TxInfo{SenderID: peerID}) + _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 2))) require.Error(t, err) - txCache := txmp.cache.(*LRUTxCache) // Make sure the second tx is removed from cache - require.Equal(t, 1, len(txCache.cacheMap)) + require.Equal(t, 1, len(txmp.cache.cacheMap)) } func TestTxMempool_EVMEviction(t *testing.T) { @@ -725,30 +721,25 @@ func TestTxMempool_EVMEviction(t *testing.T) { txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 100, NopTxConstraintsFetcher) txmp.config.Size = 1 - peerID := uint16(1) address1 := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" address2 := "0xfD23B3A9DE15e92B9ef9540E587B3661E15A12fA" // Add first transaction with priority 1 - _, err := txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 0)), TxInfo{SenderID: peerID}) + _, err := txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 0))) require.NoError(t, err) // This should evict the previous tx (priority 1 < priority 2) - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 2, 0)), TxInfo{SenderID: peerID}) + _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 2, 0))) require.NoError(t, err) - require.Equal(t, 1, txmp.priorityIndex.NumTxs()) - require.Equal(t, int64(2), txmp.priorityIndex.txs[0].priority) - // Increase mempool size to 2 txmp.config.Size = 2 - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 3, 1)), TxInfo{SenderID: peerID}) + _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 3, 1))) require.NoError(t, err) require.Equal(t, 0, txmp.pendingTxs.Size()) - require.Equal(t, 2, txmp.priorityIndex.NumTxs()) // This would evict the tx with priority 2 and cause the tx with priority 3 to go pending - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address2, 4, 0)), TxInfo{SenderID: peerID}) + _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address2, 4, 0))) require.NoError(t, err) require.Eventually(t, func() bool { @@ -762,7 +753,7 @@ func TestTxMempool_EVMEviction(t *testing.T) { tx := txmp.priorityIndex.txs[0] require.Equal(t, int64(4), tx.priority) // Should be the highest priority transaction - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address2, 5, 1)), TxInfo{SenderID: peerID}) + _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address2, 5, 1))) require.NoError(t, err) require.Equal(t, 2, txmp.priorityIndex.NumTxs()) @@ -782,7 +773,6 @@ func TestTxMempool_CheckTxSamePeer(t *testing.T) { client := &application{Application: kvstore.NewApplication()} txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 100, NopTxConstraintsFetcher) - peerID := uint16(1) rng := rand.New(rand.NewSource(time.Now().UnixNano())) prefix := make([]byte, 20) @@ -791,9 +781,9 @@ func TestTxMempool_CheckTxSamePeer(t *testing.T) { tx := []byte(fmt.Sprintf("sender-0=%X=%d", prefix, 50)) - _, err = txmp.CheckTx(ctx, tx, TxInfo{SenderID: peerID}) + _, err = txmp.CheckTx(ctx, tx) require.NoError(t, err) - _, err = txmp.CheckTx(ctx, tx, TxInfo{SenderID: peerID}) + _, err = txmp.CheckTx(ctx, tx) require.Error(t, err) } @@ -829,7 +819,7 @@ func TestTxMempool_ConcurrentTxs(t *testing.T) { var height int64 = 1 for range ticker.C { - reapedTxs := txmp.ReapMaxTxs(200) + reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(200))}) if len(reapedTxs) > 0 { responses := make([]*abci.ExecTxResult, len(reapedTxs)) for i := 0; i < len(responses); i++ { @@ -878,7 +868,7 @@ func TestTxMempool_ExpiredTxs_NumBlocks(t *testing.T) { require.Equal(t, len(tTxs), txmp.Size()) // reap 5 txs at the next height -- no txs should expire - reapedTxs := txmp.ReapMaxTxs(5) + reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(5))}) responses := make([]*abci.ExecTxResult, len(reapedTxs)) for i := 0; i < len(responses); i++ { responses[i] = &abci.ExecTxResult{Code: abci.CodeTypeOK} @@ -902,7 +892,7 @@ func TestTxMempool_ExpiredTxs_NumBlocks(t *testing.T) { // cannot guarantee that all 95 txs are remaining that should be expired and // removed. However, we do know that that at most 95 txs can be expired and // removed. - reapedTxs = txmp.ReapMaxTxs(5) + reapedTxs = txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(5))}) responses = make([]*abci.ExecTxResult, len(reapedTxs)) for i := 0; i < len(responses); i++ { responses[i] = &abci.ExecTxResult{Code: abci.CodeTypeOK} @@ -940,7 +930,6 @@ func TestReapMaxBytesMaxGas_EVMFirst(t *testing.T) { client := &application{Application: kvstore.NewApplication()} txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - peerID := uint16(1) evmAddress1 := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" evmAddress2 := "0xfD23B3A9DE15e92B9ef9540E587B3661E15A12fA" @@ -957,14 +946,14 @@ func TestReapMaxBytesMaxGas_EVMFirst(t *testing.T) { } for _, tx := range txsToAdd { - _, err := txmp.CheckTx(ctx, tx, TxInfo{SenderID: peerID}) + _, err := txmp.CheckTx(ctx, tx) require.NoError(t, err) } require.Equal(t, 5, txmp.Size()) // Reap all transactions - reapedTxs := txmp.ReapMaxBytesMaxGas(utils.Max[int64](), utils.Max[int64](), utils.Max[int64]()) + reapedTxs := txmp.ReapTxs(ReapLimits{}) require.Len(t, reapedTxs, 5) // Verify EVM transactions come first, then non-EVM @@ -1008,7 +997,7 @@ func TestBlockFailedTxNotReAdmittedAfterSecondFailure(t *testing.T) { tx := types.Tx("sender-0-0=key=1000") // Submit the tx — should enter the mempool - _, err := txmp.CheckTx(ctx, tx, TxInfo{SenderID: 0}) + _, err := txmp.CheckTx(ctx, tx) require.NoError(t, err) require.Equal(t, 1, txmp.Size()) @@ -1016,14 +1005,14 @@ func TestBlockFailedTxNotReAdmittedAfterSecondFailure(t *testing.T) { txmp.Lock() require.NoError(t, txmp.Update(ctx, 1, types.Txs{tx}, []*abci.ExecTxResult{ {Code: 11}, // out of gas - }, txmp.txConstraintsFetcher, true)) + }, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) txmp.Unlock() // Tx should be removed from the mempool require.Equal(t, 0, txmp.Size()) // First failure: tx should have been removed from cache, allowing re-entry - _, err = txmp.CheckTx(ctx, tx, TxInfo{SenderID: 0}) + _, err = txmp.CheckTx(ctx, tx) require.NoError(t, err) require.Equal(t, 1, txmp.Size()) @@ -1031,19 +1020,19 @@ func TestBlockFailedTxNotReAdmittedAfterSecondFailure(t *testing.T) { txmp.Lock() require.NoError(t, txmp.Update(ctx, 2, types.Txs{tx}, []*abci.ExecTxResult{ {Code: 11}, // out of gas again - }, txmp.txConstraintsFetcher, true)) + }, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) txmp.Unlock() require.Equal(t, 0, txmp.Size()) // Second failure: tx should remain in cache — CheckTx should reject it - _, err = txmp.CheckTx(ctx, tx, TxInfo{SenderID: 0}) + _, err = txmp.CheckTx(ctx, tx) require.Equal(t, ErrTxInCache, err) require.Equal(t, 0, txmp.Size()) // A different tx (different hash) should still be admitted differentTx := types.Tx("sender-0-0=key=2000") - _, err = txmp.CheckTx(ctx, differentTx, TxInfo{SenderID: 0}) + _, err = txmp.CheckTx(ctx, differentTx) require.NoError(t, err) require.Equal(t, 1, txmp.Size()) } @@ -1059,23 +1048,23 @@ func TestBlockFailedTxTrackerClearedOnSuccess(t *testing.T) { txHash := tx.Hash() // Submit and fail once in a block - _, err := txmp.CheckTx(ctx, tx, TxInfo{SenderID: 0}) + _, err := txmp.CheckTx(ctx, tx) require.NoError(t, err) txmp.Lock() require.NoError(t, txmp.Update(ctx, 1, types.Txs{tx}, []*abci.ExecTxResult{ {Code: 11}, - }, txmp.txConstraintsFetcher, true)) + }, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) txmp.Unlock() // Re-enter the mempool (first failure allows retry) - _, err = txmp.CheckTx(ctx, tx, TxInfo{SenderID: 0}) + _, err = txmp.CheckTx(ctx, tx) require.NoError(t, err) // This time the tx succeeds in the block txmp.Lock() require.NoError(t, txmp.Update(ctx, 2, types.Txs{tx}, []*abci.ExecTxResult{ {Code: abci.CodeTypeOK}, - }, txmp.txConstraintsFetcher, true)) + }, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) txmp.Unlock() // Success clears the failure tracker. Simulate LRU eviction of the @@ -1083,18 +1072,18 @@ func TestBlockFailedTxTrackerClearedOnSuccess(t *testing.T) { txmp.cache.Remove(txHash) // Tx should now be re-admittable - _, err = txmp.CheckTx(ctx, tx, TxInfo{SenderID: 0}) + _, err = txmp.CheckTx(ctx, tx) require.NoError(t, err) // Fail again in a block — this should be treated as a fresh first failure txmp.Lock() require.NoError(t, txmp.Update(ctx, 3, types.Txs{tx}, []*abci.ExecTxResult{ {Code: 11}, - }, txmp.txConstraintsFetcher, true)) + }, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) txmp.Unlock() // First-failure grace should be restored: tx allowed to re-enter - _, err = txmp.CheckTx(ctx, tx, TxInfo{SenderID: 0}) + _, err = txmp.CheckTx(ctx, tx) require.NoError(t, err) require.Equal(t, 1, txmp.Size()) } diff --git a/sei-tendermint/internal/mempool/recheck_drain_test.go b/sei-tendermint/internal/mempool/recheck_drain_test.go index 8583954990..7698b012aa 100644 --- a/sei-tendermint/internal/mempool/recheck_drain_test.go +++ b/sei-tendermint/internal/mempool/recheck_drain_test.go @@ -129,7 +129,7 @@ func TestTxMempool_DescendingNonceDrain(t *testing.T) { // in the priority index as the evmQueue head. for n := N - 1; n >= 0; n-- { tx := []byte(fmt.Sprintf("evm=%s=%d=1", sender.Hex(), n)) - _, err := txmp.CheckTx(ctx, tx, TxInfo{}) + _, err := txmp.CheckTx(ctx, tx) require.NoError(t, err) } @@ -179,7 +179,7 @@ func TestTxMempool_EvmNextPendingNonceIncludesPendingTransactions(t *testing.T) for _, nonce := range []uint64{7, 5, 6} { tx := []byte(fmt.Sprintf("evm=%s=%d=1", sender.Hex(), nonce)) - _, err := txmp.CheckTx(ctx, tx, TxInfo{}) + _, err := txmp.CheckTx(ctx, tx) require.NoError(t, err) } @@ -199,9 +199,9 @@ func TestTxMempool_EvmNextPendingNonceReplacesSameNonceByPriority(t *testing.T) lowPriorityTx := []byte(fmt.Sprintf("evm=%s=%d=%d", sender.Hex(), 6, 1)) highPriorityTx := []byte(fmt.Sprintf("evm=%s=%d=%d", sender.Hex(), 6, 2)) - _, err := txmp.CheckTx(ctx, lowPriorityTx, TxInfo{}) + _, err := txmp.CheckTx(ctx, lowPriorityTx) require.NoError(t, err) - _, err = txmp.CheckTx(ctx, highPriorityTx, TxInfo{}) + _, err = txmp.CheckTx(ctx, highPriorityTx) require.NoError(t, err) require.Equal(t, 2, txmp.PendingSize(), "pending store keeps both txs") diff --git a/sei-tendermint/internal/p2p/giga_router_test.go b/sei-tendermint/internal/p2p/giga_router_test.go index 698f3e398d..09f124aabb 100644 --- a/sei-tendermint/internal/p2p/giga_router_test.go +++ b/sei-tendermint/internal/p2p/giga_router_test.go @@ -334,7 +334,7 @@ func TestGigaRouter_FinalizeBlocks(t *testing.T) { } s.SpawnNamed(fmt.Sprintf("producer[%v]", i), func() error { for _, payload := range txs { - if _, err := txMempool.CheckTx(ctx, payload, mempool.TxInfo{}); err != nil { + if _, err := txMempool.CheckTx(ctx, payload); err != nil { return fmt.Errorf("txMempool.CheckTx(): %w", err) } } diff --git a/sei-tendermint/internal/rpc/core/mempool.go b/sei-tendermint/internal/rpc/core/mempool.go index d4dac73096..f7f40373bb 100644 --- a/sei-tendermint/internal/rpc/core/mempool.go +++ b/sei-tendermint/internal/rpc/core/mempool.go @@ -11,6 +11,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/state/indexer" tmmath "github.com/sei-protocol/sei-chain/sei-tendermint/libs/math" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes" ) @@ -123,7 +124,9 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque skipCount := validateSkipCount(page, perPage) - txs := env.Mempool.ReapMaxTxs(skipCount + tmmath.MinInt(perPage, totalCount-skipCount)) + txs := env.Mempool.ReapTxs(mempool.ReapLimits{ + MaxTxs: utils.Some(uint64(skipCount + tmmath.MinInt(perPage, totalCount-skipCount))), + }) if skipCount > len(txs) { skipCount = len(txs) } diff --git a/sei-tendermint/internal/state/execution.go b/sei-tendermint/internal/state/execution.go index b4186b0c84..aa747eb432 100644 --- a/sei-tendermint/internal/state/execution.go +++ b/sei-tendermint/internal/state/execution.go @@ -14,6 +14,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/eventbus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types" "github.com/sei-protocol/sei-chain/sei-tendermint/types" "github.com/sei-protocol/seilog" @@ -122,7 +123,11 @@ func (blockExec *BlockExecutor) CreateProposalBlock( // Fetch a limited amount of valid txs maxDataBytes := types.MaxDataBytes(maxBytes, evSize, state.Validators.Size()) - txs := blockExec.mempool.ReapMaxBytesMaxGas(maxDataBytes, maxGasWanted, maxGas) + txs := blockExec.mempool.ReapTxs(mempool.ReapLimits{ + MaxBytes: utils.Some(maxDataBytes), + MaxGasWanted: utils.Some(maxGasWanted), + MaxGasEstimated: utils.Some(maxGas), + }) block = state.MakeBlock(height, txs, lastCommit, evidence, proposerAddr) return block, nil } diff --git a/sei-tendermint/node/node_test.go b/sei-tendermint/node/node_test.go index 8db98beedc..cd07b9a1a4 100644 --- a/sei-tendermint/node/node_test.go +++ b/sei-tendermint/node/node_test.go @@ -341,7 +341,7 @@ func TestCreateProposalBlock(t *testing.T) { txLength := 100 for i := 0; i <= maxBytes/txLength; i++ { tx := tmrand.Bytes(txLength) - _, err := mp.CheckTx(ctx, tx, mempool.TxInfo{}) + _, err := mp.CheckTx(ctx, tx) assert.NoError(t, err) } @@ -416,7 +416,7 @@ func TestMaxTxsProposalBlockSize(t *testing.T) { // fill the mempool with one txs just below the maximum size txLength := int(types.MaxDataBytesNoEvidence(maxBytes, 1)) tx := tmrand.Bytes(txLength - 4) // to account for the varint - _, err = mp.CheckTx(ctx, tx, mempool.TxInfo{}) + _, err = mp.CheckTx(ctx, tx) assert.NoError(t, err) eventBus := eventbus.NewDefault() @@ -481,13 +481,13 @@ func TestMaxProposalBlockSize(t *testing.T) { // fill the mempool with one txs just below the maximum size txLength := int(types.MaxDataBytesNoEvidence(maxBytes, types.MaxVotesCount)) tx := tmrand.Bytes(txLength - 6) // to account for the varint - _, err = mp.CheckTx(ctx, tx, mempool.TxInfo{}) + _, err = mp.CheckTx(ctx, tx) assert.NoError(t, err) // now produce more txs than what a normal block can hold with 10 smaller txs // At the end of the test, only the single big tx should be added for range 10 { tx := tmrand.Bytes(10) - _, err := mp.CheckTx(ctx, tx, mempool.TxInfo{}) + _, err := mp.CheckTx(ctx, tx) assert.NoError(t, err) } diff --git a/sei-tendermint/rpc/client/rpc_test.go b/sei-tendermint/rpc/client/rpc_test.go index 0974efda82..09764e07d9 100644 --- a/sei-tendermint/rpc/client/rpc_test.go +++ b/sei-tendermint/rpc/client/rpc_test.go @@ -445,7 +445,7 @@ func TestClientMethodCalls(t *testing.T) { require.Equal(t, initMempoolSize+1, pool.Size()) - txs := pool.ReapMaxTxs(len(tx)) + txs := pool.ReapTxs(mempool.ReapLimits{MaxTxs: utils.Some(uint64(len(tx)))}) require.Equal(t, tx, txs[0]) pool.Flush() }) @@ -594,7 +594,7 @@ func TestClientMethodCallsAdvanced(t *testing.T) { _, _, tx := MakeTxKV() txs[i] = tx - _, err := pool.CheckTx(ctx, tx, mempool.TxInfo{}) + _, err := pool.CheckTx(ctx, tx) require.NoError(t, err) ch <- nil } @@ -636,7 +636,7 @@ func TestClientMethodCallsAdvanced(t *testing.T) { _, _, tx := MakeTxKV() - _, err := pool.CheckTx(ctx, tx, mempool.TxInfo{}) + _, err := pool.CheckTx(ctx, tx) require.NoError(t, err) close(ch) diff --git a/sei-tendermint/test/fuzz/tests/mempool_test.go b/sei-tendermint/test/fuzz/tests/mempool_test.go index 2ea0f2c5f3..32bfc3f2ed 100644 --- a/sei-tendermint/test/fuzz/tests/mempool_test.go +++ b/sei-tendermint/test/fuzz/tests/mempool_test.go @@ -17,6 +17,6 @@ func FuzzMempool(f *testing.F) { mp := mempool.NewTxMempool(cfg.ToMempoolConfig(), kvstore.NewProxy(), mempool.NopMetrics(), mempool.NopTxConstraintsFetcher) f.Fuzz(func(t *testing.T, data []byte) { - _, _ = mp.CheckTx(t.Context(), data, mempool.TxInfo{}) + _, _ = mp.CheckTx(t.Context(), data) }) } From e2953827085b45fb11d7674d55a67eddab49826c Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 19 May 2026 17:43:07 +0200 Subject: [PATCH 012/100] mempool tests compile --- .../internal/mempool/mempool_test.go | 48 +++--- .../internal/mempool/reactor/reactor_test.go | 2 +- .../internal/mempool/recheck_drain_test.go | 12 +- sei-tendermint/internal/mempool/tx_test.go | 140 ++---------------- 4 files changed, 40 insertions(+), 162 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index 7eda5fd43d..2765614d29 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -157,7 +157,7 @@ func checkTxs(ctx context.Context, t *testing.T, txmp *TxMempool, numTxs int) [] priority := int64(rng.Intn(9999-1000) + 1000) txs[i] = testTx{ - tx: []byte(fmt.Sprintf("sender-%d-%d=%X=%d", i, prefix, priority)), + tx: []byte(fmt.Sprintf("sender-%d=%X=%d", i, prefix, priority)), priority: priority, } _, err = txmp.CheckTx(ctx, txs[i].tx) @@ -736,35 +736,35 @@ func TestTxMempool_EVMEviction(t *testing.T) { txmp.config.Size = 2 _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 3, 1))) require.NoError(t, err) - require.Equal(t, 0, txmp.pendingTxs.Size()) + require.Equal(t, 0, txmp.PendingSize()) // This would evict the tx with priority 2 and cause the tx with priority 3 to go pending _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address2, 4, 0))) require.NoError(t, err) require.Eventually(t, func() bool { - return txmp.priorityIndex.NumTxs() == 1 && txmp.pendingTxs.Size() == 1 + return txmp.NumTxsNotPending() == 1 && txmp.PendingSize() == 1 }, 5*time.Second, 100*time.Millisecond, "Expected mempool state not reached") // Verify final state - require.Equal(t, 1, txmp.priorityIndex.NumTxs()) - require.Equal(t, 1, txmp.pendingTxs.Size()) + require.Equal(t, 1, txmp.NumTxsNotPending()) + require.Equal(t, 1, txmp.PendingSize()) - tx := txmp.priorityIndex.txs[0] + tx := txmp.txStore.AllReady()[0] require.Equal(t, int64(4), tx.priority) // Should be the highest priority transaction _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address2, 5, 1))) require.NoError(t, err) - require.Equal(t, 2, txmp.priorityIndex.NumTxs()) + require.Equal(t, 2, txmp.NumTxsNotPending()) - txmp.removeTx(tx, true, false, true) + //TODO: txmp.removeTx(tx, true, false, true) // Should not reenqueue - require.Equal(t, 1, txmp.priorityIndex.NumTxs()) + require.Equal(t, 1, txmp.NumTxsNotPending()) require.Eventually(t, func() bool { - return txmp.pendingTxs.Size() == 1 + return txmp.PendingSize() == 1 }, 5*time.Second, 100*time.Millisecond, "Expected pendingTxs size not reached") - require.Equal(t, 1, txmp.pendingTxs.Size()) + require.Equal(t, 1, txmp.PendingSize()) } func TestTxMempool_CheckTxSamePeer(t *testing.T) { @@ -801,7 +801,7 @@ func TestTxMempool_ConcurrentTxs(t *testing.T) { wg.Add(1) go func() { for i := 0; i < 20; i++ { - _ = checkTxs(ctx, t, txmp, 100, 0) + _ = checkTxs(ctx, t, txmp, 100) dur := rng.Intn(1000-500) + 500 time.Sleep(time.Duration(dur) * time.Millisecond) } @@ -835,7 +835,7 @@ func TestTxMempool_ConcurrentTxs(t *testing.T) { } txmp.Lock() - require.NoError(t, txmp.Update(ctx, height, reapedTxs, responses, txmp.txConstraintsFetcher, true)) + require.NoError(t, txmp.Update(ctx, height, reapedTxs, responses, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) txmp.Unlock() height++ @@ -862,9 +862,9 @@ func TestTxMempool_ExpiredTxs_NumBlocks(t *testing.T) { txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 500, NopTxConstraintsFetcher) txmp.height = 100 - txmp.config.TTLNumBlocks = 10 + txmp.config.TTLNumBlocks = utils.Some(int64(10)) - tTxs := checkTxs(ctx, t, txmp, 100, 0) + tTxs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(tTxs), txmp.Size()) // reap 5 txs at the next height -- no txs should expire @@ -875,13 +875,13 @@ func TestTxMempool_ExpiredTxs_NumBlocks(t *testing.T) { } txmp.Lock() - require.NoError(t, txmp.Update(ctx, txmp.height+1, reapedTxs, responses, txmp.txConstraintsFetcher, true)) + require.NoError(t, txmp.Update(ctx, txmp.height+1, reapedTxs, responses, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) txmp.Unlock() require.Equal(t, 95, txmp.Size()) // check more txs at height 101 - _ = checkTxs(ctx, t, txmp, 50, 1) + _ = checkTxs(ctx, t, txmp, 50) require.Equal(t, 145, txmp.Size()) // Reap 5 txs at a height that would expire all the transactions from before @@ -899,7 +899,7 @@ func TestTxMempool_ExpiredTxs_NumBlocks(t *testing.T) { } txmp.Lock() - require.NoError(t, txmp.Update(ctx, txmp.height+10, reapedTxs, responses, txmp.txConstraintsFetcher, true)) + require.NoError(t, txmp.Update(ctx, txmp.height+10, reapedTxs, responses, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) txmp.Unlock() require.GreaterOrEqual(t, txmp.Size(), 45) @@ -911,15 +911,13 @@ func TestMempoolExpiration(t *testing.T) { client := &application{Application: kvstore.NewApplication()} txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - txmp.config.TTLDuration = time.Nanosecond // we want tx to expire immediately + txmp.config.TTLDuration = utils.Some(time.Nanosecond) // we want tx to expire immediately txmp.config.RemoveExpiredTxsFromQueue = true - txs := checkTxs(ctx, t, txmp, 100, 0) - require.Equal(t, len(txs), txmp.priorityIndex.Len()) - require.Equal(t, len(txs), txmp.txStore.Size()) + txs := checkTxs(ctx, t, txmp, 100) + require.Equal(t, len(txs), txmp.Size()) time.Sleep(time.Millisecond) - txmp.purgeExpiredTxs(txmp.height) - require.Equal(t, 0, txmp.priorityIndex.Len()) - require.Equal(t, 0, txmp.txStore.Size()) + //txmp.purgeExpiredTxs(txmp.height) + require.Equal(t, 0, txmp.Size()) } // TestReapMaxBytesMaxGas_EVMFirst verifies that ReapMaxBytesMaxGas returns diff --git a/sei-tendermint/internal/mempool/reactor/reactor_test.go b/sei-tendermint/internal/mempool/reactor/reactor_test.go index f937ac6bea..ebadde6512 100644 --- a/sei-tendermint/internal/mempool/reactor/reactor_test.go +++ b/sei-tendermint/internal/mempool/reactor/reactor_test.go @@ -65,7 +65,7 @@ func checkTxs(ctx context.Context, t *testing.T, txmp *mempool.TxMempool, numTxs require.NoError(t, err) txs[i] = testTx{ - tx: []byte(fmt.Sprintf("sender-%d-%d=%X=%d", i, prefix, i+1000)), + tx: []byte(fmt.Sprintf("sender-%d=%X=%d", i, prefix, i+1000)), } _, err = txmp.CheckTx(ctx, txs[i].tx) require.NoError(t, err) diff --git a/sei-tendermint/internal/mempool/recheck_drain_test.go b/sei-tendermint/internal/mempool/recheck_drain_test.go index 7698b012aa..28a71365bd 100644 --- a/sei-tendermint/internal/mempool/recheck_drain_test.go +++ b/sei-tendermint/internal/mempool/recheck_drain_test.go @@ -16,7 +16,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" - "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) // evmNonceApp models a Sei-like EVM antehandler for mempool tests: @@ -145,11 +144,8 @@ func TestTxMempool_DescendingNonceDrain(t *testing.T) { const maxBlocks = 5 totalMined := 0 for height := int64(1); txmp.Size() > 0 && height <= maxBlocks; height++ { - txs, _ := txmp.PopTxs(ReapLimits{ + txs, _ := txmp.ReapTxsAndMark(ReapLimits{ MaxTxs: utils.Some(uint64(N)), - MaxBytes: utils.Some(int64(1 << 30)), - MaxGasWanted: utils.Some(int64(1 << 30)), - MaxGasEstimated: utils.Some(int64(1 << 30)), }) require.NotEmpty(t, txs, "PopTxs returned no txs at height %d (mempool stalled)", height) @@ -161,7 +157,7 @@ func TestTxMempool_DescendingNonceDrain(t *testing.T) { totalMined += len(txs) // recheck=false matches the post-fix Autobahn path and CometBFT's default. - require.NoError(t, txmp.Update(ctx, height, txs, txResults, NopTxConstraintsFetcher, false)) + require.NoError(t, txmp.Update(ctx, height, txs, txResults, utils.OrPanic1(NopTxConstraintsFetcher()), false)) } require.Equal(t, N, totalMined, "all N txs should have mined within %d blocks", maxBlocks) @@ -205,10 +201,10 @@ func TestTxMempool_EvmNextPendingNonceReplacesSameNonceByPriority(t *testing.T) require.NoError(t, err) require.Equal(t, 2, txmp.PendingSize(), "pending store keeps both txs") - for byAddrNonce := range txmp.byAddrNonce.Lock() { + /*for byAddrNonce := range txmp.byAddrNonce.Lock() { wtx, ok := byAddrNonce[evmAddrNonce{Address: sender, Nonce: 6}] require.True(t, ok, "nonce bookkeeping should track one occupied nonce") require.Equal(t, types.Tx(highPriorityTx).Hash(), wtx.Hash()) - } + }*/ require.Equal(t, uint64(5), txmp.EvmNextPendingNonce(sender)) } diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index f2c27b7701..bc34153aaa 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -10,7 +10,7 @@ import ( ) func TestTxStore_GetTxByHash(t *testing.T) { - txs := NewTxStore() + txs := NewTxStore(TestConfig()) wtx := &WrappedTx{ hashedTx: newHashedTx(types.Tx("test_tx")), priority: 1, @@ -18,18 +18,18 @@ func TestTxStore_GetTxByHash(t *testing.T) { } key := wtx.Hash() - res := txs.GetTxByHash(key) + res := txs.ByHash(key) require.Nil(t, res) - txs.SetTx(wtx) + txs.Insert(wtx) - res = txs.GetTxByHash(key) + res = txs.ByHash(key) require.NotNil(t, res) require.Equal(t, wtx, res) } func TestTxStore_SetTx(t *testing.T) { - txs := NewTxStore() + txs := NewTxStore(TestConfig()) wtx := &WrappedTx{ hashedTx: newHashedTx(types.Tx("test_tx")), priority: 1, @@ -37,145 +37,28 @@ func TestTxStore_SetTx(t *testing.T) { } key := wtx.Hash() - txs.SetTx(wtx) + txs.Insert(wtx) - res := txs.GetTxByHash(key) + res := txs.ByHash(key) require.NotNil(t, res) require.Equal(t, wtx, res) } -func TestTxStore_IsTxRemoved(t *testing.T) { - // Initialize the store - txs := NewTxStore() - - // Current time for timestamping transactions - now := time.Now() - - // Tests setup as a slice of anonymous structs - tests := []struct { - name string - wtx *WrappedTx - setup func(*TxStore, *WrappedTx) // Optional setup function to manipulate store state - wantRemoved bool - }{ - { - name: "Existing transaction not removed", - wtx: &WrappedTx{ - hashedTx: newHashedTx(types.Tx("tx_hash_1")), - removed: false, - timestamp: now, - }, - setup: func(ts *TxStore, w *WrappedTx) { - ts.SetTx(w) - }, - wantRemoved: false, - }, - { - name: "Existing transaction marked as removed", - wtx: &WrappedTx{ - hashedTx: newHashedTx(types.Tx("tx_hash_2")), - removed: true, - timestamp: now, - }, - setup: func(ts *TxStore, w *WrappedTx) { - ts.SetTx(w) - }, - wantRemoved: true, - }, - { - name: "Non-existing transaction", - wtx: &WrappedTx{ - hashedTx: newHashedTx(types.Tx("tx_hash_3")), - removed: false, - timestamp: now, - }, - wantRemoved: false, - }, - { - name: "Non-existing transaction but marked as removed", - wtx: &WrappedTx{ - hashedTx: newHashedTx(types.Tx("tx_hash_4")), - removed: true, - timestamp: now, - }, - wantRemoved: true, - }, - } - - // Execute test scenarios - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.setup != nil { - tt.setup(txs, tt.wtx) - } - removed := txs.IsTxRemoved(tt.wtx) - require.Equal(t, tt.wantRemoved, removed) - }) - } -} - -func TestTxStore_GetOrSetPeerByTxHash(t *testing.T) { - txs := NewTxStore() - wtx := &WrappedTx{ - hashedTx: newHashedTx(types.Tx("test_tx")), - priority: 1, - timestamp: time.Now(), - } - - key := wtx.Hash() - txs.SetTx(wtx) - - res, ok := txs.GetOrSetPeerByTxHash(types.Tx([]byte("test_tx_2")).Hash(), 15) - require.Nil(t, res) - require.False(t, ok) - - res, ok = txs.GetOrSetPeerByTxHash(key, 15) - require.NotNil(t, res) - require.False(t, ok) - - res, ok = txs.GetOrSetPeerByTxHash(key, 15) - require.NotNil(t, res) - require.True(t, ok) - - require.True(t, txs.TxHasPeer(key, 15)) - require.False(t, txs.TxHasPeer(key, 16)) -} - -func TestTxStore_RemoveTx(t *testing.T) { - txs := NewTxStore() - wtx := &WrappedTx{ - hashedTx: newHashedTx(types.Tx("test_tx")), - priority: 1, - timestamp: time.Now(), - } - - txs.SetTx(wtx) - - key := wtx.Hash() - res := txs.GetTxByHash(key) - require.NotNil(t, res) - - txs.RemoveTx(res) - - res = txs.GetTxByHash(key) - require.Nil(t, res) -} - func TestTxStore_Size(t *testing.T) { - txStore := NewTxStore() + txStore := NewTxStore(TestConfig()) numTxs := 1000 for i := range numTxs { - txStore.SetTx(&WrappedTx{ + txStore.Insert(&WrappedTx{ hashedTx: newHashedTx(fmt.Appendf(nil, "test_tx_%d", i)), priority: int64(i), timestamp: time.Now(), }) } - require.Equal(t, numTxs, txStore.Size()) + require.Equal(t, numTxs, txStore.State().total.count) } - +/* func TestPendingTxsPopTxsGood(t *testing.T) { pendingTxs := NewPendingTxs(DefaultConfig()) for _, test := range []struct { @@ -325,3 +208,4 @@ func TestPendingTxs_InsertCondition(t *testing.T) { err = pendingTxs.Insert(tx3) require.NotNil(t, err) } +*/ From c2d3f0673d0bb1a5ee00a7a2c78281edd5d55196 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 19 May 2026 17:46:38 +0200 Subject: [PATCH 013/100] wip --- sei-tendermint/internal/rpc/core/mempool.go | 11 ++++++----- sei-tendermint/internal/state/tx_filter_test.go | 3 +-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sei-tendermint/internal/rpc/core/mempool.go b/sei-tendermint/internal/rpc/core/mempool.go index f7f40373bb..adcd201cad 100644 --- a/sei-tendermint/internal/rpc/core/mempool.go +++ b/sei-tendermint/internal/rpc/core/mempool.go @@ -24,7 +24,7 @@ import ( // https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_async // Deprecated and should be removed in 0.37 func (env *Environment) BroadcastTxAsync(ctx context.Context, req *coretypes.RequestBroadcastTx) (*coretypes.ResultBroadcastTx, error) { - go func() { _, _ = env.Mempool.CheckTx(ctx, req.Tx, mempool.TxInfo{}) }() + go func() { _, _ = env.Mempool.CheckTx(ctx, req.Tx) }() return &coretypes.ResultBroadcastTx{Hash: req.Tx.Hash().Bytes()}, nil } @@ -38,7 +38,7 @@ func (env *Environment) BroadcastTxSync(ctx context.Context, req *coretypes.Requ // DeliverTx result. // More: https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_sync func (env *Environment) BroadcastTx(ctx context.Context, req *coretypes.RequestBroadcastTx) (*coretypes.ResultBroadcastTx, error) { - r, err := env.Mempool.CheckTx(ctx, req.Tx, mempool.TxInfo{}) + r, err := env.Mempool.CheckTx(ctx, req.Tx) if err != nil { return nil, err } @@ -54,7 +54,7 @@ func (env *Environment) BroadcastTx(ctx context.Context, req *coretypes.RequestB // BroadcastTxCommit returns with the responses from CheckTx and DeliverTx. // More: https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_commit func (env *Environment) BroadcastTxCommit(ctx context.Context, req *coretypes.RequestBroadcastTx) (*coretypes.ResultBroadcastTxCommit, error) { - r, err := env.Mempool.CheckTx(ctx, req.Tx, mempool.TxInfo{}) + r, err := env.Mempool.CheckTx(ctx, req.Tx) if err != nil { return nil, err } @@ -135,7 +135,7 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque return &coretypes.ResultUnconfirmedTxs{ Count: len(result), Total: totalCount, - TotalBytes: env.Mempool.SizeBytes(), + TotalBytes: utils.Clamp[int64](env.Mempool.SizeBytes()), Txs: result, }, nil } @@ -146,7 +146,8 @@ func (env *Environment) NumUnconfirmedTxs(ctx context.Context) (*coretypes.Resul return &coretypes.ResultUnconfirmedTxs{ Count: env.Mempool.Size(), Total: env.Mempool.Size(), - TotalBytes: env.Mempool.SizeBytes()}, nil + TotalBytes: utils.Clamp[int64](env.Mempool.SizeBytes()), + }, nil } // CheckTx checks the transaction without executing it. The transaction won't diff --git a/sei-tendermint/internal/state/tx_filter_test.go b/sei-tendermint/internal/state/tx_filter_test.go index 653de9a687..9a37d0fc3a 100644 --- a/sei-tendermint/internal/state/tx_filter_test.go +++ b/sei-tendermint/internal/state/tx_filter_test.go @@ -31,8 +31,7 @@ func TestTxFilter(t *testing.T) { state, err := sm.MakeGenesisState(genDoc) require.NoError(t, err) - f := sm.TxConstraintsFetcherForState(state) - constraints, err := f() + constraints := sm.TxConstraintsForState(state) require.NoError(t, err) txSize := types.ComputeProtoSizeForTxs([]types.Tx{tc.tx}) if tc.isErr { From bdcd0a22a2c08585dfa852ff86e993bb2985de65 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 12:50:20 +0200 Subject: [PATCH 014/100] codex WIP --- sei-tendermint/internal/mempool/mempool.go | 4 +- .../internal/mempool/mempool_test.go | 79 +------- sei-tendermint/internal/mempool/tx.go | 4 +- sei-tendermint/internal/mempool/tx_test.go | 177 +++--------------- 4 files changed, 36 insertions(+), 228 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 92354888fe..56be988112 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -216,7 +216,7 @@ func NewTxMempool( txsAvailable: make(chan struct{}, 1), height: -1, metrics: metrics, - txStore: NewTxStore(cfg), + txStore: NewTxStore(cfg,app), txConstraintsFetcher: txConstraintsFetcher, priorityReservoir: reservoir.New[int64](cfg.DropPriorityReservoirSize, cfg.DropPriorityThreshold, nil), // Use non-deterministic RNG cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), @@ -432,7 +432,7 @@ func (txmp *TxMempool) SafeGetTxsForHashes(txHashes []types.TxHash) (types.Txs, func (txmp *TxMempool) Flush() { txmp.mtx.Lock() defer txmp.mtx.Unlock() - txmp.txStore = NewTxStore(txmp.config) + txmp.txStore = NewTxStore(txmp.config,txmp.app) txmp.cache.Reset() } diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index 2765614d29..3cc973dc66 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -42,6 +42,14 @@ type testTx struct { var DefaultGasEstimated = int64(1) var DefaultGasWanted = int64(1) +func (app *application) EvmNonce(common.Address) uint64 { + return 0 +} + +func (app *application) EvmBalance(common.Address, []byte) *big.Int { + return big.NewInt(0) +} + func (app *application) CheckTx(_ context.Context, req *abci.RequestCheckTxV2) *abci.ResponseCheckTxV2 { var priority int64 @@ -546,77 +554,6 @@ func TestTxMempool_CheckTxExceedsMaxSize(t *testing.T) { require.NoError(t, err) } -func TestTxMempool_Reap_SkipGasUnfitAndCollectMinTxs(t *testing.T) { - ctx := t.Context() - - app := &application{Application: kvstore.NewApplication()} - client := app - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - - // Insert one high-priority tx that is unfit by gas (exceeds maxGasEstimated) - gwBig := int64(100) - geBig := int64(100) - app.gasWanted = &gwBig - app.gasEstimated = &geBig - bigTx := []byte(fmt.Sprintf("sender-big=key=%d", 1000000)) - _, err := txmp.CheckTx(ctx, bigTx) - require.NoError(t, err) - - // Now insert many small, lower-priority txs that fit well under the gas limit - gwSmall := int64(1) - geSmall := int64(1) - app.gasWanted = &gwSmall - app.gasEstimated = &geSmall - for i := 0; i < 50; i++ { - tx := []byte(fmt.Sprintf("sender-%d=key=%d", i, 1000-i)) - _, err := txmp.CheckTx(ctx, tx) - require.NoError(t, err) - } - - // Reap with a maxGasEstimated that makes the first tx unfit but allows many small txs - reaped := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(50))}) - require.Len(t, reaped, MinTxsToPeek) - - // Ensure all reaped small txs are under gas constraint - for _, rtx := range reaped { - _ = rtx // gas constraints are enforced by ReapMaxBytesMaxGas; count assertion suffices here - } -} - -func TestTxMempool_Reap_SkipGasUnfitStopsAtMinEvenWithCapacity(t *testing.T) { - ctx := t.Context() - - app := &application{Application: kvstore.NewApplication()} - client := app - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - - // First tx: unfit by gas (bigger than limit), highest priority - gwBig := int64(100) - geBig := int64(100) - app.gasWanted = &gwBig - app.gasEstimated = &geBig - bigTx := []byte(fmt.Sprintf("sender-big=key=%d", 1000000)) - _, err := txmp.CheckTx(ctx, bigTx) - require.NoError(t, err) - - // Insert many small txs that fit; plenty of capacity for more than 10 - gwSmall := int64(1) - geSmall := int64(1) - app.gasWanted = &gwSmall - app.gasEstimated = &geSmall - for i := 0; i < 100; i++ { - tx := []byte(fmt.Sprintf("sender-sm-%d=key=%d", i, 2000-i)) - _, err := txmp.CheckTx(ctx, tx) - require.NoError(t, err) - } - - // Make the gas limit very small so the first (big) tx is unfit and we only collect MinTxsPerBlock - reaped := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(10))}) - require.Len(t, reaped, MinTxsToPeek) -} - func TestTxMempool_Prioritization(t *testing.T) { ctx := t.Context() diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 7a6503731f..ae1683a9aa 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -126,11 +126,12 @@ type txStore struct { readyTxs *clist.CList[types.Tx] } -func NewTxStore(config *Config) *txStore { +func NewTxStore(config *Config, app *proxy.Proxy) *txStore { softLimit := txCounter{count: config.Size, bytes: utils.Clamp[uint64](config.MaxTxsBytes)} hardLimit := txCounter{count: 2 * softLimit.count, bytes: 2 * softLimit.bytes} inner := &txStoreInner{ byHash: map[types.TxHash]*WrappedTx{}, + byNonce: map[evmAddrNonce]*WrappedTx{}, accounts: map[common.Address]*evmAccount{}, softLimit: softLimit, hardLimit: hardLimit, @@ -138,6 +139,7 @@ func NewTxStore(config *Config) *txStore { } return &txStore{ config: config, + app: app, inner: utils.NewRWMutex(inner), readyTxs: clist.New[types.Tx](), state: inner.state.Subscribe(), diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index bc34153aaa..4640ce4a43 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -2,15 +2,35 @@ package mempool import ( "fmt" + "math/big" "testing" "time" + "github.com/ethereum/go-ethereum/common" + abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) +type txStoreTestApp struct { + abci.BaseApplication +} + +func (txStoreTestApp) EvmNonce(common.Address) uint64 { + return 0 +} + +func (txStoreTestApp) EvmBalance(common.Address, []byte) *big.Int { + return big.NewInt(0) +} + +func newTxStoreForTest() *txStore { + return NewTxStore(TestConfig(), proxy.New(txStoreTestApp{}, proxy.NopMetrics())) +} + func TestTxStore_GetTxByHash(t *testing.T) { - txs := NewTxStore(TestConfig()) + txs := newTxStoreForTest() wtx := &WrappedTx{ hashedTx: newHashedTx(types.Tx("test_tx")), priority: 1, @@ -29,7 +49,7 @@ func TestTxStore_GetTxByHash(t *testing.T) { } func TestTxStore_SetTx(t *testing.T) { - txs := NewTxStore(TestConfig()) + txs := newTxStoreForTest() wtx := &WrappedTx{ hashedTx: newHashedTx(types.Tx("test_tx")), priority: 1, @@ -45,7 +65,7 @@ func TestTxStore_SetTx(t *testing.T) { } func TestTxStore_Size(t *testing.T) { - txStore := NewTxStore(TestConfig()) + txStore := newTxStoreForTest() numTxs := 1000 for i := range numTxs { @@ -58,154 +78,3 @@ func TestTxStore_Size(t *testing.T) { require.Equal(t, numTxs, txStore.State().total.count) } -/* -func TestPendingTxsPopTxsGood(t *testing.T) { - pendingTxs := NewPendingTxs(DefaultConfig()) - for _, test := range []struct { - origLen int - popIndices []int - expected []int - }{ - { - origLen: 1, - popIndices: []int{}, - expected: []int{0}, - }, { - origLen: 1, - popIndices: []int{0}, - expected: []int{}, - }, { - origLen: 2, - popIndices: []int{0}, - expected: []int{1}, - }, { - origLen: 2, - popIndices: []int{1}, - expected: []int{0}, - }, { - origLen: 2, - popIndices: []int{0, 1}, - expected: []int{}, - }, { - origLen: 3, - popIndices: []int{1}, - expected: []int{0, 2}, - }, { - origLen: 3, - popIndices: []int{0, 2}, - expected: []int{1}, - }, { - origLen: 3, - popIndices: []int{0, 1, 2}, - expected: []int{}, - }, { - origLen: 5, - popIndices: []int{0, 1, 4}, - expected: []int{2, 3}, - }, { - origLen: 5, - popIndices: []int{1, 3}, - expected: []int{0, 2, 4}, - }, - } { - for inner := range pendingTxs.inner.Lock() { - inner.txs = []*WrappedTx{} - pendingTxs.sizeBytes.Store(0) - for i := 0; i < test.origLen; i++ { - inner.txs = append(inner.txs, &WrappedTx{ - hashedTx: newHashedTx(types.Tx{byte(i)}), - peers: map[uint16]struct{}{uint16(i): {}}, - }) - } - pendingTxs.popTxsAtIndices(inner, test.popIndices) - require.Equal(t, len(test.expected), len(inner.txs)) - for i, e := range test.expected { - _, ok := inner.txs[i].peers[uint16(e)] - require.True(t, ok) - } - } - } -} - -func TestPendingTxsPopTxsBad(t *testing.T) { - pendingTxs := NewPendingTxs(DefaultConfig()) - // out of range - require.Panics(t, func() { - for inner := range pendingTxs.inner.Lock() { - pendingTxs.popTxsAtIndices(inner, []int{0}) - } - }) - // out of order - for inner := range pendingTxs.inner.Lock() { - inner.txs = []*WrappedTx{{}, {}, {}} - } - require.Panics(t, func() { - for inner := range pendingTxs.inner.Lock() { - pendingTxs.popTxsAtIndices(inner, []int{1, 0}) - } - }) - // duplicate - require.Panics(t, func() { - for inner := range pendingTxs.inner.Lock() { - pendingTxs.popTxsAtIndices(inner, []int{2, 2}) - } - }) -} - -func TestPendingTxs_InsertCondition(t *testing.T) { - mempoolCfg := DefaultConfig() - - // First test exceeding number of txs - mempoolCfg.PendingSize = 2 - - pendingTxs := NewPendingTxs(mempoolCfg) - - // Transaction setup - tx1 := &WrappedTx{ - hashedTx: newHashedTx(types.Tx("tx1_data")), - priority: 1, - } - tx1Size := tx1.Size() - - tx2 := &WrappedTx{ - hashedTx: newHashedTx(types.Tx("tx2_data")), - priority: 2, - } - tx2Size := tx2.Size() - - err := pendingTxs.Insert(tx1) - require.Nil(t, err) - - err = pendingTxs.Insert(tx2) - require.Nil(t, err) - - // Should fail due to pending store size limit - tx3 := &WrappedTx{ - hashedTx: newHashedTx(types.Tx("tx3_data_exceeding_pending_size")), - priority: 3, - } - - err = pendingTxs.Insert(tx3) - require.NotNil(t, err) - - // Second test exceeding byte size condition - mempoolCfg.PendingSize = 5 - pendingTxs = NewPendingTxs(mempoolCfg) - mempoolCfg.MaxPendingTxsBytes = int64(tx1Size + tx2Size) - - err = pendingTxs.Insert(tx1) - require.Nil(t, err) - - err = pendingTxs.Insert(tx2) - require.Nil(t, err) - - // Should fail due to exceeding max pending transaction bytes - tx3 = &WrappedTx{ - hashedTx: newHashedTx(types.Tx("tx3_small_but_exceeds_byte_limit")), - priority: 3, - } - - err = pendingTxs.Insert(tx3) - require.NotNil(t, err) -} -*/ From b7f02dc23f2ca356d96816dcd66f89850f70fe8c Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 13:18:12 +0200 Subject: [PATCH 015/100] codex WIP --- sei-tendermint/internal/mempool/mempool.go | 10 +-- .../internal/mempool/mempool_bench_test.go | 3 +- .../internal/mempool/mempool_test.go | 65 ++++++++++++++----- 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 56be988112..b5bf5257f6 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -216,7 +216,7 @@ func NewTxMempool( txsAvailable: make(chan struct{}, 1), height: -1, metrics: metrics, - txStore: NewTxStore(cfg,app), + txStore: NewTxStore(cfg, app), txConstraintsFetcher: txConstraintsFetcher, priorityReservoir: reservoir.New[int64](cfg.DropPriorityReservoirSize, cfg.DropPriorityThreshold, nil), // Use non-deterministic RNG cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), @@ -351,9 +351,9 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response txmp.metrics.NumberOfSuccessfulCheckTxs.Add(1) txmp.metrics.observeCheckTxPriorityDistribution(res.Priority, false, "", false) - // if the tx doesn't have a gas estimate, fallback to gas wanted + // Normalize the estimate. estimatedGas := res.GasEstimated - if estimatedGas >= MinGasEVMTx && estimatedGas <= res.GasWanted { + if estimatedGas < MinGasEVMTx || estimatedGas > res.GasWanted { estimatedGas = res.GasWanted } wtx := &WrappedTx{ @@ -432,7 +432,7 @@ func (txmp *TxMempool) SafeGetTxsForHashes(txHashes []types.TxHash) (types.Txs, func (txmp *TxMempool) Flush() { txmp.mtx.Lock() defer txmp.mtx.Unlock() - txmp.txStore = NewTxStore(txmp.config,txmp.app) + txmp.txStore = NewTxStore(txmp.config, txmp.app) txmp.cache.Reset() } @@ -512,7 +512,7 @@ func (txmp *TxMempool) Update( Type: abci.CheckTxTypeV2Recheck, }) // If recheck fails, just remove the tx. - if err != nil || res.IsOK() { + if err != nil || !res.IsOK() { txHashes[wtx.Hash()] = struct{}{} } else { newPriorities[wtx.Hash()] = res.Priority diff --git a/sei-tendermint/internal/mempool/mempool_bench_test.go b/sei-tendermint/internal/mempool/mempool_bench_test.go index 84ce7f5b53..614a770b69 100644 --- a/sei-tendermint/internal/mempool/mempool_bench_test.go +++ b/sei-tendermint/internal/mempool/mempool_bench_test.go @@ -6,10 +6,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - "github.com/sei-protocol/sei-chain/sei-tendermint/abci/example/kvstore" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" ) func BenchmarkTxMempool_CheckTx(b *testing.B) { diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index 3cc973dc66..1532ab7d9c 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -14,7 +14,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/stretchr/testify/require" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" "github.com/sei-protocol/sei-chain/sei-tendermint/abci/example/code" @@ -22,6 +21,7 @@ import ( abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) @@ -185,6 +185,36 @@ func convertTex(in []testTx) types.Txs { return out } +func totalTxSizeBytes(txs []testTx) uint64 { + var total uint64 + for _, tx := range txs { + total += uint64(len(tx.tx)) + } + return total +} + +func totalRawTxSizeBytes(txs []types.Tx) uint64 { + var total uint64 + for _, tx := range txs { + total += uint64(len(tx)) + } + return total +} + +func expectedReapCountByBytes(txs []testTx, maxBytes int64) int { + var total int64 + count := 0 + for _, tx := range txs { + txSize := types.ComputeProtoSizeForTxs([]types.Tx{tx.tx}) + if maxBytes-total < txSize { + break + } + total += txSize + count++ + } + return count +} + func TestTxMempool_TxsAvailable(t *testing.T) { ctx := t.Context() @@ -251,7 +281,7 @@ func TestTxMempool_Size(t *testing.T) { txs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(txs), txmp.Size()) require.Equal(t, 0, txmp.PendingSize()) - require.Equal(t, int64(5690), txmp.SizeBytes()) + require.Equal(t, totalTxSizeBytes(txs), txmp.SizeBytes()) rawTxs := make([]types.Tx, len(txs)) for i, tx := range txs { @@ -268,7 +298,7 @@ func TestTxMempool_Size(t *testing.T) { txmp.Unlock() require.Equal(t, len(rawTxs)/2, txmp.Size()) - require.Equal(t, int64(2850), txmp.SizeBytes()) + require.Equal(t, totalRawTxSizeBytes(rawTxs[50:]), txmp.SizeBytes()) } func TestTxMempool_Flush(t *testing.T) { @@ -279,7 +309,7 @@ func TestTxMempool_Flush(t *testing.T) { txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) txs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(txs), txmp.Size()) - require.Equal(t, int64(5690), txmp.SizeBytes()) + require.Equal(t, totalTxSizeBytes(txs), txmp.SizeBytes()) rawTxs := make([]types.Tx, len(txs)) for i, tx := range txs { @@ -297,7 +327,7 @@ func TestTxMempool_Flush(t *testing.T) { txmp.Flush() require.Zero(t, txmp.Size()) - require.Equal(t, int64(0), txmp.SizeBytes()) + require.Zero(t, txmp.SizeBytes()) } func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { @@ -309,7 +339,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) tTxs := checkTxs(ctx, t, txmp, 100) // all txs request 1 gas unit require.Equal(t, len(tTxs), txmp.Size()) - require.Equal(t, int64(5690), txmp.SizeBytes()) + require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) txMap := make(map[types.TxHash]testTx) priorities := make([]int64, len(tTxs)) @@ -323,6 +353,11 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { return priorities[i] > priorities[j] }) + sortedTxs := append([]testTx(nil), tTxs...) + sort.Slice(sortedTxs, func(i, j int) bool { + return sortedTxs[i].priority > sortedTxs[j].priority + }) + ensurePrioritized := func(reapedTxs types.Txs) { reapedPriorities := make([]int64, len(reapedTxs)) for i, rTx := range reapedTxs { @@ -341,7 +376,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { reapedTxs := txmp.ReapTxs(ReapLimits{MaxGasWanted: utils.Some(int64(50))}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) - require.Equal(t, int64(5690), txmp.SizeBytes()) + require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) require.Len(t, reapedTxs, 50) }() @@ -352,8 +387,8 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { reapedTxs := txmp.ReapTxs(ReapLimits{MaxBytes: utils.Some(int64(1000))}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) - require.Equal(t, int64(5690), txmp.SizeBytes()) - require.GreaterOrEqual(t, len(reapedTxs), 16) + require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) + require.Len(t, reapedTxs, expectedReapCountByBytes(sortedTxs, 1000)) }() // Reap by both transaction bytes and gas, where the size yields 31 reaped @@ -367,8 +402,8 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { }) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) - require.Equal(t, int64(5690), txmp.SizeBytes()) - require.Len(t, reapedTxs, 25) + require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) + require.Len(t, reapedTxs, min(expectedReapCountByBytes(sortedTxs, 1500), 30)) }() // Reap by min transactions in block regardless of gas limit. @@ -446,7 +481,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) tTxs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(tTxs), txmp.Size()) - require.Equal(t, int64(5690), txmp.SizeBytes()) + require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) txMap := make(map[types.TxHash]testTx) priorities := make([]int64, len(tTxs)) @@ -478,7 +513,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { reapedTxs := txmp.ReapTxs(ReapLimits{}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) - require.Equal(t, int64(5690), txmp.SizeBytes()) + require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) require.Len(t, reapedTxs, len(tTxs)) }() @@ -489,7 +524,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(1))}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) - require.Equal(t, int64(5690), txmp.SizeBytes()) + require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) require.Len(t, reapedTxs, 1) }() @@ -500,7 +535,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(len(tTxs) / 2))}) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) - require.Equal(t, int64(5690), txmp.SizeBytes()) + require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) require.Len(t, reapedTxs, len(tTxs)/2) }() From 9f0f768b4241f77fbca2bcdf833feea409e91712 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 13:54:57 +0200 Subject: [PATCH 016/100] codex WIP --- sei-tendermint/internal/mempool/mempool.go | 5 +- .../internal/mempool/mempool_test.go | 78 +++++++++---------- sei-tendermint/internal/mempool/tx.go | 36 ++++++--- 3 files changed, 65 insertions(+), 54 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index b5bf5257f6..4bb7a635c4 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -395,7 +395,10 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response return nil, err } - txmp.txStore.Insert(wtx) + if err := txmp.txStore.Insert(wtx); err!=nil { + txmp.cache.Remove(wtx.Hash()) + return nil, err + } txmp.metrics.InsertedTxs.Add(1) txmp.metrics.TxSizeBytes.Add(float64(wtx.Size())) diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index 1532ab7d9c..1bebb30c39 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -622,15 +622,6 @@ func TestTxMempool_Prioritization(t *testing.T) { rng.Shuffle(len(txsCopy), func(i, j int) { txsCopy[i], txsCopy[j] = txsCopy[j], txsCopy[i] }) - txs = [][]byte{ - []byte(fmt.Sprintf("sender-0-1=peer=%d", 9)), - []byte(fmt.Sprintf("sender-1-1=peer=%d", 8)), - []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 7, 0)), - []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 9, 1)), - []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address2, 6, 0)), - []byte(fmt.Sprintf("sender-2-1=peer=%d", 5)), - []byte(fmt.Sprintf("sender-3-1=peer=%d", 4)), - } txsCopy = append(txsCopy, evmTxs...) for i := range txsCopy { @@ -638,52 +629,53 @@ func TestTxMempool_Prioritization(t *testing.T) { require.NoError(t, err) } - // Reap the transactions - reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(len(txs)))}) - // Check if the reaped transactions are in the correct order of their priorities - for _, tx := range txs { - fmt.Printf("expected: %s\n", string(tx)) - } - fmt.Println("**************") - for _, reapedTx := range reapedTxs { - fmt.Printf("received: %s\n", string(reapedTx)) - } - for i, reapedTx := range reapedTxs { - require.Equal(t, txs[i], []byte(reapedTx)) + expectedReapedTxs := types.Txs{ + []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 7, 0)), + []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 9, 1)), + []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address2, 6, 0)), + []byte(fmt.Sprintf("sender-0-1=peer=%d", 9)), + []byte(fmt.Sprintf("sender-1-1=peer=%d", 8)), + []byte(fmt.Sprintf("sender-2-1=peer=%d", 5)), + []byte(fmt.Sprintf("sender-3-1=peer=%d", 4)), } + + reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(len(expectedReapedTxs)))}) + require.Equal(t, expectedReapedTxs, reapedTxs) } -func TestTxMempool_PendingStoreSize(t *testing.T) { +func TestTxMempool_RemoveCacheWhenPendingTxIsFull(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 100, NopTxConstraintsFetcher) - txmp.config.PendingSize = 1 - - address1 := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" + cfg := TestConfig() + cfg.CacheSize = 10 + cfg.Size = 1 + cfg.PendingSize = 0 + txmp := NewTxMempool(cfg, proxy.New(client, proxy.NopMetrics()), NopMetrics(), NopTxConstraintsFetcher) - _, err := txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 1))) + firstTx := []byte("sender-0=peer=100") + _, err := txmp.CheckTx(ctx, firstTx) require.NoError(t, err) - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 2))) - require.Error(t, err) - require.Contains(t, err.Error(), "mempool pending set is full") -} -func TestTxMempool_RemoveCacheWhenPendingTxIsFull(t *testing.T) { - ctx := t.Context() + // The store only reports mempool-full once insertion crosses the hard limit + // and compaction drops the newly inserted low-priority tx. + _, err = txmp.CheckTx(ctx, []byte("sender-1=peer=50")) + require.NoError(t, err) - client := &application{Application: kvstore.NewApplication()} + rejectedTx := []byte("sender-2=peer=1") + _, err = txmp.CheckTx(ctx, rejectedTx) + require.ErrorIs(t, err, errMempoolFull) - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 10, NopTxConstraintsFetcher) - txmp.config.PendingSize = 1 - address1 := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" - _, err := txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 1))) - require.NoError(t, err) - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 2))) - require.Error(t, err) - // Make sure the second tx is removed from cache - require.Equal(t, 1, len(txmp.cache.cacheMap)) + require.Equal(t, 1, txmp.Size()) + // The rejected transaction should be removed from cache so it can be retried later. + _, rejectedInCache := txmp.cache.cacheMap[txmp.cache.toCacheKey(types.Tx(rejectedTx).Hash())] + require.False(t, rejectedInCache) + + _, err = txmp.CheckTx(ctx, rejectedTx) + require.ErrorIs(t, err, errMempoolFull) + _, rejectedInCache = txmp.cache.cacheMap[txmp.cache.toCacheKey(types.Tx(rejectedTx).Hash())] + require.False(t, rejectedInCache) } func TestTxMempool_EVMEviction(t *testing.T) { diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index ae1683a9aa..51aeea43a7 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -7,6 +7,7 @@ import ( "maps" "math/big" "slices" + "errors" "time" "github.com/ethereum/go-ethereum/common" @@ -16,6 +17,11 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) +var errDuplicateTx = errors.New("duplicate tx") +var errOldNonce = errors.New("nonce too old") +var errSameNonce = errors.New("tx with this nonce already in mempool") +var errMempoolFull = errors.New("mempool full") + type hashedTx struct { tx types.Tx hash types.TxHash @@ -127,7 +133,7 @@ type txStore struct { } func NewTxStore(config *Config, app *proxy.Proxy) *txStore { - softLimit := txCounter{count: config.Size, bytes: utils.Clamp[uint64](config.MaxTxsBytes)} + softLimit := txCounter{count: config.Size + config.PendingSize, bytes: utils.Clamp[uint64](config.MaxTxsBytes + config.MaxPendingTxsBytes)} hardLimit := txCounter{count: 2 * softLimit.count, bytes: 2 * softLimit.bytes} inner := &txStoreInner{ byHash: map[types.TxHash]*WrappedTx{}, @@ -192,9 +198,9 @@ func (txs *txStore) ByHash(key types.TxHash) *WrappedTx { panic("unreachable") } -func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) bool { +func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { if _, ok := inner.byHash[wtx.Hash()]; ok { - return false + return errDuplicateTx } state := inner.state.Load() if evm, ok := wtx.evm.Get(); ok { @@ -209,17 +215,20 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) bool { } // Reject transactions with old nonces. if evm.nonce < account.firstNonce { - return false + return errOldNonce } an := evmAddrNonce{evm.address, evm.nonce} if old, ok := inner.byNonce[an]; ok { + if old.reaped { + return errSameNonce + } // If the old tx is ready but the new tx is not, then reject the new tx. if old.evm.OrPanic("non-evm tx").nonce < account.nextNonce && account.balance.Cmp(evm.requiredBalance) < 0 { - return false + return errSameNonce } // If the old tx has >= priority, then reject new tx. if old.priority >= wtx.priority { - return false + return errSameNonce } // Remove the old transaction. delete(inner.byHash, old.Hash()) @@ -255,7 +264,7 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) bool { } inner.byHash[wtx.Hash()] = wtx inner.state.Store(state) - return true + return nil } // WARNING: works only if wtx has been already inserted. @@ -304,14 +313,21 @@ func (inner *txStoreInner) inInclusionOrder() []*WrappedTx { return append(ready, pending...) } -// SetTx stores a *WrappedTx by its hash. -func (txs *txStore) Insert(wtx *WrappedTx) { +// Inserts a new transaction to txStore. +// txStore takes ownership of wtx. +func (txs *txStore) Insert(wtx *WrappedTx) error { for inner := range txs.inner.Lock() { - txs.insert(inner, wtx) + if err:=txs.insert(inner, wtx); err!=nil { + return err + } if total := inner.state.Load().total; !total.LessEqual(&inner.hardLimit) { txs.compact(inner, false) + if _,ok := inner.byHash[wtx.Hash()]; !ok { + return errMempoolFull + } } } + return nil } // O(m log m) From 4bff32edbb8829b9ef0da4d8f347c13d84919190 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 14:29:46 +0200 Subject: [PATCH 017/100] moved cache --- sei-tendermint/internal/mempool/cache.go | 7 ++++ sei-tendermint/internal/mempool/mempool.go | 44 +++++++--------------- sei-tendermint/internal/mempool/tx.go | 42 ++++++++++++++++++--- 3 files changed, 58 insertions(+), 35 deletions(-) diff --git a/sei-tendermint/internal/mempool/cache.go b/sei-tendermint/internal/mempool/cache.go index 76d23c9d49..343ad76a3b 100644 --- a/sei-tendermint/internal/mempool/cache.go +++ b/sei-tendermint/internal/mempool/cache.go @@ -43,6 +43,13 @@ func NewLRUTxCache(cacheSize int, maxKeyLen int) *LRUTxCache { } } +func (c *LRUTxCache) Has(txHash types.TxHash) bool { + c.mtx.Lock() + defer c.mtx.Unlock() + _,ok := c.cacheMap[c.toCacheKey(txHash)] + return ok +} + func (c *LRUTxCache) Reset() { c.mtx.Lock() defer c.mtx.Unlock() diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 4bb7a635c4..db86ab23f3 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -170,10 +170,6 @@ type TxMempool struct { // height defines the last block height process during Update() height int64 - // cache defines a fixed-size cache of already seen transactions as this - // reduces pressure on the proxyApp. - cache *LRUTxCache - // blockFailedTxs tracks tx hashes that have previously failed during // block execution. Used to prevent infinite re-entry of txs that // consistently fail before fee charging in DeliverTx. @@ -219,8 +215,6 @@ func NewTxMempool( txStore: NewTxStore(cfg, app), txConstraintsFetcher: txConstraintsFetcher, priorityReservoir: reservoir.New[int64](cfg.DropPriorityReservoirSize, cfg.DropPriorityThreshold, nil), // Use non-deterministic RNG - cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), - blockFailedTxs: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), } if cfg.DuplicateTxsCacheSize > 0 { @@ -326,10 +320,9 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response // We add the transaction to the mempool's cache and if the // transaction is already present in the cache, i.e. false is returned, then we // check if we've seen this transaction and error if we have. - if !txmp.cache.Push(hTx.Hash()) { + if txmp.txStore.CacheHas(hTx.Hash()) { return nil, ErrTxInCache } - txmp.metrics.CacheSize.Set(float64(txmp.cache.Size())) // Check TTL cache to see if we've recently processed this transaction // Only execute TTL cache logic if we're using a real TTL cache (not NOP) @@ -340,12 +333,13 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response if err != nil || !res.IsOK() { txmp.metrics.NumberOfFailedCheckTxs.Add(1) txmp.metrics.observeCheckTxPriorityDistribution(0, false, "", true) - txmp.cache.Remove(hTx.Hash()) } if err != nil { + txmp.txStore.CachePush(hTx.Hash()) return nil, err } if !res.IsOK() { + txmp.txStore.CachePush(hTx.Hash()) return res.ResponseCheckTx, nil } txmp.metrics.NumberOfSuccessfulCheckTxs.Add(1) @@ -391,12 +385,12 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response if err := wtx.check(constraints); err != nil { // ignore bad transactions logger.Info("rejected bad transaction", "priority", wtx.priority, "tx", wtx.Hash(), "post_check_err", err) + txmp.txStore.CachePush(hTx.Hash()) txmp.metrics.FailedTxs.Add(1) return nil, err } if err := txmp.txStore.Insert(wtx); err!=nil { - txmp.cache.Remove(wtx.Hash()) return nil, err } @@ -436,7 +430,6 @@ func (txmp *TxMempool) Flush() { txmp.mtx.Lock() defer txmp.mtx.Unlock() txmp.txStore = NewTxStore(txmp.config, txmp.app) - txmp.cache.Reset() } // ReapMaxBytesMaxGas returns a list of transactions within the provided size @@ -486,27 +479,14 @@ func (txmp *TxMempool) Update( return txConstraints, nil } - txHashes := map[types.TxHash]struct{}{} + txResults := map[types.TxHash]bool{} for i, tx := range blockTxs { - txHash := tx.Hash() - txHashes[txHash] = struct{}{} - // Remove transaction from the mempool, no matter if it succeeded, or not. - if execTxResult[i].Code == abci.CodeTypeOK { - // add the valid committed transaction to the cache (if missing) - txmp.cache.Push(txHash) - txmp.blockFailedTxs.Remove(txHash) - } else if !txmp.config.KeepInvalidTxsInCache { - if txmp.blockFailedTxs.Push(txHash) { - // First block failure: allow one retry - txmp.cache.Remove(txHash) - } - // Subsequent failures: leave in cache to prevent infinite re-entry - } + txResults[tx.Hash()] = execTxResult[i].Code == abci.CodeTypeOK } newPriorities := map[types.TxHash]int64{} if recheck { for _, wtx := range txmp.txStore.AllReady() { - if _, ok := txHashes[wtx.Hash()]; ok { + if _, ok := txResults[wtx.Hash()]; ok { continue } txmp.metrics.RecheckTimes.Add(1) @@ -514,10 +494,14 @@ func (txmp *TxMempool) Update( Tx: wtx.Tx(), Type: abci.CheckTxTypeV2Recheck, }) - // If recheck fails, just remove the tx. if err != nil || !res.IsOK() { - txHashes[wtx.Hash()] = struct{}{} + // If recheck fails, just remove the tx. + // TODO(gprusak): we emulate the fact that we don't want this tx + // by saying that it was already executed - this way it is pushed to cache and removed from mempool. + // It deserves more explicit handling though. + txResults[wtx.Hash()] = true } else { + // If succeeds, we just care about the new priority. newPriorities[wtx.Hash()] = res.Priority } } @@ -525,7 +509,7 @@ func (txmp *TxMempool) Update( txmp.txStore.Update(updateSpec{ Now: time.Now(), Height: blockHeight, - ToRemove: txHashes, + TxResults: txResults, NewPriorities: newPriorities, Constraints: txConstraints, }) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 51aeea43a7..d22f3f0f1f 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -127,13 +127,24 @@ type txStoreInner struct { type txStore struct { config *Config app *proxy.Proxy + + // Cache of already seen txs, reducess pressure on app. + // It is a superset of transactions in txStore. + // * successfully inserted transactions are automatically added to cache. + // * txs which fail Insert() are NOT added to cache and can be reattempted later. + // * invalid transactions can be recorded via CachePush. + // * txs dropped due to pruning are removed from cache. + // * txs successfully executed are kept in cache to avoid reinsert. + // * txs failed execution are eligible to be reexecuted once (iff config.KeepInvalidTxsInCache). + cache *LRUTxCache + failedTxs *LRUTxCache inner utils.RWMutex[*txStoreInner] state utils.AtomicRecv[txStoreState] readyTxs *clist.CList[types.Tx] } -func NewTxStore(config *Config, app *proxy.Proxy) *txStore { - softLimit := txCounter{count: config.Size + config.PendingSize, bytes: utils.Clamp[uint64](config.MaxTxsBytes + config.MaxPendingTxsBytes)} +func NewTxStore(cfg *Config, app *proxy.Proxy) *txStore { + softLimit := txCounter{count: cfg.Size + cfg.PendingSize, bytes: utils.Clamp[uint64](cfg.MaxTxsBytes + cfg.MaxPendingTxsBytes)} hardLimit := txCounter{count: 2 * softLimit.count, bytes: 2 * softLimit.bytes} inner := &txStoreInner{ byHash: map[types.TxHash]*WrappedTx{}, @@ -144,7 +155,9 @@ func NewTxStore(config *Config, app *proxy.Proxy) *txStore { state: utils.NewAtomicSend(txStoreState{}), } return &txStore{ - config: config, + config: cfg, + cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), + failedTxs:NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), app: app, inner: utils.NewRWMutex(inner), readyTxs: clist.New[types.Tx](), @@ -152,6 +165,15 @@ func NewTxStore(config *Config, app *proxy.Proxy) *txStore { } } +// Checks if cache contains a given hash. +func (txs *txStore) CacheHas(txHash types.TxHash) bool { + return txs.cache.Has(txHash) +} + +func (txs *txStore) CachePush(txHash types.TxHash) { + txs.cache.Push(txHash) +} + // Size returns the total number of transactions in the store. func (txs *txStore) State() txStoreState { return txs.state.Load() } @@ -327,6 +349,7 @@ func (txs *txStore) Insert(wtx *WrappedTx) error { } } } + txs.cache.Push(wtx.Hash()) return nil } @@ -349,6 +372,7 @@ func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { if total.LessEqual(&inner.softLimit) { txs.insert(inner, wtx) } else { + txs.cache.Remove(wtx.Hash()) if el, ok := wtx.readyEl.Get(); ok { txs.readyTxs.Remove(el) } @@ -359,7 +383,8 @@ func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { type updateSpec struct { Now time.Time Height int64 - ToRemove map[types.TxHash]struct{} + // Indicates whether tx succeeded. + TxResults map[types.TxHash]bool Constraints TxConstraints NewPriorities map[types.TxHash]int64 } @@ -375,7 +400,8 @@ func (txs *txStore) Update(spec updateSpec) { } for inner := range txs.inner.Lock() { toRemove := func(wtx *WrappedTx) bool { - if _, ok := spec.ToRemove[wtx.Hash()]; ok { + // Executed transactions should be removed. + if _, ok := spec.TxResults[wtx.Hash()]; ok { return true } if wtx.reaped { @@ -399,6 +425,12 @@ func (txs *txStore) Update(spec updateSpec) { } for txHash, wtx := range inner.byHash { if toRemove(wtx) { + if txs.config.KeepInvalidTxsInCache && !spec.TxResults[txHash] { + // Failed txs are eligible for reexection once. + if !txs.failedTxs.Push(txHash) { + txs.cache.Remove(txHash) + } + } delete(inner.byHash, txHash) if el, ok := wtx.readyEl.Get(); ok { txs.readyTxs.Remove(el) From f51ec39b61677248f3e5fbc4fb1c370055f4aee5 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 15:26:08 +0200 Subject: [PATCH 018/100] reaped requires a better handling --- sei-tendermint/internal/mempool/mempool.go | 1 + .../internal/mempool/mempool_test.go | 49 ++++++++++--------- sei-tendermint/internal/mempool/tx.go | 34 ++++++------- 3 files changed, 41 insertions(+), 43 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index db86ab23f3..8d98ca0008 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -85,6 +85,7 @@ type Config struct { // Limit the total size of all txs in the pending set. MaxPendingTxsBytes int64 + // Whether expired READY transactions should be pruned from mempool (PENDING expired are always prunned) RemoveExpiredTxsFromQueue bool // DropPriorityThreshold defines the percentage of transactions with the lowest diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index 1bebb30c39..b14939ebbc 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -649,33 +649,36 @@ func TestTxMempool_RemoveCacheWhenPendingTxIsFull(t *testing.T) { client := &application{Application: kvstore.NewApplication()} cfg := TestConfig() - cfg.CacheSize = 10 - cfg.Size = 1 + cfg.CacheSize = 100 + cfg.Size = 5 cfg.PendingSize = 0 txmp := NewTxMempool(cfg, proxy.New(client, proxy.NopMetrics()), NopMetrics(), NopTxConstraintsFetcher) - firstTx := []byte("sender-0=peer=100") - _, err := txmp.CheckTx(ctx, firstTx) - require.NoError(t, err) - - // The store only reports mempool-full once insertion crosses the hard limit - // and compaction drops the newly inserted low-priority tx. - _, err = txmp.CheckTx(ctx, []byte("sender-1=peer=50")) - require.NoError(t, err) + insertedTxs := make([]types.Tx, 0, 2*cfg.Size+1) + pruned := false + for i := range 100 { + tx := types.Tx(fmt.Appendf(nil, "sender-%d=peer=%d", i, i)) + insertedTxs = append(insertedTxs, tx) + expectedSize := len(insertedTxs) + _, err := txmp.CheckTx(ctx, tx) + if err != nil { + require.ErrorIs(t, err, errMempoolFull) + } + if txmp.Size() < expectedSize { + pruned = true + break + } + } - rejectedTx := []byte("sender-2=peer=1") - _, err = txmp.CheckTx(ctx, rejectedTx) - require.ErrorIs(t, err, errMempoolFull) + require.True(t, pruned) + require.LessOrEqual(t, txmp.Size(), cfg.Size) + require.Positive(t, txmp.Size()) - require.Equal(t, 1, txmp.Size()) - // The rejected transaction should be removed from cache so it can be retried later. - _, rejectedInCache := txmp.cache.cacheMap[txmp.cache.toCacheKey(types.Tx(rejectedTx).Hash())] - require.False(t, rejectedInCache) - - _, err = txmp.CheckTx(ctx, rejectedTx) - require.ErrorIs(t, err, errMempoolFull) - _, rejectedInCache = txmp.cache.cacheMap[txmp.cache.toCacheKey(types.Tx(rejectedTx).Hash())] - require.False(t, rejectedInCache) + for _, tx := range insertedTxs { + inMempool := txmp.txStore.ByHash(tx.Hash()) != nil + inCache := txmp.txStore.CacheHas(tx.Hash()) + require.Equal(t, inMempool, inCache) + } } func TestTxMempool_EVMEviction(t *testing.T) { @@ -1031,7 +1034,7 @@ func TestBlockFailedTxTrackerClearedOnSuccess(t *testing.T) { // Success clears the failure tracker. Simulate LRU eviction of the // main cache entry so we can verify the tracker was actually reset. - txmp.cache.Remove(txHash) + txmp.txStore.cache.Remove(txHash) // Tx should now be re-admittable _, err = txmp.CheckTx(ctx, tx) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index d22f3f0f1f..c78bedb4e9 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -253,6 +253,7 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { return errSameNonce } // Remove the old transaction. + txs.cache.Remove(old.Hash()) delete(inner.byHash, old.Hash()) if el, ok := wtx.readyEl.Get(); ok { txs.readyTxs.Remove(el) @@ -399,20 +400,8 @@ func (txs *txStore) Update(spec updateSpec) { minTime = utils.Some(spec.Now.Add(-d)) } for inner := range txs.inner.Lock() { - toRemove := func(wtx *WrappedTx) bool { - // Executed transactions should be removed. - if _, ok := spec.TxResults[wtx.Hash()]; ok { - return true - } - if wtx.reaped { - // If we already reaped the transaction, we shouldn't lose track of it. - return false - } - if wtx.check(spec.Constraints) != nil { - return true - } - // Consider expiration. - if inner.isReady(wtx) && !txs.config.RemoveExpiredTxsFromQueue { + isExpired := func(wtx *WrappedTx) bool { + if !txs.config.RemoveExpiredTxsFromQueue && inner.isReady(wtx) { return false } if t, ok := minTime.Get(); ok && wtx.timestamp.Before(t) { @@ -424,21 +413,26 @@ func (txs *txStore) Update(spec updateSpec) { return false } for txHash, wtx := range inner.byHash { - if toRemove(wtx) { - if txs.config.KeepInvalidTxsInCache && !spec.TxResults[txHash] { + // Executed transactions should be removed. + remove := false + if success, ok := spec.TxResults[wtx.Hash()]; ok { + if txs.config.KeepInvalidTxsInCache && !success { // Failed txs are eligible for reexection once. if !txs.failedTxs.Push(txHash) { txs.cache.Remove(txHash) } } + remove = true + } else { + remove = !wtx.reaped && (isExpired(wtx) || wtx.check(spec.Constraints) != nil) + } + if remove { delete(inner.byHash, txHash) if el, ok := wtx.readyEl.Get(); ok { txs.readyTxs.Remove(el) } - } else { - if newPriority, ok := spec.NewPriorities[wtx.Hash()]; ok { - wtx.priority = newPriority - } + } else if newPriority, ok := spec.NewPriorities[wtx.Hash()]; ok { + wtx.priority = newPriority } } txs.compact(inner, true) From 17702a913ba574031365e89f93c8453929b20217 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 19:23:16 +0200 Subject: [PATCH 019/100] no reaped again --- .../internal/autobahn/producer/state.go | 4 +- sei-tendermint/internal/mempool/mempool.go | 22 +++---- sei-tendermint/internal/mempool/tx.go | 60 +++++++------------ sei-tendermint/internal/rpc/core/mempool.go | 4 +- sei-tendermint/internal/state/execution.go | 4 +- 5 files changed, 34 insertions(+), 60 deletions(-) diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index a0b17ef827..1495f0fe47 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -73,12 +73,12 @@ func (s *State) makePayload(ctx context.Context) (*types.Payload, error) { return nil, err } - txs, gasEstimated := s.txMempool.ReapTxsAndMark(mempool.ReapLimits{ + txs, gasEstimated := s.txMempool.ReapTxs(mempool.ReapLimits{ MaxTxs: utils.Some(min(types.MaxTxsPerBlock, s.cfg.maxTxsPerBlock())), MaxBytes: utils.Some(utils.Clamp[int64](types.MaxTxsBytesPerBlock)), MaxGasWanted: utils.Some(s.cfg.MaxGasPerBlockI64()), MaxGasEstimated: utils.Some(s.cfg.MaxGasPerBlockI64()), - }) + }, true) payloadTxs := make([][]byte, 0, len(txs)) for _, tx := range txs { payloadTxs = append(payloadTxs, tx) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 8d98ca0008..46bed85e75 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -433,8 +433,8 @@ func (txmp *TxMempool) Flush() { txmp.txStore = NewTxStore(txmp.config, txmp.app) } -// ReapMaxBytesMaxGas returns a list of transactions within the provided size -// and gas constraints. The returned list starts with EVM transactions (in priority order), +// ReapTxs returns a list of transactions within the provided constraints and their total gas estimate. +// The returned list starts with EVM transactions (in priority order), // followed by non-EVM transactions (in priority order). // There are 4 types of constraints. // 1. maxBytes - stops pulling txs from mempool once maxBytes is hit. @@ -443,20 +443,12 @@ func (txmp *TxMempool) Flush() { // 3. maxGasEstimated - similar to maxGasWanted but will use the estimated gas used for EVM txs // while still using gas wanted for cosmos txs. Can be set to -1 to be ignored. // -// NOTE: -// - Transactions returned are not removed from the mempool transaction -// store or indexes. -func (txmp *TxMempool) ReapTxs(limits ReapLimits) types.Txs { - txmp.mtx.Lock() - defer txmp.mtx.Unlock() - txs, _ := txmp.txStore.Reap(limits, false) - return txs -} - -func (txmp *TxMempool) ReapTxsAndMark(limits ReapLimits) (types.Txs, int64) { +// NOTE: Transactions are removed from the mempool iff remove == true. +// Either way, the transactions stay in the LRU cache. +func (txmp *TxMempool) ReapTxs(limits ReapLimits, remove bool) (types.Txs, int64) { txmp.mtx.Lock() defer txmp.mtx.Unlock() - return txmp.txStore.Reap(limits, true) + return txmp.txStore.Reap(limits, remove) } // Update iterates over all the transactions provided by the block producer, @@ -486,7 +478,7 @@ func (txmp *TxMempool) Update( } newPriorities := map[types.TxHash]int64{} if recheck { - for _, wtx := range txmp.txStore.AllReady() { + for _, wtx := range txmp.txStore.ReadyTxs() { if _, ok := txResults[wtx.Hash()]; ok { continue } diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index c78bedb4e9..fba1c0e158 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -4,7 +4,6 @@ import ( "cmp" "context" "fmt" - "maps" "math/big" "slices" "errors" @@ -50,7 +49,6 @@ type WrappedTx struct { evm utils.Option[evmTx] // evm transaction info readyEl utils.Option[*clist.CElement[types.Tx]] - reaped bool } func (wtx *WrappedTx) check(c TxConstraints) error { @@ -192,24 +190,17 @@ func (txs *txStore) NextNonce(addr common.Address) uint64 { return txs.app.EvmNonce(addr) } -// GetAllTxs returns all the transactions currently in the store. -func (txs *txStore) GetAllTxs() []*WrappedTx { +// Returns all ready txs. +func (txs *txStore) ReadyTxs() []*WrappedTx { + var res []*WrappedTx for inner := range txs.inner.RLock() { - return slices.Collect(maps.Values(inner.byHash)) - } - panic("unreachable") -} - -func (txs *txStore) AllReady() []*WrappedTx { - var ready []*WrappedTx - for inner := range txs.inner.RLock() { - for _, wtx := range inner.byHash { + for _,wtx := range inner.byHash { if inner.isReady(wtx) { - ready = append(ready, wtx) + res = append(res,wtx) } } } - return ready + return res } // GetTxByHash returns a *WrappedTx by the transaction's hash. @@ -241,9 +232,6 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { } an := evmAddrNonce{evm.address, evm.nonce} if old, ok := inner.byNonce[an]; ok { - if old.reaped { - return errSameNonce - } // If the old tx is ready but the new tx is not, then reject the new tx. if old.evm.OrPanic("non-evm tx").nonce < account.nextNonce && account.balance.Cmp(evm.requiredBalance) < 0 { return errSameNonce @@ -413,18 +401,16 @@ func (txs *txStore) Update(spec updateSpec) { return false } for txHash, wtx := range inner.byHash { - // Executed transactions should be removed. - remove := false + remove := isExpired(wtx) || wtx.check(spec.Constraints) != nil if success, ok := spec.TxResults[wtx.Hash()]; ok { + // Executed transactions should be removed. + remove = true if txs.config.KeepInvalidTxsInCache && !success { // Failed txs are eligible for reexection once. if !txs.failedTxs.Push(txHash) { txs.cache.Remove(txHash) } } - remove = true - } else { - remove = !wtx.reaped && (isExpired(wtx) || wtx.check(spec.Constraints) != nil) } if remove { delete(inner.byHash, txHash) @@ -448,9 +434,9 @@ type ReapLimits struct { // Reap returns a list of transactions within the provided tx, // byte, and gas constraints together with the total estimated gas for the -// returned transactions. +// returned transactions. Reaped txs are removed iff remove == true. // O(m log m) where m is the size of the txStore. -func (txs *txStore) Reap(l ReapLimits, markReaped bool) (types.Txs, int64) { +func (txs *txStore) Reap(l ReapLimits, remove bool) (types.Txs, int64) { maxTxs := l.MaxTxs.Or(utils.Max[uint64]()) maxBytes := l.MaxBytes.Or(utils.Max[int64]()) maxGasWanted := l.MaxGasWanted.Or(utils.Max[int64]()) @@ -472,19 +458,6 @@ func (txs *txStore) Reap(l ReapLimits, markReaped bool) (types.Txs, int64) { for inner := range txs.inner.Lock() { if uint64(inner.state.Load().ready.count) >= txs.config.TxNotifyThreshold { for _, wtx := range inner.inInclusionOrder() { - // Transactions are reaped to be included in a block at a particular height. - // In case of tendermint, txs are not reaped "in advance" - before the next block is proposed, - // the previous one needs to be finalized. - // In case of autobahn Reap and Update are called asynchronously, because execution is async. - // Consecutive calls to Reap should NOT return the same txs. - // Also in autobahn we have a guarantee that reaped transactions will be included, because - // every producer builds their blocks unanonimously, therefore reaped transactions will be eventually - // removed (once sequenced). - // TODO(gprusak): this is a weak constract between autobahn and mempool and may lead to mempool capacity - // leakage if violated. Redesign later. - if wtx.reaped { - continue - } if uint64(len(wtxs)) >= maxTxs || !inner.isReady(wtx) { break } @@ -498,14 +471,23 @@ func (txs *txStore) Reap(l ReapLimits, markReaped bool) (types.Txs, int64) { break } // include tx and update totals - wtx.reaped = markReaped totalSize += wtx.protoSize totalGasWanted += wtx.gasWanted totalGasEstimated += wtx.estimatedGas wtxs = append(wtxs, wtx) } } + if remove { + for _,wtx := range wtxs { + delete(inner.byHash, wtx.Hash()) + if el, ok := wtx.readyEl.Get(); ok { + txs.readyTxs.Remove(el) + } + txs.compact(inner,false) + } + } } + // EVM txs go first. var evmTxs, nonEvmTxs types.Txs for _, wtx := range wtxs { diff --git a/sei-tendermint/internal/rpc/core/mempool.go b/sei-tendermint/internal/rpc/core/mempool.go index adcd201cad..bba84876f0 100644 --- a/sei-tendermint/internal/rpc/core/mempool.go +++ b/sei-tendermint/internal/rpc/core/mempool.go @@ -124,9 +124,9 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque skipCount := validateSkipCount(page, perPage) - txs := env.Mempool.ReapTxs(mempool.ReapLimits{ + txs,_ := env.Mempool.ReapTxs(mempool.ReapLimits{ MaxTxs: utils.Some(uint64(skipCount + tmmath.MinInt(perPage, totalCount-skipCount))), - }) + }, false) if skipCount > len(txs) { skipCount = len(txs) } diff --git a/sei-tendermint/internal/state/execution.go b/sei-tendermint/internal/state/execution.go index aa747eb432..a1f2a8541c 100644 --- a/sei-tendermint/internal/state/execution.go +++ b/sei-tendermint/internal/state/execution.go @@ -123,11 +123,11 @@ func (blockExec *BlockExecutor) CreateProposalBlock( // Fetch a limited amount of valid txs maxDataBytes := types.MaxDataBytes(maxBytes, evSize, state.Validators.Size()) - txs := blockExec.mempool.ReapTxs(mempool.ReapLimits{ + txs,_ := blockExec.mempool.ReapTxs(mempool.ReapLimits{ MaxBytes: utils.Some(maxDataBytes), MaxGasWanted: utils.Some(maxGasWanted), MaxGasEstimated: utils.Some(maxGas), - }) + }, false) block = state.MakeBlock(height, txs, lastCommit, evidence, proposerAddr) return block, nil } From 2a7bc71406edec9e15b9112a1765cb370fb31982 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 19:26:08 +0200 Subject: [PATCH 020/100] codex WIP --- .../internal/mempool/mempool_test.go | 34 +++++++++---------- .../internal/mempool/recheck_drain_test.go | 6 ++-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index b14939ebbc..b2f73d18b6 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -373,7 +373,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapTxs(ReapLimits{MaxGasWanted: utils.Some(int64(50))}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxGasWanted: utils.Some(int64(50))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) @@ -384,7 +384,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapTxs(ReapLimits{MaxBytes: utils.Some(int64(1000))}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxBytes: utils.Some(int64(1000))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) @@ -396,10 +396,10 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapTxs(ReapLimits{ + reapedTxs, _ := txmp.ReapTxs(ReapLimits{ MaxBytes: utils.Some(int64(1500)), MaxGasWanted: utils.Some(int64(30)), - }) + }, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) @@ -410,7 +410,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapTxs(ReapLimits{MaxGasWanted: utils.Some(int64(2))}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxGasWanted: utils.Some(int64(2))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Len(t, reapedTxs, 2) @@ -420,7 +420,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(50))}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(50))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Len(t, reapedTxs, 50) @@ -464,7 +464,7 @@ func TestTxMempool_ReapMaxBytesMaxGas_FallbackToGasWanted(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(50))}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(50))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Len(t, reapedTxs, 50) @@ -510,7 +510,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapTxs(ReapLimits{}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) @@ -521,7 +521,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(1))}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(1))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) @@ -532,7 +532,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(len(tTxs) / 2))}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(len(tTxs) / 2))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) @@ -560,7 +560,7 @@ func TestTxMempool_ReapMaxBytesMaxGas_MinGasEVMTxThreshold(t *testing.T) { // With MinGasEVMTx=21000, estimatedGas (10000) is ignored and we fallback to gasWanted (50000). // Setting maxGasEstimated below gasWanted should therefore result in 0 reaped txs. - reaped := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(40000))}) + reaped, _ := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(40000))}, false) require.Len(t, reaped, 0) // Note: If MinGasEVMTx is changed to 0, the same scenario would use estimatedGas (10000) @@ -639,7 +639,7 @@ func TestTxMempool_Prioritization(t *testing.T) { []byte(fmt.Sprintf("sender-3-1=peer=%d", 4)), } - reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(len(expectedReapedTxs)))}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(len(expectedReapedTxs)))}, false) require.Equal(t, expectedReapedTxs, reapedTxs) } @@ -717,7 +717,7 @@ func TestTxMempool_EVMEviction(t *testing.T) { require.Equal(t, 1, txmp.NumTxsNotPending()) require.Equal(t, 1, txmp.PendingSize()) - tx := txmp.txStore.AllReady()[0] + tx := txmp.txStore.ReadyTxs()[0] require.Equal(t, int64(4), tx.priority) // Should be the highest priority transaction _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address2, 5, 1))) @@ -786,7 +786,7 @@ func TestTxMempool_ConcurrentTxs(t *testing.T) { var height int64 = 1 for range ticker.C { - reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(200))}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(200))}, false) if len(reapedTxs) > 0 { responses := make([]*abci.ExecTxResult, len(reapedTxs)) for i := 0; i < len(responses); i++ { @@ -835,7 +835,7 @@ func TestTxMempool_ExpiredTxs_NumBlocks(t *testing.T) { require.Equal(t, len(tTxs), txmp.Size()) // reap 5 txs at the next height -- no txs should expire - reapedTxs := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(5))}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(5))}, false) responses := make([]*abci.ExecTxResult, len(reapedTxs)) for i := 0; i < len(responses); i++ { responses[i] = &abci.ExecTxResult{Code: abci.CodeTypeOK} @@ -859,7 +859,7 @@ func TestTxMempool_ExpiredTxs_NumBlocks(t *testing.T) { // cannot guarantee that all 95 txs are remaining that should be expired and // removed. However, we do know that that at most 95 txs can be expired and // removed. - reapedTxs = txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(5))}) + reapedTxs, _ = txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(5))}, false) responses = make([]*abci.ExecTxResult, len(reapedTxs)) for i := 0; i < len(responses); i++ { responses[i] = &abci.ExecTxResult{Code: abci.CodeTypeOK} @@ -918,7 +918,7 @@ func TestReapMaxBytesMaxGas_EVMFirst(t *testing.T) { require.Equal(t, 5, txmp.Size()) // Reap all transactions - reapedTxs := txmp.ReapTxs(ReapLimits{}) + reapedTxs, _ := txmp.ReapTxs(ReapLimits{}, false) require.Len(t, reapedTxs, 5) // Verify EVM transactions come first, then non-EVM diff --git a/sei-tendermint/internal/mempool/recheck_drain_test.go b/sei-tendermint/internal/mempool/recheck_drain_test.go index 28a71365bd..9fb48246b9 100644 --- a/sei-tendermint/internal/mempool/recheck_drain_test.go +++ b/sei-tendermint/internal/mempool/recheck_drain_test.go @@ -144,9 +144,9 @@ func TestTxMempool_DescendingNonceDrain(t *testing.T) { const maxBlocks = 5 totalMined := 0 for height := int64(1); txmp.Size() > 0 && height <= maxBlocks; height++ { - txs, _ := txmp.ReapTxsAndMark(ReapLimits{ - MaxTxs: utils.Some(uint64(N)), - }) + txs, _ := txmp.ReapTxs(ReapLimits{ + MaxTxs: utils.Some(uint64(N)), + }, true) require.NotEmpty(t, txs, "PopTxs returned no txs at height %d (mempool stalled)", height) txResults := make([]*abci.ExecTxResult, len(txs)) From 6ea4f7a4d7fd21dea383fae2fb653eef1cbd62b6 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 19:52:46 +0200 Subject: [PATCH 021/100] codex WIP --- sei-tendermint/internal/mempool/cache_test.go | 10 +-- .../internal/mempool/mempool_test.go | 68 +++---------------- .../internal/mempool/reactor/reactor_test.go | 21 ------ .../internal/mempool/recheck_drain_test.go | 15 ++-- sei-tendermint/internal/mempool/tx.go | 12 ++-- 5 files changed, 31 insertions(+), 95 deletions(-) diff --git a/sei-tendermint/internal/mempool/cache_test.go b/sei-tendermint/internal/mempool/cache_test.go index 246185b31c..cd386259a6 100644 --- a/sei-tendermint/internal/mempool/cache_test.go +++ b/sei-tendermint/internal/mempool/cache_test.go @@ -403,20 +403,22 @@ func TestLRUTxCache_EdgeCases(t *testing.T) { cache := NewLRUTxCache(0, 0) tx := types.Tx("test").Hash() - // Should handle zero size gracefully + // Zero-sized cache is effectively disabled. result := cache.Push(tx) assert.True(t, result) - assert.Equal(t, 1, cache.Size()) + assert.Equal(t, 0, cache.Size()) + assert.False(t, cache.Has(tx)) }) t.Run("NegativeSizeCache", func(t *testing.T) { cache := NewLRUTxCache(-1, 0) tx := types.Tx("test").Hash() - // Should handle negative size gracefully + // Negative-sized cache is effectively disabled. result := cache.Push(tx) assert.True(t, result) - assert.Equal(t, 1, cache.Size()) + assert.Equal(t, 0, cache.Size()) + assert.False(t, cache.Has(tx)) }) t.Run("NilTransaction", func(t *testing.T) { diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index b2f73d18b6..345bf3bb43 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -681,60 +681,7 @@ func TestTxMempool_RemoveCacheWhenPendingTxIsFull(t *testing.T) { } } -func TestTxMempool_EVMEviction(t *testing.T) { - ctx := t.Context() - - client := &application{Application: kvstore.NewApplication()} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 100, NopTxConstraintsFetcher) - txmp.config.Size = 1 - - address1 := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" - address2 := "0xfD23B3A9DE15e92B9ef9540E587B3661E15A12fA" - - // Add first transaction with priority 1 - _, err := txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 1, 0))) - require.NoError(t, err) - - // This should evict the previous tx (priority 1 < priority 2) - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 2, 0))) - require.NoError(t, err) - // Increase mempool size to 2 - txmp.config.Size = 2 - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address1, 3, 1))) - require.NoError(t, err) - require.Equal(t, 0, txmp.PendingSize()) - - // This would evict the tx with priority 2 and cause the tx with priority 3 to go pending - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address2, 4, 0))) - require.NoError(t, err) - - require.Eventually(t, func() bool { - return txmp.NumTxsNotPending() == 1 && txmp.PendingSize() == 1 - }, 5*time.Second, 100*time.Millisecond, "Expected mempool state not reached") - - // Verify final state - require.Equal(t, 1, txmp.NumTxsNotPending()) - require.Equal(t, 1, txmp.PendingSize()) - - tx := txmp.txStore.ReadyTxs()[0] - require.Equal(t, int64(4), tx.priority) // Should be the highest priority transaction - - _, err = txmp.CheckTx(ctx, []byte(fmt.Sprintf("evm-sender=%s=%d=%d", address2, 5, 1))) - require.NoError(t, err) - require.Equal(t, 2, txmp.NumTxsNotPending()) - - //TODO: txmp.removeTx(tx, true, false, true) - // Should not reenqueue - require.Equal(t, 1, txmp.NumTxsNotPending()) - - require.Eventually(t, func() bool { - return txmp.PendingSize() == 1 - }, 5*time.Second, 100*time.Millisecond, "Expected pendingTxs size not reached") - require.Equal(t, 1, txmp.PendingSize()) -} - -func TestTxMempool_CheckTxSamePeer(t *testing.T) { +func TestTxMempool_CheckTxDuplicateRejected(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} @@ -878,18 +825,23 @@ func TestMempoolExpiration(t *testing.T) { client := &application{Application: kvstore.NewApplication()} txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - txmp.config.TTLDuration = utils.Some(time.Nanosecond) // we want tx to expire immediately + txmp.config.TTLDuration = utils.Some(time.Nanosecond) txmp.config.RemoveExpiredTxsFromQueue = true txs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(txs), txmp.Size()) + time.Sleep(time.Millisecond) - //txmp.purgeExpiredTxs(txmp.height) + + txmp.Lock() + require.NoError(t, txmp.Update(ctx, 1, nil, nil, utils.OrPanic1(txmp.txConstraintsFetcher()), true)) + txmp.Unlock() + require.Equal(t, 0, txmp.Size()) } -// TestReapMaxBytesMaxGas_EVMFirst verifies that ReapMaxBytesMaxGas returns +// TestTxMempool_ReapTxs_EVMFirst verifies that ReapTxs returns // EVM transactions first, followed by non-EVM transactions. -func TestReapMaxBytesMaxGas_EVMFirst(t *testing.T) { +func TestTxMempool_ReapTxs_EVMFirst(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} diff --git a/sei-tendermint/internal/mempool/reactor/reactor_test.go b/sei-tendermint/internal/mempool/reactor/reactor_test.go index ebadde6512..21a14c22e6 100644 --- a/sei-tendermint/internal/mempool/reactor/reactor_test.go +++ b/sei-tendermint/internal/mempool/reactor/reactor_test.go @@ -392,27 +392,6 @@ func TestReactorConcurrency(t *testing.T) { wg.Wait() } -func TestReactorNoBroadcastToSender(t *testing.T) { - numTxs := 1000 - numNodes := 2 - ctx := t.Context() - - rts := setupReactors(ctx, t, numNodes) - t.Cleanup(leaktest.Check(t)) - - primary := rts.nodes[0] - secondary := rts.nodes[1] - - _ = checkTxs(ctx, t, rts.mempools[primary], numTxs) - - rts.start(t) - time.Sleep(100 * time.Millisecond) - - require.Eventually(t, func() bool { - return rts.mempools[secondary].Size() == 0 - }, time.Minute, 100*time.Millisecond) -} - func TestReactor_MaxTxBytes(t *testing.T) { numNodes := 2 cfg := config.TestConfig() diff --git a/sei-tendermint/internal/mempool/recheck_drain_test.go b/sei-tendermint/internal/mempool/recheck_drain_test.go index 9fb48246b9..5fb09618fb 100644 --- a/sei-tendermint/internal/mempool/recheck_drain_test.go +++ b/sei-tendermint/internal/mempool/recheck_drain_test.go @@ -16,6 +16,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" + "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) // evmNonceApp models a Sei-like EVM antehandler for mempool tests: @@ -179,8 +180,8 @@ func TestTxMempool_EvmNextPendingNonceIncludesPendingTransactions(t *testing.T) require.NoError(t, err) } - require.Equal(t, 2, txmp.NumTxsNotPending()) - require.Equal(t, 1, txmp.PendingSize()) + require.Equal(t, 3, txmp.NumTxsNotPending()) + require.Equal(t, 0, txmp.PendingSize()) require.Equal(t, uint64(8), txmp.EvmNextPendingNonce(sender)) } @@ -200,11 +201,9 @@ func TestTxMempool_EvmNextPendingNonceReplacesSameNonceByPriority(t *testing.T) _, err = txmp.CheckTx(ctx, highPriorityTx) require.NoError(t, err) - require.Equal(t, 2, txmp.PendingSize(), "pending store keeps both txs") - /*for byAddrNonce := range txmp.byAddrNonce.Lock() { - wtx, ok := byAddrNonce[evmAddrNonce{Address: sender, Nonce: 6}] - require.True(t, ok, "nonce bookkeeping should track one occupied nonce") - require.Equal(t, types.Tx(highPriorityTx).Hash(), wtx.Hash()) - }*/ + require.Equal(t, 1, txmp.PendingSize()) + require.Equal(t, 1, txmp.Size()) + require.Nil(t, txmp.txStore.ByHash(types.Tx(lowPriorityTx).Hash())) + require.NotNil(t, txmp.txStore.ByHash(types.Tx(highPriorityTx).Hash())) require.Equal(t, uint64(5), txmp.EvmNextPendingNonce(sender)) } diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index fba1c0e158..d2e1180c9a 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -405,10 +405,14 @@ func (txs *txStore) Update(spec updateSpec) { if success, ok := spec.TxResults[wtx.Hash()]; ok { // Executed transactions should be removed. remove = true - if txs.config.KeepInvalidTxsInCache && !success { - // Failed txs are eligible for reexection once. - if !txs.failedTxs.Push(txHash) { - txs.cache.Remove(txHash) + if !txs.config.KeepInvalidTxsInCache { + if !success { + // Failed txs are eligible for reexection once. + if txs.failedTxs.Push(txHash) { + txs.cache.Remove(txHash) + } + } else { + txs.failedTxs.Remove(txHash) } } } From 97717e758afe9655a370ecab5a2ba0fe0e88675d Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 19:54:41 +0200 Subject: [PATCH 022/100] fmt --- sei-tendermint/config/config.go | 4 +- .../internal/autobahn/producer/state.go | 4 +- sei-tendermint/internal/mempool/cache.go | 2 +- sei-tendermint/internal/mempool/mempool.go | 4 +- sei-tendermint/internal/mempool/tx.go | 58 +++++++++---------- sei-tendermint/internal/rpc/core/mempool.go | 2 +- sei-tendermint/internal/state/execution.go | 2 +- sei-tendermint/internal/state/tx_filter.go | 2 +- 8 files changed, 39 insertions(+), 39 deletions(-) diff --git a/sei-tendermint/config/config.go b/sei-tendermint/config/config.go index 593d009a5e..49b3dfcd85 100644 --- a/sei-tendermint/config/config.go +++ b/sei-tendermint/config/config.go @@ -13,8 +13,8 @@ import ( mempoolcfg "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" tmos "github.com/sei-protocol/sei-chain/sei-tendermint/libs/os" - "github.com/sei-protocol/sei-chain/sei-tendermint/types" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) const ( @@ -860,7 +860,7 @@ type MempoolConfig struct { DropPriorityReservoirSize int `mapstructure:"drop-priority-reservoir-size"` } -func (cfg *MempoolConfig) ToMempoolConfig() *mempoolcfg.Config { +func (cfg *MempoolConfig) ToMempoolConfig() *mempoolcfg.Config { mcfg := &mempoolcfg.Config{ Size: cfg.Size, MaxTxsBytes: cfg.MaxTxsBytes, diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index 1495f0fe47..3bb2409c5a 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -85,8 +85,8 @@ func (s *State) makePayload(ctx context.Context) (*types.Payload, error) { } payload, err := types.PayloadBuilder{ CreatedAt: time.Now(), - TotalGas: uint64(gasEstimated), // nolint:gosec // always non-negative - Txs: payloadTxs, + TotalGas: uint64(gasEstimated), // nolint:gosec // always non-negative + Txs: payloadTxs, }.Build() // This should never happen: we construct the payload from correctly sized data. if err != nil { diff --git a/sei-tendermint/internal/mempool/cache.go b/sei-tendermint/internal/mempool/cache.go index 343ad76a3b..a730fdf61d 100644 --- a/sei-tendermint/internal/mempool/cache.go +++ b/sei-tendermint/internal/mempool/cache.go @@ -46,7 +46,7 @@ func NewLRUTxCache(cacheSize int, maxKeyLen int) *LRUTxCache { func (c *LRUTxCache) Has(txHash types.TxHash) bool { c.mtx.Lock() defer c.mtx.Unlock() - _,ok := c.cacheMap[c.toCacheKey(txHash)] + _, ok := c.cacheMap[c.toCacheKey(txHash)] return ok } diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 46bed85e75..93f9d92537 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -391,7 +391,7 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response return nil, err } - if err := txmp.txStore.Insert(wtx); err!=nil { + if err := txmp.txStore.Insert(wtx); err != nil { return nil, err } @@ -474,7 +474,7 @@ func (txmp *TxMempool) Update( txResults := map[types.TxHash]bool{} for i, tx := range blockTxs { - txResults[tx.Hash()] = execTxResult[i].Code == abci.CodeTypeOK + txResults[tx.Hash()] = execTxResult[i].Code == abci.CodeTypeOK } newPriorities := map[types.TxHash]int64{} if recheck { diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index d2e1180c9a..a85a46b1e1 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -3,10 +3,10 @@ package mempool import ( "cmp" "context" + "errors" "fmt" "math/big" "slices" - "errors" "time" "github.com/ethereum/go-ethereum/common" @@ -123,8 +123,8 @@ type txStoreInner struct { } type txStore struct { - config *Config - app *proxy.Proxy + config *Config + app *proxy.Proxy // Cache of already seen txs, reducess pressure on app. // It is a superset of transactions in txStore. @@ -134,11 +134,11 @@ type txStore struct { // * txs dropped due to pruning are removed from cache. // * txs successfully executed are kept in cache to avoid reinsert. // * txs failed execution are eligible to be reexecuted once (iff config.KeepInvalidTxsInCache). - cache *LRUTxCache + cache *LRUTxCache failedTxs *LRUTxCache - inner utils.RWMutex[*txStoreInner] - state utils.AtomicRecv[txStoreState] - readyTxs *clist.CList[types.Tx] + inner utils.RWMutex[*txStoreInner] + state utils.AtomicRecv[txStoreState] + readyTxs *clist.CList[types.Tx] } func NewTxStore(cfg *Config, app *proxy.Proxy) *txStore { @@ -153,17 +153,17 @@ func NewTxStore(cfg *Config, app *proxy.Proxy) *txStore { state: utils.NewAtomicSend(txStoreState{}), } return &txStore{ - config: cfg, - cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), - failedTxs:NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), - app: app, - inner: utils.NewRWMutex(inner), - readyTxs: clist.New[types.Tx](), - state: inner.state.Subscribe(), + config: cfg, + cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), + failedTxs: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), + app: app, + inner: utils.NewRWMutex(inner), + readyTxs: clist.New[types.Tx](), + state: inner.state.Subscribe(), } } -// Checks if cache contains a given hash. +// Checks if cache contains a given hash. func (txs *txStore) CacheHas(txHash types.TxHash) bool { return txs.cache.Has(txHash) } @@ -194,9 +194,9 @@ func (txs *txStore) NextNonce(addr common.Address) uint64 { func (txs *txStore) ReadyTxs() []*WrappedTx { var res []*WrappedTx for inner := range txs.inner.RLock() { - for _,wtx := range inner.byHash { + for _, wtx := range inner.byHash { if inner.isReady(wtx) { - res = append(res,wtx) + res = append(res, wtx) } } } @@ -275,7 +275,7 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { } inner.byHash[wtx.Hash()] = wtx inner.state.Store(state) - return nil + return nil } // WARNING: works only if wtx has been already inserted. @@ -324,22 +324,22 @@ func (inner *txStoreInner) inInclusionOrder() []*WrappedTx { return append(ready, pending...) } -// Inserts a new transaction to txStore. +// Inserts a new transaction to txStore. // txStore takes ownership of wtx. func (txs *txStore) Insert(wtx *WrappedTx) error { for inner := range txs.inner.Lock() { - if err:=txs.insert(inner, wtx); err!=nil { + if err := txs.insert(inner, wtx); err != nil { return err } if total := inner.state.Load().total; !total.LessEqual(&inner.hardLimit) { txs.compact(inner, false) - if _,ok := inner.byHash[wtx.Hash()]; !ok { + if _, ok := inner.byHash[wtx.Hash()]; !ok { return errMempoolFull } } } txs.cache.Push(wtx.Hash()) - return nil + return nil } // O(m log m) @@ -370,10 +370,10 @@ func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { } type updateSpec struct { - Now time.Time - Height int64 + Now time.Time + Height int64 // Indicates whether tx succeeded. - TxResults map[types.TxHash]bool + TxResults map[types.TxHash]bool Constraints TxConstraints NewPriorities map[types.TxHash]int64 } @@ -389,7 +389,7 @@ func (txs *txStore) Update(spec updateSpec) { } for inner := range txs.inner.Lock() { isExpired := func(wtx *WrappedTx) bool { - if !txs.config.RemoveExpiredTxsFromQueue && inner.isReady(wtx) { + if !txs.config.RemoveExpiredTxsFromQueue && inner.isReady(wtx) { return false } if t, ok := minTime.Get(); ok && wtx.timestamp.Before(t) { @@ -482,16 +482,16 @@ func (txs *txStore) Reap(l ReapLimits, remove bool) (types.Txs, int64) { } } if remove { - for _,wtx := range wtxs { + for _, wtx := range wtxs { delete(inner.byHash, wtx.Hash()) if el, ok := wtx.readyEl.Get(); ok { txs.readyTxs.Remove(el) } - txs.compact(inner,false) + txs.compact(inner, false) } } } - + // EVM txs go first. var evmTxs, nonEvmTxs types.Txs for _, wtx := range wtxs { diff --git a/sei-tendermint/internal/rpc/core/mempool.go b/sei-tendermint/internal/rpc/core/mempool.go index bba84876f0..f3ea01ceba 100644 --- a/sei-tendermint/internal/rpc/core/mempool.go +++ b/sei-tendermint/internal/rpc/core/mempool.go @@ -124,7 +124,7 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque skipCount := validateSkipCount(page, perPage) - txs,_ := env.Mempool.ReapTxs(mempool.ReapLimits{ + txs, _ := env.Mempool.ReapTxs(mempool.ReapLimits{ MaxTxs: utils.Some(uint64(skipCount + tmmath.MinInt(perPage, totalCount-skipCount))), }, false) if skipCount > len(txs) { diff --git a/sei-tendermint/internal/state/execution.go b/sei-tendermint/internal/state/execution.go index a1f2a8541c..e422e2b7a6 100644 --- a/sei-tendermint/internal/state/execution.go +++ b/sei-tendermint/internal/state/execution.go @@ -123,7 +123,7 @@ func (blockExec *BlockExecutor) CreateProposalBlock( // Fetch a limited amount of valid txs maxDataBytes := types.MaxDataBytes(maxBytes, evSize, state.Validators.Size()) - txs,_ := blockExec.mempool.ReapTxs(mempool.ReapLimits{ + txs, _ := blockExec.mempool.ReapTxs(mempool.ReapLimits{ MaxBytes: utils.Some(maxDataBytes), MaxGasWanted: utils.Some(maxGasWanted), MaxGasEstimated: utils.Some(maxGas), diff --git a/sei-tendermint/internal/state/tx_filter.go b/sei-tendermint/internal/state/tx_filter.go index 809c1b2be7..01ee35d411 100644 --- a/sei-tendermint/internal/state/tx_filter.go +++ b/sei-tendermint/internal/state/tx_filter.go @@ -48,7 +48,7 @@ func TxConstraintsFetcherFromStore(store Store) mempool.TxConstraintsFetcher { return mempool.TxConstraints{}, err } - return TxConstraintsForState(state),nil + return TxConstraintsForState(state), nil } } From 0b9bf27af64482b96e5103c566040449020edf78 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 20:35:32 +0200 Subject: [PATCH 023/100] some updates and documentation --- sei-tendermint/internal/mempool/tx.go | 35 +++++++++++++++++++-------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index a85a46b1e1..0031efe48b 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -122,6 +122,16 @@ type txStoreInner struct { state utils.AtomicSend[txStoreState] } +// Properties: +// * tx is ready if all txs with lower nonces are ready or executed AND +// balance >= tx.requiredBalance +// * we keep at most 1 tx per nonce +// * we prefer ready tx to pending tx (then tx with the higher priority) for the same nonce +// * we don't store txs below account nonce. +// * account nonces are evaluated once per height +// * we keep at least capacity and up to 2*capacity txs +// * we reap by highest prio, while respecting nonces. +// * non-evm txs are always ready type txStore struct { config *Config app *proxy.Proxy @@ -135,7 +145,10 @@ type txStore struct { // * txs successfully executed are kept in cache to avoid reinsert. // * txs failed execution are eligible to be reexecuted once (iff config.KeepInvalidTxsInCache). cache *LRUTxCache + // Tracks transactions which already failed execution once + // but are eligible for reexecution (not added yet to cache) failedTxs *LRUTxCache + inner utils.RWMutex[*txStoreInner] state utils.AtomicRecv[txStoreState] readyTxs *clist.CList[types.Tx] @@ -168,6 +181,7 @@ func (txs *txStore) CacheHas(txHash types.TxHash) bool { return txs.cache.Has(txHash) } +// Pushes a tx to cache, effectively blocking it from being inserted. func (txs *txStore) CachePush(txHash types.TxHash) { txs.cache.Push(txHash) } @@ -181,6 +195,9 @@ func (txs *txStore) WaitForTxs(ctx context.Context) error { return err } +// Nonce for the next tx of the given account to insert to mempool. +// It takes into consideration the account nonce at the last executed block +// and all the txs currently queued in the mempool. func (txs *txStore) NextNonce(addr common.Address) uint64 { for inner := range txs.inner.RLock() { if acc, ok := inner.accounts[addr]; ok { @@ -203,12 +220,13 @@ func (txs *txStore) ReadyTxs() []*WrappedTx { return res } -// GetTxByHash returns a *WrappedTx by the transaction's hash. -func (txs *txStore) ByHash(key types.TxHash) *WrappedTx { +func (txs *txStore) ByHash(key types.TxHash) (types.Tx,bool) { for inner := range txs.inner.RLock() { - return inner.byHash[key] + if wtx,ok := inner.byHash[key]; ok { + return wtx.Tx(),true + } } - panic("unreachable") + return nil,false } func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { @@ -291,8 +309,6 @@ func (inner *txStoreInner) isReady(wtx *WrappedTx) bool { // Cosmos transactions are all considered ready and from different accounts, so only priority is relevant. func (inner *txStoreInner) inInclusionOrder() []*WrappedTx { // Split txs into ready and pending. - // TODO(gprusak): we can precisely preallocate ready and pending in a single array, - // based on inner.state.total.count and inner.state.ready.count var ready, pending []*WrappedTx for _, wtx := range inner.byHash { if inner.isReady(wtx) { @@ -342,10 +358,11 @@ func (txs *txStore) Insert(wtx *WrappedTx) error { return nil } -// O(m log m) +// O(m log m), prunes transactions above softLimit and recomputes all the indices. func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { // Order all txs by priority. wtxs := inner.inInclusionOrder() + // Reset internal state. inner.state.Store(txStoreState{}) inner.byHash = map[types.TxHash]*WrappedTx{} inner.byNonce = map[evmAddrNonce]*WrappedTx{} @@ -358,9 +375,7 @@ func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { for _, wtx := range wtxs { total := inner.state.Load().total total.Inc(wtx.Size()) - if total.LessEqual(&inner.softLimit) { - txs.insert(inner, wtx) - } else { + if !total.LessEqual(&inner.softLimit) || txs.insert(inner, wtx) != nil { txs.cache.Remove(wtx.Hash()) if el, ok := wtx.readyEl.Get(); ok { txs.readyTxs.Remove(el) From d25b16dde7e91b62ff318f83688116d4be40ca38 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 20:43:17 +0200 Subject: [PATCH 024/100] some fixes --- sei-tendermint/internal/mempool/mempool.go | 11 +- .../internal/mempool/mempool_test.go | 2 +- .../internal/mempool/recheck_drain_test.go | 6 +- sei-tendermint/internal/mempool/tx.go | 113 ++++++++++-------- sei-tendermint/internal/mempool/tx_test.go | 23 ++-- 5 files changed, 82 insertions(+), 73 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 93f9d92537..51e37b9f3e 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -409,16 +409,7 @@ func (txmp *TxMempool) SafeGetTxsForHashes(txHashes []types.TxHash) (types.Txs, txmp.mtx.RLock() defer txmp.mtx.RUnlock() - txs := make([]types.Tx, 0, len(txHashes)) - missing := []types.TxHash{} - for _, txHash := range txHashes { - if wtx := txmp.txStore.ByHash(txHash); wtx != nil { - txs = append(txs, wtx.Tx()) - } else { - missing = append(missing, txHash) - } - } - return txs, missing + return txmp.txStore.SafeGetTxsForHashes(txHashes) } // Flush empties the mempool. It acquires a read-lock, fetches all the diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index 345bf3bb43..c119601fe2 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -675,7 +675,7 @@ func TestTxMempool_RemoveCacheWhenPendingTxIsFull(t *testing.T) { require.Positive(t, txmp.Size()) for _, tx := range insertedTxs { - inMempool := txmp.txStore.ByHash(tx.Hash()) != nil + _, inMempool := txmp.txStore.ByHash(tx.Hash()) inCache := txmp.txStore.CacheHas(tx.Hash()) require.Equal(t, inMempool, inCache) } diff --git a/sei-tendermint/internal/mempool/recheck_drain_test.go b/sei-tendermint/internal/mempool/recheck_drain_test.go index 5fb09618fb..db8b01d6e8 100644 --- a/sei-tendermint/internal/mempool/recheck_drain_test.go +++ b/sei-tendermint/internal/mempool/recheck_drain_test.go @@ -203,7 +203,9 @@ func TestTxMempool_EvmNextPendingNonceReplacesSameNonceByPriority(t *testing.T) require.Equal(t, 1, txmp.PendingSize()) require.Equal(t, 1, txmp.Size()) - require.Nil(t, txmp.txStore.ByHash(types.Tx(lowPriorityTx).Hash())) - require.NotNil(t, txmp.txStore.ByHash(types.Tx(highPriorityTx).Hash())) + _, ok := txmp.txStore.ByHash(types.Tx(lowPriorityTx).Hash()) + require.False(t, ok) + _, ok = txmp.txStore.ByHash(types.Tx(highPriorityTx).Hash()) + require.True(t, ok) require.Equal(t, uint64(5), txmp.EvmNextPendingNonce(sender)) } diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 0031efe48b..2511c238fe 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -177,40 +177,40 @@ func NewTxStore(cfg *Config, app *proxy.Proxy) *txStore { } // Checks if cache contains a given hash. -func (txs *txStore) CacheHas(txHash types.TxHash) bool { - return txs.cache.Has(txHash) +func (s *txStore) CacheHas(txHash types.TxHash) bool { + return s.cache.Has(txHash) } // Pushes a tx to cache, effectively blocking it from being inserted. -func (txs *txStore) CachePush(txHash types.TxHash) { - txs.cache.Push(txHash) +func (s *txStore) CachePush(txHash types.TxHash) { + s.cache.Push(txHash) } // Size returns the total number of transactions in the store. -func (txs *txStore) State() txStoreState { return txs.state.Load() } +func (s *txStore) State() txStoreState { return s.state.Load() } // WaitForTxs waits until the store becomes non-empty. -func (txs *txStore) WaitForTxs(ctx context.Context) error { - _, err := txs.state.Wait(ctx, func(s txStoreState) bool { return s.ready.count > 0 }) +func (s *txStore) WaitForTxs(ctx context.Context) error { + _, err := s.state.Wait(ctx, func(state txStoreState) bool { return state.ready.count > 0 }) return err } // Nonce for the next tx of the given account to insert to mempool. // It takes into consideration the account nonce at the last executed block // and all the txs currently queued in the mempool. -func (txs *txStore) NextNonce(addr common.Address) uint64 { - for inner := range txs.inner.RLock() { +func (s *txStore) NextNonce(addr common.Address) uint64 { + for inner := range s.inner.RLock() { if acc, ok := inner.accounts[addr]; ok { return acc.nextNonce } } - return txs.app.EvmNonce(addr) + return s.app.EvmNonce(addr) } // Returns all ready txs. -func (txs *txStore) ReadyTxs() []*WrappedTx { +func (s *txStore) ReadyTxs() []*WrappedTx { var res []*WrappedTx - for inner := range txs.inner.RLock() { + for inner := range s.inner.RLock() { for _, wtx := range inner.byHash { if inner.isReady(wtx) { res = append(res, wtx) @@ -220,16 +220,31 @@ func (txs *txStore) ReadyTxs() []*WrappedTx { return res } -func (txs *txStore) ByHash(key types.TxHash) (types.Tx,bool) { - for inner := range txs.inner.RLock() { - if wtx,ok := inner.byHash[key]; ok { - return wtx.Tx(),true +func (s *txStore) ByHash(key types.TxHash) (types.Tx, bool) { + for inner := range s.inner.RLock() { + if wtx, ok := inner.byHash[key]; ok { + return wtx.Tx(), true } } - return nil,false + return nil, false } -func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { +func (s *txStore) SafeGetTxsForHashes(txHashes []types.TxHash) (types.Txs, []types.TxHash) { + got := make([]types.Tx, 0, len(txHashes)) + missing := make([]types.TxHash, 0) + for inner := range s.inner.RLock() { + for _, txHash := range txHashes { + if wtx, ok := inner.byHash[txHash]; ok { + got = append(got, wtx.Tx()) + } else { + missing = append(missing, txHash) + } + } + } + return got, missing +} + +func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { if _, ok := inner.byHash[wtx.Hash()]; ok { return errDuplicateTx } @@ -239,8 +254,8 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { account, ok := inner.accounts[evm.address] if !ok { // TODO(gprusak): consider whether we should move these queries out of the mutex. - b := txs.app.EvmBalance(evm.address, evm.seiAddress) - n := txs.app.EvmNonce(evm.address) + b := s.app.EvmBalance(evm.address, evm.seiAddress) + n := s.app.EvmNonce(evm.address) account = &evmAccount{b, n, n} inner.accounts[evm.address] = account } @@ -259,10 +274,10 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { return errSameNonce } // Remove the old transaction. - txs.cache.Remove(old.Hash()) + s.cache.Remove(old.Hash()) delete(inner.byHash, old.Hash()) if el, ok := wtx.readyEl.Get(); ok { - txs.readyTxs.Remove(el) + s.readyTxs.Remove(el) } state.ready.Dec(old.Size()) state.total.Dec(old.Size()) @@ -280,7 +295,7 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { account.nextNonce += 1 state.ready.Inc(wtx.Size()) if !wtx.readyEl.IsPresent() { - wtx.readyEl = utils.Some(txs.readyTxs.PushBack(wtx.Tx())) + wtx.readyEl = utils.Some(s.readyTxs.PushBack(wtx.Tx())) } } } else { @@ -288,7 +303,7 @@ func (txs *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { state.total.Inc(wtx.Size()) state.ready.Inc(wtx.Size()) if !wtx.readyEl.IsPresent() { - wtx.readyEl = utils.Some(txs.readyTxs.PushBack(wtx.Tx())) + wtx.readyEl = utils.Some(s.readyTxs.PushBack(wtx.Tx())) } } inner.byHash[wtx.Hash()] = wtx @@ -342,24 +357,24 @@ func (inner *txStoreInner) inInclusionOrder() []*WrappedTx { // Inserts a new transaction to txStore. // txStore takes ownership of wtx. -func (txs *txStore) Insert(wtx *WrappedTx) error { - for inner := range txs.inner.Lock() { - if err := txs.insert(inner, wtx); err != nil { +func (s *txStore) Insert(wtx *WrappedTx) error { + for inner := range s.inner.Lock() { + if err := s.insert(inner, wtx); err != nil { return err } if total := inner.state.Load().total; !total.LessEqual(&inner.hardLimit) { - txs.compact(inner, false) + s.compact(inner, false) if _, ok := inner.byHash[wtx.Hash()]; !ok { return errMempoolFull } } } - txs.cache.Push(wtx.Hash()) + s.cache.Push(wtx.Hash()) return nil } // O(m log m), prunes transactions above softLimit and recomputes all the indices. -func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { +func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) { // Order all txs by priority. wtxs := inner.inInclusionOrder() // Reset internal state. @@ -375,10 +390,10 @@ func (txs *txStore) compact(inner *txStoreInner, clearAccounts bool) { for _, wtx := range wtxs { total := inner.state.Load().total total.Inc(wtx.Size()) - if !total.LessEqual(&inner.softLimit) || txs.insert(inner, wtx) != nil { - txs.cache.Remove(wtx.Hash()) + if !total.LessEqual(&inner.softLimit) || s.insert(inner, wtx) != nil { + s.cache.Remove(wtx.Hash()) if el, ok := wtx.readyEl.Get(); ok { - txs.readyTxs.Remove(el) + s.readyTxs.Remove(el) } } } @@ -393,18 +408,18 @@ type updateSpec struct { NewPriorities map[types.TxHash]int64 } -func (txs *txStore) Update(spec updateSpec) { +func (s *txStore) Update(spec updateSpec) { minHeight := utils.None[int64]() - if ttl, ok := txs.config.TTLNumBlocks.Get(); ok && spec.Height > ttl { + if ttl, ok := s.config.TTLNumBlocks.Get(); ok && spec.Height > ttl { minHeight = utils.Some(spec.Height - ttl) } minTime := utils.None[time.Time]() - if d, ok := txs.config.TTLDuration.Get(); ok { + if d, ok := s.config.TTLDuration.Get(); ok { minTime = utils.Some(spec.Now.Add(-d)) } - for inner := range txs.inner.Lock() { + for inner := range s.inner.Lock() { isExpired := func(wtx *WrappedTx) bool { - if !txs.config.RemoveExpiredTxsFromQueue && inner.isReady(wtx) { + if !s.config.RemoveExpiredTxsFromQueue && inner.isReady(wtx) { return false } if t, ok := minTime.Get(); ok && wtx.timestamp.Before(t) { @@ -420,27 +435,27 @@ func (txs *txStore) Update(spec updateSpec) { if success, ok := spec.TxResults[wtx.Hash()]; ok { // Executed transactions should be removed. remove = true - if !txs.config.KeepInvalidTxsInCache { + if !s.config.KeepInvalidTxsInCache { if !success { // Failed txs are eligible for reexection once. - if txs.failedTxs.Push(txHash) { - txs.cache.Remove(txHash) + if s.failedTxs.Push(txHash) { + s.cache.Remove(txHash) } } else { - txs.failedTxs.Remove(txHash) + s.failedTxs.Remove(txHash) } } } if remove { delete(inner.byHash, txHash) if el, ok := wtx.readyEl.Get(); ok { - txs.readyTxs.Remove(el) + s.readyTxs.Remove(el) } } else if newPriority, ok := spec.NewPriorities[wtx.Hash()]; ok { wtx.priority = newPriority } } - txs.compact(inner, true) + s.compact(inner, true) } } @@ -455,7 +470,7 @@ type ReapLimits struct { // byte, and gas constraints together with the total estimated gas for the // returned transactions. Reaped txs are removed iff remove == true. // O(m log m) where m is the size of the txStore. -func (txs *txStore) Reap(l ReapLimits, remove bool) (types.Txs, int64) { +func (s *txStore) Reap(l ReapLimits, remove bool) (types.Txs, int64) { maxTxs := l.MaxTxs.Or(utils.Max[uint64]()) maxBytes := l.MaxBytes.Or(utils.Max[int64]()) maxGasWanted := l.MaxGasWanted.Or(utils.Max[int64]()) @@ -474,8 +489,8 @@ func (txs *txStore) Reap(l ReapLimits, remove bool) (types.Txs, int64) { totalSize := int64(0) var wtxs []*WrappedTx - for inner := range txs.inner.Lock() { - if uint64(inner.state.Load().ready.count) >= txs.config.TxNotifyThreshold { + for inner := range s.inner.Lock() { + if uint64(inner.state.Load().ready.count) >= s.config.TxNotifyThreshold { for _, wtx := range inner.inInclusionOrder() { if uint64(len(wtxs)) >= maxTxs || !inner.isReady(wtx) { break @@ -500,9 +515,9 @@ func (txs *txStore) Reap(l ReapLimits, remove bool) (types.Txs, int64) { for _, wtx := range wtxs { delete(inner.byHash, wtx.Hash()) if el, ok := wtx.readyEl.Get(); ok { - txs.readyTxs.Remove(el) + s.readyTxs.Remove(el) } - txs.compact(inner, false) + s.compact(inner, false) } } } diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index 4640ce4a43..588f7e9a2b 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -38,14 +38,15 @@ func TestTxStore_GetTxByHash(t *testing.T) { } key := wtx.Hash() - res := txs.ByHash(key) + res, ok := txs.ByHash(key) + require.False(t, ok) require.Nil(t, res) - txs.Insert(wtx) + require.NoError(t, txs.Insert(wtx)) - res = txs.ByHash(key) - require.NotNil(t, res) - require.Equal(t, wtx, res) + res, ok = txs.ByHash(key) + require.True(t, ok) + require.Equal(t, wtx.Tx(), res) } func TestTxStore_SetTx(t *testing.T) { @@ -57,11 +58,11 @@ func TestTxStore_SetTx(t *testing.T) { } key := wtx.Hash() - txs.Insert(wtx) + require.NoError(t, txs.Insert(wtx)) - res := txs.ByHash(key) - require.NotNil(t, res) - require.Equal(t, wtx, res) + res, ok := txs.ByHash(key) + require.True(t, ok) + require.Equal(t, wtx.Tx(), res) } func TestTxStore_Size(t *testing.T) { @@ -69,11 +70,11 @@ func TestTxStore_Size(t *testing.T) { numTxs := 1000 for i := range numTxs { - txStore.Insert(&WrappedTx{ + require.NoError(t, txStore.Insert(&WrappedTx{ hashedTx: newHashedTx(fmt.Appendf(nil, "test_tx_%d", i)), priority: int64(i), timestamp: time.Now(), - }) + })) } require.Equal(t, numTxs, txStore.State().total.count) From f55487d24b8d90ab1951e837c496b904ad522a67 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 20:44:55 +0200 Subject: [PATCH 025/100] test fix --- sei-tendermint/internal/consensus/mempool_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sei-tendermint/internal/consensus/mempool_test.go b/sei-tendermint/internal/consensus/mempool_test.go index 15e2411b4e..e08ebef9c5 100644 --- a/sei-tendermint/internal/consensus/mempool_test.go +++ b/sei-tendermint/internal/consensus/mempool_test.go @@ -247,7 +247,10 @@ func TestMempoolRmBadTx(t *testing.T) { // check for the tx for { - txs := cs.txMempool.ReapTxs(mempool.ReapLimits{MaxBytes: utils.Some(int64(len(txBytes)))}) + txs, _ := cs.txMempool.ReapTxs( + mempool.ReapLimits{MaxBytes: utils.Some(int64(len(txBytes)))}, + false, + ) if len(txs) == 0 { emptyMempoolCh <- struct{}{} return From 81c762c6b7df356a97cac07d6b4f3a9535153e3d Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 20:58:23 +0200 Subject: [PATCH 026/100] termination fix --- sei-cosmos/server/start.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sei-cosmos/server/start.go b/sei-cosmos/server/start.go index 1b8e6c8e53..514c922cc7 100644 --- a/sei-cosmos/server/start.go +++ b/sei-cosmos/server/start.go @@ -360,7 +360,7 @@ func startInProcess( } defer func() { if tmNode.IsRunning() { - tmNode.Wait() + tmNode.Stop() } }() // Add the tx service to the gRPC router. We only need to register this @@ -464,8 +464,6 @@ func startInProcess( } } - // Defer cancelling as the last so that it is called first during unwinding. - defer cancel() // wait for signal capture and gracefully return return WaitForQuitSignals(goCtx, restartCh) } From f83f36f84c8bb2b2bef72567638c4bdf14cdeb3f Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 21:00:26 +0200 Subject: [PATCH 027/100] fmt --- sei-tendermint/internal/mempool/tx.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 2511c238fe..41d84fae76 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -123,15 +123,15 @@ type txStoreInner struct { } // Properties: -// * tx is ready if all txs with lower nonces are ready or executed AND -// balance >= tx.requiredBalance -// * we keep at most 1 tx per nonce -// * we prefer ready tx to pending tx (then tx with the higher priority) for the same nonce -// * we don't store txs below account nonce. -// * account nonces are evaluated once per height -// * we keep at least capacity and up to 2*capacity txs -// * we reap by highest prio, while respecting nonces. -// * non-evm txs are always ready +// - tx is ready if all txs with lower nonces are ready or executed AND +// balance >= tx.requiredBalance +// - we keep at most 1 tx per nonce +// - we prefer ready tx to pending tx (then tx with the higher priority) for the same nonce +// - we don't store txs below account nonce. +// - account nonces are evaluated once per height +// - we keep at least capacity and up to 2*capacity txs +// - we reap by highest prio, while respecting nonces. +// - non-evm txs are always ready type txStore struct { config *Config app *proxy.Proxy @@ -144,14 +144,14 @@ type txStore struct { // * txs dropped due to pruning are removed from cache. // * txs successfully executed are kept in cache to avoid reinsert. // * txs failed execution are eligible to be reexecuted once (iff config.KeepInvalidTxsInCache). - cache *LRUTxCache + cache *LRUTxCache // Tracks transactions which already failed execution once // but are eligible for reexecution (not added yet to cache) failedTxs *LRUTxCache - inner utils.RWMutex[*txStoreInner] - state utils.AtomicRecv[txStoreState] - readyTxs *clist.CList[types.Tx] + inner utils.RWMutex[*txStoreInner] + state utils.AtomicRecv[txStoreState] + readyTxs *clist.CList[types.Tx] } func NewTxStore(cfg *Config, app *proxy.Proxy) *txStore { From 5a41d0554e930be12ecca4c7cacb8f61faa18c33 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 21:03:31 +0200 Subject: [PATCH 028/100] lint --- sei-tendermint/internal/mempool/tx.go | 2 +- sei-tendermint/internal/rpc/core/mempool.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 41d84fae76..ca6e991253 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -490,7 +490,7 @@ func (s *txStore) Reap(l ReapLimits, remove bool) (types.Txs, int64) { var wtxs []*WrappedTx for inner := range s.inner.Lock() { - if uint64(inner.state.Load().ready.count) >= s.config.TxNotifyThreshold { + if uint64(inner.state.Load().ready.count) >= s.config.TxNotifyThreshold { //nolint:gosec // count is non-negative for _, wtx := range inner.inInclusionOrder() { if uint64(len(wtxs)) >= maxTxs || !inner.isReady(wtx) { break diff --git a/sei-tendermint/internal/rpc/core/mempool.go b/sei-tendermint/internal/rpc/core/mempool.go index 8751639d19..b61cdae14c 100644 --- a/sei-tendermint/internal/rpc/core/mempool.go +++ b/sei-tendermint/internal/rpc/core/mempool.go @@ -137,7 +137,7 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque skipCount := validateSkipCount(page, perPage) txs, _ := env.Mempool.ReapTxs(mempool.ReapLimits{ - MaxTxs: utils.Some(uint64(skipCount + tmmath.MinInt(perPage, totalCount-skipCount))), + MaxTxs: utils.Some(uint64(skipCount + tmmath.MinInt(perPage, totalCount-skipCount))), //nolint:gosec // guaranteed to be non-negative }, false) if skipCount > len(txs) { skipCount = len(txs) From 90291871ab863bcd19daa6aff0674e1d79e25182 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 22:04:30 +0200 Subject: [PATCH 029/100] fixes --- sei-tendermint/internal/mempool/mempool.go | 11 ++--------- sei-tendermint/internal/mempool/tx.go | 16 ++++++++++++++++ sei-tendermint/rpc/client/rpc_test.go | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 51e37b9f3e..b374fe17dc 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -412,16 +412,9 @@ func (txmp *TxMempool) SafeGetTxsForHashes(txHashes []types.TxHash) (types.Txs, return txmp.txStore.SafeGetTxsForHashes(txHashes) } -// Flush empties the mempool. It acquires a read-lock, fetches all the -// transactions currently in the transaction store and removes each transaction -// from the store and all indexes and finally resets the cache. -// -// NOTE: -// - Flushing the mempool may leave the mempool in an inconsistent state. +// Flush empties the mempool. func (txmp *TxMempool) Flush() { - txmp.mtx.Lock() - defer txmp.mtx.Unlock() - txmp.txStore = NewTxStore(txmp.config, txmp.app) + txmp.txStore.Clear() } // ReapTxs returns a list of transactions within the provided constraints and their total gas estimate. diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index ca6e991253..b198ade62f 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -176,6 +176,22 @@ func NewTxStore(cfg *Config, app *proxy.Proxy) *txStore { } } +func (s *txStore) Clear() { + for inner := range s.inner.Lock() { + s.cache.Reset() + s.failedTxs.Reset() + inner.byHash = map[types.TxHash]*WrappedTx{} + inner.byNonce = map[evmAddrNonce]*WrappedTx{} + inner.accounts = map[common.Address]*evmAccount{} + inner.state.Store(txStoreState{}) + for el := s.readyTxs.Front(); el != nil; { + next := el.Next() + s.readyTxs.Remove(el) + el = next + } + } +} + // Checks if cache contains a given hash. func (s *txStore) CacheHas(txHash types.TxHash) bool { return s.cache.Has(txHash) diff --git a/sei-tendermint/rpc/client/rpc_test.go b/sei-tendermint/rpc/client/rpc_test.go index 09764e07d9..5827a929e2 100644 --- a/sei-tendermint/rpc/client/rpc_test.go +++ b/sei-tendermint/rpc/client/rpc_test.go @@ -445,7 +445,7 @@ func TestClientMethodCalls(t *testing.T) { require.Equal(t, initMempoolSize+1, pool.Size()) - txs := pool.ReapTxs(mempool.ReapLimits{MaxTxs: utils.Some(uint64(len(tx)))}) + txs, _ := pool.ReapTxs(mempool.ReapLimits{MaxTxs: utils.Some(uint64(len(tx)))}, false) require.Equal(t, tx, txs[0]) pool.Flush() }) From 77e8580e0ee61400764aef455ec6e9580b6a1fbb Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 22:13:11 +0200 Subject: [PATCH 030/100] fixes --- sei-tendermint/internal/autobahn/producer/state.go | 2 +- sei-tendermint/internal/mempool/mempool.go | 14 +++----------- sei-tendermint/internal/mempool/tx.go | 7 ++++++- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index 3bb2409c5a..07fff9e134 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -66,7 +66,7 @@ func (s *State) makePayload(ctx context.Context) (*types.Payload, error) { // Wait for transactions. We give up and produce an empty block if mempool is empty for // cfg.BlockInterval. _ = utils.WithTimeout(ctx, s.cfg.BlockInterval, func(ctx context.Context) error { - return s.txMempool.TxStore().WaitForTxs(ctx) + return s.txMempool.WaitForTxs(ctx) }) // If the context has been cancelled though, we just fail. if err := ctx.Err(); err != nil { diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index b374fe17dc..265d9eed96 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -150,11 +150,6 @@ func DefaultConfig() *Config { } } -type evmAddrNonce struct { - Address common.Address - Nonce uint64 -} - // TxMempool defines a prioritized mempool data structure used by the v1 mempool // reactor. It keeps a thread-safe priority queue of transactions that is used // when a block proposer constructs a block and a thread-safe linked-list that @@ -231,7 +226,9 @@ func (txmp *TxMempool) EvmNextPendingNonce(addr common.Address) uint64 { return txmp.txStore.NextNonce(addr) } -func (txmp *TxMempool) TxStore() *txStore { return txmp.txStore } +func (txmp *TxMempool) WaitForTxs(ctx context.Context) error { + return txmp.txStore.WaitForTxs(ctx) +} // Lock obtains a write-lock on the mempool. A caller must be sure to explicitly // release the lock when finished. @@ -406,9 +403,6 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response } func (txmp *TxMempool) SafeGetTxsForHashes(txHashes []types.TxHash) (types.Txs, []types.TxHash) { - txmp.mtx.RLock() - defer txmp.mtx.RUnlock() - return txmp.txStore.SafeGetTxsForHashes(txHashes) } @@ -430,8 +424,6 @@ func (txmp *TxMempool) Flush() { // NOTE: Transactions are removed from the mempool iff remove == true. // Either way, the transactions stay in the LRU cache. func (txmp *TxMempool) ReapTxs(limits ReapLimits, remove bool) (types.Txs, int64) { - txmp.mtx.Lock() - defer txmp.mtx.Unlock() return txmp.txStore.Reap(limits, remove) } diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index b198ade62f..c2b0a08200 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -21,6 +21,11 @@ var errOldNonce = errors.New("nonce too old") var errSameNonce = errors.New("tx with this nonce already in mempool") var errMempoolFull = errors.New("mempool full") +type evmAddrNonce struct { + Address common.Address + Nonce uint64 +} + type hashedTx struct { tx types.Tx hash types.TxHash @@ -205,7 +210,7 @@ func (s *txStore) CachePush(txHash types.TxHash) { // Size returns the total number of transactions in the store. func (s *txStore) State() txStoreState { return s.state.Load() } -// WaitForTxs waits until the store becomes non-empty. +// WaitForTxs waits until there is >0 ready txs. func (s *txStore) WaitForTxs(ctx context.Context) error { _, err := s.state.Wait(ctx, func(state txStoreState) bool { return state.ready.count > 0 }) return err From 7297954843934991b72edc5ce285be376673f08d Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 22:27:27 +0200 Subject: [PATCH 031/100] codex WIP --- sei-tendermint/internal/mempool/mempool.go | 3 ++- sei-tendermint/internal/mempool/metrics.go | 2 +- sei-tendermint/internal/mempool/tx.go | 5 ++++- sei-tendermint/internal/mempool/tx_test.go | 18 ++---------------- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 265d9eed96..584efa906d 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -208,7 +208,7 @@ func NewTxMempool( txsAvailable: make(chan struct{}, 1), height: -1, metrics: metrics, - txStore: NewTxStore(cfg, app), + txStore: NewTxStore(cfg, app, metrics), txConstraintsFetcher: txConstraintsFetcher, priorityReservoir: reservoir.New[int64](cfg.DropPriorityReservoirSize, cfg.DropPriorityThreshold, nil), // Use non-deterministic RNG } @@ -389,6 +389,7 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response } if err := txmp.txStore.Insert(wtx); err != nil { + txmp.metrics.RejectedTxs.Add(1) return nil, err } diff --git a/sei-tendermint/internal/mempool/metrics.go b/sei-tendermint/internal/mempool/metrics.go index db055894fe..a94293e851 100644 --- a/sei-tendermint/internal/mempool/metrics.go +++ b/sei-tendermint/internal/mempool/metrics.go @@ -60,7 +60,7 @@ type Metrics struct { // RejectedTxs defines the number of rejected transactions. These are // transactions that passed CheckTx but failed to make it into the mempool - // due to resource limits, e.g. mempool is full and no lower priority + // due to other constraints, e.g. mempool is full and no lower priority // transactions exist in the mempool. //metrics:Number of rejected transactions. RejectedTxs metrics.Counter diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index c2b0a08200..f94ad245a4 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -140,6 +140,7 @@ type txStoreInner struct { type txStore struct { config *Config app *proxy.Proxy + metrics *Metrics // Cache of already seen txs, reducess pressure on app. // It is a superset of transactions in txStore. @@ -159,7 +160,7 @@ type txStore struct { readyTxs *clist.CList[types.Tx] } -func NewTxStore(cfg *Config, app *proxy.Proxy) *txStore { +func NewTxStore(cfg *Config, app *proxy.Proxy, metrics *Metrics) *txStore { softLimit := txCounter{count: cfg.Size + cfg.PendingSize, bytes: utils.Clamp[uint64](cfg.MaxTxsBytes + cfg.MaxPendingTxsBytes)} hardLimit := txCounter{count: 2 * softLimit.count, bytes: 2 * softLimit.bytes} inner := &txStoreInner{ @@ -175,6 +176,7 @@ func NewTxStore(cfg *Config, app *proxy.Proxy) *txStore { cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), failedTxs: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), app: app, + metrics: metrics, inner: utils.NewRWMutex(inner), readyTxs: clist.New[types.Tx](), state: inner.state.Subscribe(), @@ -413,6 +415,7 @@ func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) { total.Inc(wtx.Size()) if !total.LessEqual(&inner.softLimit) || s.insert(inner, wtx) != nil { s.cache.Remove(wtx.Hash()) + s.metrics.EvictedTxs.Add(1) if el, ok := wtx.readyEl.Get(); ok { s.readyTxs.Remove(el) } diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index 588f7e9a2b..48cfb54cd9 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -2,31 +2,17 @@ package mempool import ( "fmt" - "math/big" "testing" "time" - "github.com/ethereum/go-ethereum/common" - abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/abci/example/kvstore" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) -type txStoreTestApp struct { - abci.BaseApplication -} - -func (txStoreTestApp) EvmNonce(common.Address) uint64 { - return 0 -} - -func (txStoreTestApp) EvmBalance(common.Address, []byte) *big.Int { - return big.NewInt(0) -} - func newTxStoreForTest() *txStore { - return NewTxStore(TestConfig(), proxy.New(txStoreTestApp{}, proxy.NopMetrics())) + return NewTxStore(TestConfig(), proxy.New(kvstore.NewApplication(), proxy.NopMetrics()), NopMetrics()) } func TestTxStore_GetTxByHash(t *testing.T) { From dc9e8202bb7183ddbae85f99815059f4a08f4bfc Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 22:40:29 +0200 Subject: [PATCH 032/100] metric fixes --- sei-tendermint/internal/mempool/tx.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index f94ad245a4..b96ddc4ed6 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -186,6 +186,7 @@ func NewTxStore(cfg *Config, app *proxy.Proxy, metrics *Metrics) *txStore { func (s *txStore) Clear() { for inner := range s.inner.Lock() { s.cache.Reset() + s.metrics.CacheSize.Set(float64(s.cache.Size())) s.failedTxs.Reset() inner.byHash = map[types.TxHash]*WrappedTx{} inner.byNonce = map[evmAddrNonce]*WrappedTx{} @@ -207,6 +208,7 @@ func (s *txStore) CacheHas(txHash types.TxHash) bool { // Pushes a tx to cache, effectively blocking it from being inserted. func (s *txStore) CachePush(txHash types.TxHash) { s.cache.Push(txHash) + s.metrics.CacheSize.Set(float64(s.cache.Size())) } // Size returns the total number of transactions in the store. @@ -298,7 +300,9 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { } // Remove the old transaction. s.cache.Remove(old.Hash()) + s.metrics.CacheSize.Set(float64(s.cache.Size())) delete(inner.byHash, old.Hash()) + s.metrics.RemovedTxs.Add(1) if el, ok := wtx.readyEl.Get(); ok { s.readyTxs.Remove(el) } @@ -393,6 +397,7 @@ func (s *txStore) Insert(wtx *WrappedTx) error { } } s.cache.Push(wtx.Hash()) + s.metrics.CacheSize.Set(float64(s.cache.Size())) return nil } @@ -415,6 +420,8 @@ func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) { total.Inc(wtx.Size()) if !total.LessEqual(&inner.softLimit) || s.insert(inner, wtx) != nil { s.cache.Remove(wtx.Hash()) + s.metrics.CacheSize.Set(float64(s.cache.Size())) + s.metrics.RemovedTxs.Add(1) s.metrics.EvictedTxs.Add(1) if el, ok := wtx.readyEl.Get(); ok { s.readyTxs.Remove(el) @@ -455,7 +462,8 @@ func (s *txStore) Update(spec updateSpec) { return false } for txHash, wtx := range inner.byHash { - remove := isExpired(wtx) || wtx.check(spec.Constraints) != nil + expired := isExpired(wtx) + remove := expired || wtx.check(spec.Constraints) != nil if success, ok := spec.TxResults[wtx.Hash()]; ok { // Executed transactions should be removed. remove = true @@ -464,6 +472,7 @@ func (s *txStore) Update(spec updateSpec) { // Failed txs are eligible for reexection once. if s.failedTxs.Push(txHash) { s.cache.Remove(txHash) + s.metrics.CacheSize.Set(float64(s.cache.Size())) } } else { s.failedTxs.Remove(txHash) @@ -471,7 +480,11 @@ func (s *txStore) Update(spec updateSpec) { } } if remove { + if expired { + s.metrics.ExpiredTxs.Add(1) + } delete(inner.byHash, txHash) + s.metrics.RemovedTxs.Add(1) if el, ok := wtx.readyEl.Get(); ok { s.readyTxs.Remove(el) } @@ -538,6 +551,7 @@ func (s *txStore) Reap(l ReapLimits, remove bool) (types.Txs, int64) { if remove { for _, wtx := range wtxs { delete(inner.byHash, wtx.Hash()) + s.metrics.RemovedTxs.Add(1) if el, ok := wtx.readyEl.Get(); ok { s.readyTxs.Remove(el) } From fc4a26e2574651693c15e69c2d29714eaf2e9bba Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 22:43:19 +0200 Subject: [PATCH 033/100] test fix --- sei-tendermint/internal/mempool/tx.go | 4 +- sei-tendermint/rpc/client/rpc_test.go | 53 ++++++++++++--------------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index b96ddc4ed6..777e985839 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -138,8 +138,8 @@ type txStoreInner struct { // - we reap by highest prio, while respecting nonces. // - non-evm txs are always ready type txStore struct { - config *Config - app *proxy.Proxy + config *Config + app *proxy.Proxy metrics *Metrics // Cache of already seen txs, reducess pressure on app. diff --git a/sei-tendermint/rpc/client/rpc_test.go b/sei-tendermint/rpc/client/rpc_test.go index 5827a929e2..ec39af03e2 100644 --- a/sei-tendermint/rpc/client/rpc_test.go +++ b/sei-tendermint/rpc/client/rpc_test.go @@ -12,7 +12,6 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/stretchr/testify/assert" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/sei-protocol/sei-chain/sei-tendermint/config" @@ -290,10 +289,10 @@ func TestClientMethodCalls(t *testing.T) { err = client.WaitForHeight(ctx, c, apph, nil) require.NoError(t, err) res, err := c.ABCIQuery(ctx, "/key", k) + require.NoError(t, err) qres := res.Response - if assert.NoError(t, err) && assert.True(t, qres.IsOK()) { - require.Equal(t, v, qres.Value) - } + require.True(t, qres.IsOK()) + require.Equal(t, v, qres.Value) }) t.Run("AppCalls", func(t *testing.T) { ctx := t.Context() @@ -323,10 +322,9 @@ func TestClientMethodCalls(t *testing.T) { _qres, err := c.ABCIQueryWithOptions(ctx, "/key", k, client.ABCIQueryOptions{Prove: false}) require.NoError(t, err) qres := _qres.Response - if assert.True(t, qres.IsOK()) { - require.Equal(t, k, qres.Key) - require.Equal(t, v, qres.Value) - } + require.True(t, qres.IsOK()) + require.Equal(t, k, qres.Key) + require.Equal(t, v, qres.Value) // make sure we can lookup the tx with proof ptx, err := c.Tx(ctx, bres.Hash, true) @@ -358,22 +356,20 @@ func TestClientMethodCalls(t *testing.T) { blockResults, err := c.BlockResults(ctx, &txh) require.NoError(t, err, "%d: %+v", i, err) require.Equal(t, txh, blockResults.Height) - if assert.Equal(t, 1, len(blockResults.TxsResults)) { - // check success code - require.Equal(t, 0, blockResults.TxsResults[0].Code) - } + require.Len(t, blockResults.TxsResults, 1) + // check success code + require.Equal(t, uint32(0), blockResults.TxsResults[0].Code) // check blockchain info, now that we know there is info info, err := c.BlockchainInfo(ctx, apph, apph) require.NoError(t, err) require.True(t, info.LastHeight >= apph) - if assert.Equal(t, 1, len(info.BlockMetas)) { - lastMeta := info.BlockMetas[0] - require.Equal(t, apph, lastMeta.Header.Height) - blockData := block.Block - require.Equal(t, blockData.Header.AppHash, lastMeta.Header.AppHash) - require.Equal(t, block.BlockID, lastMeta.BlockID) - } + require.Len(t, info.BlockMetas, 1) + lastMeta := info.BlockMetas[0] + require.Equal(t, apph, lastMeta.Header.Height) + blockData := block.Block + require.Equal(t, blockData.Header.AppHash, lastMeta.Header.AppHash) + require.Equal(t, block.BlockID, lastMeta.BlockID) // and get the corresponding commit with the same apphash commit, err := c.Commit(ctx, &apph) @@ -618,11 +614,11 @@ func TestClientMethodCallsAdvanced(t *testing.T) { if i == 2 { perPage = 2 } - assert.Equal(t, perPage, res.Count) - assert.Equal(t, 5, res.Total) - assert.Equal(t, pool.SizeBytes(), res.TotalBytes) + require.Equal(t, perPage, int(res.Count)) + require.Equal(t, 5, int(res.Total)) + require.Equal(t, pool.SizeBytes(), uint64(res.TotalBytes)) for _, tx := range res.Txs { - assert.Contains(t, txs, tx) + require.Contains(t, txs, tx) } } } @@ -654,9 +650,9 @@ func TestClientMethodCallsAdvanced(t *testing.T) { res, err := mc.NumUnconfirmedTxs(ctx) require.NoError(t, err, "%d: %+v", i, err) - assert.Equal(t, mempoolSize, res.Count) - assert.Equal(t, mempoolSize, res.Total) - assert.Equal(t, pool.SizeBytes(), res.TotalBytes) + require.Equal(t, mempoolSize, int(res.Count)) + require.Equal(t, mempoolSize, int(res.Total)) + require.Equal(t, pool.SizeBytes(), uint64(res.TotalBytes)) } pool.Flush() @@ -770,9 +766,8 @@ func TestClientMethodCallsAdvanced(t *testing.T) { require.Equal(t, find.Hash, ptx.Hash) // time to verify the proof - if assert.Equal(t, find.Tx, ptx.Proof.Data) { - require.NoError(t, ptx.Proof.Proof.Verify(ptx.Proof.RootHash, find.Hash)) - } + require.Equal(t, find.Tx, ptx.Proof.Data) + require.NoError(t, ptx.Proof.Proof.Verify(ptx.Proof.RootHash, find.Hash)) // query by height result, err = c.TxSearch(ctx, fmt.Sprintf("tx.height=%d", find.Height), true, nil, nil, "asc") From cfc0541f97a88870979c4b3ad00bf9d22defc178 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 23:17:31 +0200 Subject: [PATCH 034/100] codex WIP --- sei-tendermint/internal/mempool/tx_test.go | 137 +++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index 48cfb54cd9..817c8bdda5 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -2,11 +2,15 @@ package mempool import ( "fmt" + "math/big" + "slices" "testing" "time" + "github.com/ethereum/go-ethereum/common" "github.com/sei-protocol/sei-chain/sei-tendermint/abci/example/kvstore" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) @@ -65,3 +69,136 @@ func TestTxStore_Size(t *testing.T) { require.Equal(t, numTxs, txStore.State().total.count) } + +func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { + rng := utils.TestRng() + app := newEVMNonceApp() + txStore := NewTxStore(TestConfig(), proxy.New(app, proxy.NopMetrics()), NopMetrics()) + + type accountCase struct { + address common.Address + baseNonce uint64 + lastNonce uint64 + byNonce map[uint64]*WrappedTx + txs []*WrappedTx + } + + makeTx := func(address common.Address, nonce uint64) *WrappedTx { + return &WrappedTx{ + hashedTx: newHashedTx(utils.GenBytes(rng, 32)), + timestamp: time.Now(), + priority: rng.Int63n(1_000_000) + 1, + gasWanted: 1, + estimatedGas: 1, + evm: utils.Some(evmTx{ + address: address, + seiAddress: address.Bytes(), + nonce: nonce, + requiredBalance: big.NewInt(0), + }), + } + } + + // Seed the store with sparse per-account nonce ranges so each account has a + // mix of contiguous ready transactions and gaps that keep later transactions + // pending. + accounts := make([]accountCase, 8) + expectedInserted := 0 + for i := range accounts { + accounts[i].address = common.BytesToAddress(utils.GenBytes(rng, common.AddressLength)) + accounts[i].baseNonce = uint64(rng.Intn(20) + 1) + accounts[i].byNonce = map[uint64]*WrappedTx{} + rangeLen := rng.Intn(16) + 12 + accounts[i].lastNonce = accounts[i].baseNonce + uint64(rangeLen-1) + app.nextNonce[accounts[i].address.Hex()] = accounts[i].baseNonce + insertedForAccount := 0 + for offset := range rangeLen { + if rng.Intn(100) >= 80 { + continue + } + wtx := makeTx(accounts[i].address, accounts[i].baseNonce+uint64(offset)) + accounts[i].txs = append(accounts[i].txs, wtx) + accounts[i].byNonce[wtx.EVMNonce()] = wtx + require.NoError(t, txStore.Insert(wtx)) + expectedInserted++ + insertedForAccount++ + } + require.Positive(t, insertedForAccount) + + rejected := makeTx(accounts[i].address, accounts[i].baseNonce-1) + require.ErrorIs(t, txStore.Insert(rejected), errOldNonce) + } + + require.Equal(t, expectedInserted, txStore.State().total.count) + + // Advance the per-account nonce frontier in several randomized rounds and + // verify that Update removes every transaction that fell below the account + // nonce while preserving the rest. + for height := range int64(5) { + for _, account := range accounts { + currentNonce := app.nextNonce[account.address.Hex()] + if currentNonce > 0 { + rejected := makeTx(account.address, currentNonce-1) + require.ErrorIs(t, txStore.Insert(rejected), errOldNonce) + } + maxAdvance := max(0,int(account.lastNonce-currentNonce) + 4) + for range rng.Intn(maxAdvance + 1) { + app.markMined(account.address.Hex()) + } + } + + txStore.Update(updateSpec{ + Now: time.Now(), + Height: height + 1, + TxResults: map[types.TxHash]bool{}, + Constraints: NopTxConstraints(), + NewPriorities: map[types.TxHash]int64{}, + }) + + // Derive the expected remaining/ready sets from the test model: + // all txs at or above the current account nonce remain present, and the + // ready prefix is the contiguous run starting at the current nonce. + expectedRemaining := 0 + expectedReady := 0 + expectedReaped := make(types.Txs, 0, expectedRemaining) + for _, account := range accounts { + currentNonce := app.nextNonce[account.address.Hex()] + for nonce, wtx := range account.byNonce { + got, ok := txStore.ByHash(wtx.Hash()) + if nonce < currentNonce { + require.False(t, ok) + continue + } + require.True(t, ok) + require.Equal(t, wtx.Tx(), got) + expectedRemaining++ + } + for nonce := currentNonce; ; nonce++ { + wtx, ok := account.byNonce[nonce] + if !ok { + break + } + expectedReady++ + expectedReaped = append(expectedReaped, wtx.Tx()) + } + } + state := txStore.State() + require.Equal(t, expectedRemaining, state.total.count) + require.Equal(t, expectedReady, state.ready.count) + + // The ready set must agree across all public/readable surfaces: Reap and + // the internal ready list. + reaped, _ := txStore.Reap(ReapLimits{ + MaxTxs: utils.Some(uint64(expectedRemaining)), + }, false) + listed := make(types.Txs, 0, expectedReady) + for el := txStore.readyTxs.Front(); el != nil; el = el.Next() { + listed = append(listed, el.Value()) + } + slices.SortFunc(reaped, slices.Compare[types.Tx]) + slices.SortFunc(listed, slices.Compare[types.Tx]) + slices.SortFunc(expectedReaped, slices.Compare[types.Tx]) + require.ElementsMatch(t, expectedReaped, reaped) + require.ElementsMatch(t, expectedReaped, listed) + } +} From 917a73968e7ec73c384599bfc0be569ab0dcb870 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 23:29:56 +0200 Subject: [PATCH 035/100] codex WIP --- .../internal/mempool/recheck_drain_test.go | 49 +++++++++---- sei-tendermint/internal/mempool/tx.go | 3 + sei-tendermint/internal/mempool/tx_test.go | 68 ++++++++++++++----- 3 files changed, 91 insertions(+), 29 deletions(-) diff --git a/sei-tendermint/internal/mempool/recheck_drain_test.go b/sei-tendermint/internal/mempool/recheck_drain_test.go index db8b01d6e8..b54f4ad2a4 100644 --- a/sei-tendermint/internal/mempool/recheck_drain_test.go +++ b/sei-tendermint/internal/mempool/recheck_drain_test.go @@ -28,21 +28,40 @@ type evmNonceApp struct { abci.Application mu sync.Mutex - nextNonce map[string]uint64 + nextNonce map[common.Address]uint64 + balance map[common.Address]*big.Int } func newEVMNonceApp() *evmNonceApp { - return &evmNonceApp{nextNonce: map[string]uint64{}} + return &evmNonceApp{ + nextNonce: map[common.Address]uint64{}, + balance: map[common.Address]*big.Int{}, + } } // markMined bumps the sender's next-expected nonce by 1, simulating that the // previous next-expected nonce just landed in a block. -func (a *evmNonceApp) markMined(sender string) { +func (a *evmNonceApp) markMined(sender common.Address) { a.mu.Lock() a.nextNonce[sender]++ a.mu.Unlock() } +func (a *evmNonceApp) setBalance(sender common.Address, balance *big.Int) { + a.mu.Lock() + a.balance[sender] = new(big.Int).Set(balance) + a.mu.Unlock() +} + +func (a *evmNonceApp) balanceOf(sender common.Address) *big.Int { + a.mu.Lock() + defer a.mu.Unlock() + if balance, ok := a.balance[sender]; ok { + return new(big.Int).Set(balance) + } + return big.NewInt(0) +} + func (a *evmNonceApp) parseTx(tx []byte) (sender string, nonce uint64, priority int64, ok bool) { parts := bytes.Split(tx, []byte("=")) if len(parts) != 4 || string(parts[0]) != "evm" { @@ -64,9 +83,10 @@ func (a *evmNonceApp) CheckTx(_ context.Context, req *abci.RequestCheckTxV2) *ab if !ok { return &abci.ResponseCheckTxV2{ResponseCheckTx: &abci.ResponseCheckTx{Code: 1}} } + senderAddr := common.HexToAddress(sender) a.mu.Lock() - expected := a.nextNonce[sender] + expected := a.nextNonce[senderAddr] a.mu.Unlock() if nonce < expected { @@ -82,8 +102,8 @@ func (a *evmNonceApp) CheckTx(_ context.Context, req *abci.RequestCheckTxV2) *ab GasEstimated: DefaultGasEstimated, }, EVMNonce: nonce, - EVMSenderAddress: common.HexToAddress(sender), - SeiSenderAddress: sdk.AccAddress(common.HexToAddress(sender).Bytes()), + EVMSenderAddress: senderAddr, + SeiSenderAddress: sdk.AccAddress(senderAddr.Bytes()), IsEVM: true, EVMRequiredBalance: big.NewInt(0), } @@ -97,10 +117,15 @@ func (a *evmNonceApp) GetTxPriorityHint(context.Context, *abci.RequestGetTxPrior func (a *evmNonceApp) EvmNonce(addr common.Address) uint64 { a.mu.Lock() defer a.mu.Unlock() - return a.nextNonce[addr.Hex()] + return a.nextNonce[addr] } -func (a *evmNonceApp) EvmBalance(common.Address, []byte) *big.Int { +func (a *evmNonceApp) EvmBalance(addr common.Address, _ []byte) *big.Int { + a.mu.Lock() + defer a.mu.Unlock() + if balance, ok := a.balance[addr]; ok { + return new(big.Int).Set(balance) + } return big.NewInt(0) } @@ -152,7 +177,7 @@ func TestTxMempool_DescendingNonceDrain(t *testing.T) { txResults := make([]*abci.ExecTxResult, len(txs)) for i := range txs { - app.markMined(sender.Hex()) + app.markMined(sender) txResults[i] = &abci.ExecTxResult{Code: code.CodeTypeOK} } totalMined += len(txs) @@ -163,7 +188,7 @@ func TestTxMempool_DescendingNonceDrain(t *testing.T) { require.Equal(t, N, totalMined, "all N txs should have mined within %d blocks", maxBlocks) require.Zero(t, txmp.Size(), "mempool should fully drain within %d blocks", maxBlocks) - require.Equal(t, uint64(N), app.nextNonce[sender.Hex()], "all N nonces should have been mined") + require.Equal(t, uint64(N), app.nextNonce[sender], "all N nonces should have been mined") } func TestTxMempool_EvmNextPendingNonceIncludesPendingTransactions(t *testing.T) { @@ -171,7 +196,7 @@ func TestTxMempool_EvmNextPendingNonceIncludesPendingTransactions(t *testing.T) sender := common.HexToAddress("0x00000000000000000000000000000000000000aa") app := newEVMNonceApp() - app.nextNonce[sender.Hex()] = 5 + app.nextNonce[sender] = 5 txmp := setup(t, proxy.New(app, proxy.NopMetrics()), 5000, NopTxConstraintsFetcher) for _, nonce := range []uint64{7, 5, 6} { @@ -190,7 +215,7 @@ func TestTxMempool_EvmNextPendingNonceReplacesSameNonceByPriority(t *testing.T) sender := common.HexToAddress("0x00000000000000000000000000000000000000bb") app := newEVMNonceApp() - app.nextNonce[sender.Hex()] = 5 + app.nextNonce[sender] = 5 txmp := setup(t, proxy.New(app, proxy.NopMetrics()), 5000, NopTxConstraintsFetcher) lowPriorityTx := []byte(fmt.Sprintf("evm=%s=%d=%d", sender.Hex(), 6, 1)) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 777e985839..3aaff1f154 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -157,6 +157,9 @@ type txStore struct { inner utils.RWMutex[*txStoreInner] state utils.AtomicRecv[txStoreState] + // List of transactions that were ready now OR at some point in the past. + // It is used for gossip and has to be stable - we cannot afford removing and reinserting transactions to this list, + // because it would cause them to be regossiped. readyTxs *clist.CList[types.Tx] } diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index 817c8bdda5..1531797425 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -3,7 +3,6 @@ package mempool import ( "fmt" "math/big" - "slices" "testing" "time" @@ -84,6 +83,7 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { } makeTx := func(address common.Address, nonce uint64) *WrappedTx { + requiredBalance := big.NewInt(rng.Int63n(256)) return &WrappedTx{ hashedTx: newHashedTx(utils.GenBytes(rng, 32)), timestamp: time.Now(), @@ -94,7 +94,7 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { address: address, seiAddress: address.Bytes(), nonce: nonce, - requiredBalance: big.NewInt(0), + requiredBalance: requiredBalance, }), } } @@ -103,6 +103,7 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { // mix of contiguous ready transactions and gaps that keep later transactions // pending. accounts := make([]accountCase, 8) + everReady := map[types.TxHash]struct{}{} expectedInserted := 0 for i := range accounts { accounts[i].address = common.BytesToAddress(utils.GenBytes(rng, common.AddressLength)) @@ -110,7 +111,8 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { accounts[i].byNonce = map[uint64]*WrappedTx{} rangeLen := rng.Intn(16) + 12 accounts[i].lastNonce = accounts[i].baseNonce + uint64(rangeLen-1) - app.nextNonce[accounts[i].address.Hex()] = accounts[i].baseNonce + app.nextNonce[accounts[i].address] = accounts[i].baseNonce + app.setBalance(accounts[i].address, big.NewInt(rng.Int63n(256))) insertedForAccount := 0 for offset := range rangeLen { if rng.Intn(100) >= 80 { @@ -131,20 +133,37 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { require.Equal(t, expectedInserted, txStore.State().total.count) + // Seed the stable-ready history with transactions that are already ready + // after the initial inserts. + for _, account := range accounts { + balance := app.balanceOf(account.address) + for nonce := account.baseNonce; ; nonce++ { + wtx, ok := account.byNonce[nonce] + if !ok { + break + } + if wtx.evm.OrPanic("evm tx").requiredBalance.Cmp(balance) > 0 { + break + } + everReady[wtx.Hash()] = struct{}{} + } + } + // Advance the per-account nonce frontier in several randomized rounds and // verify that Update removes every transaction that fell below the account // nonce while preserving the rest. for height := range int64(5) { for _, account := range accounts { - currentNonce := app.nextNonce[account.address.Hex()] + currentNonce := app.nextNonce[account.address] if currentNonce > 0 { rejected := makeTx(account.address, currentNonce-1) require.ErrorIs(t, txStore.Insert(rejected), errOldNonce) } maxAdvance := max(0,int(account.lastNonce-currentNonce) + 4) for range rng.Intn(maxAdvance + 1) { - app.markMined(account.address.Hex()) + app.markMined(account.address) } + app.setBalance(account.address, big.NewInt(rng.Int63n(256))) } txStore.Update(updateSpec{ @@ -160,9 +179,10 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { // ready prefix is the contiguous run starting at the current nonce. expectedRemaining := 0 expectedReady := 0 - expectedReaped := make(types.Txs, 0, expectedRemaining) + expectedStableReady := 0 for _, account := range accounts { - currentNonce := app.nextNonce[account.address.Hex()] + currentNonce := app.nextNonce[account.address] + balance := app.balanceOf(account.address) for nonce, wtx := range account.byNonce { got, ok := txStore.ByHash(wtx.Hash()) if nonce < currentNonce { @@ -172,33 +192,47 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { require.True(t, ok) require.Equal(t, wtx.Tx(), got) expectedRemaining++ + if _, wasReady := everReady[wtx.Hash()]; wasReady { + expectedStableReady++ + } } for nonce := currentNonce; ; nonce++ { wtx, ok := account.byNonce[nonce] if !ok { break } + if wtx.evm.OrPanic("evm tx").requiredBalance.Cmp(balance) > 0 { + break + } expectedReady++ - expectedReaped = append(expectedReaped, wtx.Tx()) + if _, wasReady := everReady[wtx.Hash()]; !wasReady { + everReady[wtx.Hash()] = struct{}{} + expectedStableReady++ + } } } state := txStore.State() require.Equal(t, expectedRemaining, state.total.count) require.Equal(t, expectedReady, state.ready.count) - // The ready set must agree across all public/readable surfaces: Reap and - // the internal ready list. + // Reap returns the currently ready transactions, while readyTxs is a + // stable list of transactions that have become ready at least once and + // have not been removed from the store. reaped, _ := txStore.Reap(ReapLimits{ MaxTxs: utils.Some(uint64(expectedRemaining)), }, false) - listed := make(types.Txs, 0, expectedReady) + listed := make(types.Txs, 0, expectedStableReady) + listedSet := make(map[types.TxHash]struct{}, expectedStableReady) for el := txStore.readyTxs.Front(); el != nil; el = el.Next() { - listed = append(listed, el.Value()) + tx := el.Value() + listed = append(listed, tx) + listedSet[tx.Hash()] = struct{}{} + } + require.Len(t, reaped, expectedReady) + require.Len(t, listed, expectedStableReady) + for _, tx := range reaped { + _, ok := listedSet[tx.Hash()] + require.True(t, ok) } - slices.SortFunc(reaped, slices.Compare[types.Tx]) - slices.SortFunc(listed, slices.Compare[types.Tx]) - slices.SortFunc(expectedReaped, slices.Compare[types.Tx]) - require.ElementsMatch(t, expectedReaped, reaped) - require.ElementsMatch(t, expectedReaped, listed) } } From cf803e1927abf5e39ea9a1079db34d325f62177a Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 20 May 2026 23:56:16 +0200 Subject: [PATCH 036/100] CList.Clear and replacement test --- sei-tendermint/internal/libs/clist/clist.go | 15 +++ .../internal/libs/clist/clist_test.go | 12 +-- sei-tendermint/internal/mempool/tx.go | 21 ++--- sei-tendermint/internal/mempool/tx_test.go | 91 +++++++++++++++++++ 4 files changed, 118 insertions(+), 21 deletions(-) diff --git a/sei-tendermint/internal/libs/clist/clist.go b/sei-tendermint/internal/libs/clist/clist.go index 7307c0b497..f74c586d37 100644 --- a/sei-tendermint/internal/libs/clist/clist.go +++ b/sei-tendermint/internal/libs/clist/clist.go @@ -194,6 +194,21 @@ func (l *CList[T]) Back() *CElement[T] { return back } +func (l *CList[T]) Clear() { + l.mtx.Lock() + defer l.mtx.Unlock() + + for el := l.head; el != nil; { + next := el.Next() + el.setRemoved() + el = next + } + l.waitCh = make(chan struct{}) + l.head = nil + l.tail = nil + l.len = 0 +} + // Panics if list grows beyond its max length. func (l *CList[T]) PushBack(v T) *CElement[T] { l.mtx.Lock() diff --git a/sei-tendermint/internal/libs/clist/clist_test.go b/sei-tendermint/internal/libs/clist/clist_test.go index 3573cdb7dd..9a8fdb8723 100644 --- a/sei-tendermint/internal/libs/clist/clist_test.go +++ b/sei-tendermint/internal/libs/clist/clist_test.go @@ -75,13 +75,7 @@ func TestGCFifo(t *testing.T) { }) } - for el := l.Front(); el != nil; { - l.Remove(el) - // oldEl := el - el = el.Next() - // oldEl.DetachPrev() - // oldEl.DetachNext() - } + l.Clear() tickerQuitCh := make(chan struct{}) tickerDoneCh := make(chan struct{}) @@ -242,9 +236,7 @@ func TestScanRightDeleteRandom(t *testing.T) { close(stop) // And remove all the elements. - for el := l.Front(); el != nil; el = el.Next() { - l.Remove(el) - } + l.Clear() if l.Len() != 0 { t.Fatal("Failed to remove all elements from CList") } diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 3aaff1f154..aa04273732 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -195,11 +195,7 @@ func (s *txStore) Clear() { inner.byNonce = map[evmAddrNonce]*WrappedTx{} inner.accounts = map[common.Address]*evmAccount{} inner.state.Store(txStoreState{}) - for el := s.readyTxs.Front(); el != nil; { - next := el.Next() - s.readyTxs.Remove(el) - el = next - } + s.readyTxs.Clear() } } @@ -293,8 +289,9 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { } an := evmAddrNonce{evm.address, evm.nonce} if old, ok := inner.byNonce[an]; ok { + oldReady := old.evm.OrPanic("non-evm tx").nonce < account.nextNonce // If the old tx is ready but the new tx is not, then reject the new tx. - if old.evm.OrPanic("non-evm tx").nonce < account.nextNonce && account.balance.Cmp(evm.requiredBalance) < 0 { + if oldReady && account.balance.Cmp(evm.requiredBalance) < 0 { return errSameNonce } // If the old tx has >= priority, then reject new tx. @@ -306,12 +303,14 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { s.metrics.CacheSize.Set(float64(s.cache.Size())) delete(inner.byHash, old.Hash()) s.metrics.RemovedTxs.Add(1) - if el, ok := wtx.readyEl.Get(); ok { - s.readyTxs.Remove(el) - } - state.ready.Dec(old.Size()) state.total.Dec(old.Size()) - state.ready.Inc(wtx.Size()) + if oldReady { + state.ready.Dec(old.Size()) + state.ready.Inc(wtx.Size()) + if !wtx.readyEl.IsPresent() { + wtx.readyEl = utils.Some(s.readyTxs.PushBack(wtx.Tx())) + } + } } state.total.Inc(wtx.Size()) inner.byNonce[an] = wtx diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index 1531797425..3b5c3ff45a 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -18,6 +18,18 @@ func newTxStoreForTest() *txStore { return NewTxStore(TestConfig(), proxy.New(kvstore.NewApplication(), proxy.NopMetrics()), NopMetrics()) } +func txStoreStateForTest(ready, pending []*WrappedTx) txStoreState { + state := txStoreState{} + for _, wtx := range ready { + state.ready.Inc(wtx.Size()) + state.total.Inc(wtx.Size()) + } + for _, wtx := range pending { + state.total.Inc(wtx.Size()) + } + return state +} + func TestTxStore_GetTxByHash(t *testing.T) { txs := newTxStoreForTest() wtx := &WrappedTx{ @@ -236,3 +248,82 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { } } } + +func TestTxStore_ReplacesSameNonceByHigherPriority(t *testing.T) { + rng := utils.TestRng() + app := newEVMNonceApp() + txStore := NewTxStore(TestConfig(), proxy.New(app, proxy.NopMetrics()), NopMetrics()) + + makeTx := func(address common.Address, nonce uint64, priority int64, requiredBalance int) *WrappedTx { + return &WrappedTx{ + hashedTx: newHashedTx(utils.GenBytes(rng, rng.Intn(48)+16)), + timestamp: time.Now(), + priority: priority, + gasWanted: 1, + estimatedGas: 1, + evm: utils.Some(evmTx{ + address: address, + seiAddress: address.Bytes(), + nonce: nonce, + requiredBalance: big.NewInt(int64(requiredBalance)), + }), + } + } + + assertState := func(expected txStoreState, expectedReady types.Txs) { + t.Helper() + require.Equal(t, expected, txStore.State()) + reaped, _ := txStore.Reap(ReapLimits{MaxTxs: utils.Some(uint64(expected.total.count))}, false) + require.Equal(t, expectedReady, reaped) + } + + address := common.BytesToAddress(utils.GenBytes(rng, common.AddressLength)) + app.nextNonce[address] = 7 + app.setBalance(address, big.NewInt(100)) + + // Insert one ready transaction, then replace it with a higher-priority ready + // transaction for the same nonce. + old := makeTx(address, 7, 10, 20) + require.NoError(t, txStore.Insert(old)) + assertState(txStoreStateForTest([]*WrappedTx{old}, nil), types.Txs{old.Tx()}) + + replacement := makeTx(address, 7, 20, 30) + require.NoError(t, txStore.Insert(replacement)) + assertState(txStoreStateForTest([]*WrappedTx{replacement}, nil), types.Txs{replacement.Tx()}) + _, ok := txStore.ByHash(old.Hash()) + require.False(t, ok) + got, ok := txStore.ByHash(replacement.Hash()) + require.True(t, ok) + require.Equal(t, replacement.Tx(), got) + + // A higher-priority transaction that would no longer be ready must not + // replace the current ready transaction for the same nonce. + blocked := makeTx(address, 7, 30, 101) + require.ErrorIs(t, txStore.Insert(blocked), errSameNonce) + + assertState(txStoreStateForTest([]*WrappedTx{replacement}, nil), types.Txs{replacement.Tx()}) + got, ok = txStore.ByHash(replacement.Hash()) + require.True(t, ok) + require.Equal(t, replacement.Tx(), got) + _, ok = txStore.ByHash(blocked.Hash()) + require.False(t, ok) + + // If the existing transaction is pending, priority alone decides + // replacement for the same nonce. + txStore.Clear() + app.nextNonce[address] = 7 + app.setBalance(address, big.NewInt(0)) + + pending := makeTx(address, 7, 70, 40) + require.NoError(t, txStore.Insert(pending)) + assertState(txStoreStateForTest(nil, []*WrappedTx{pending}), nil) + + pendingReplacement := makeTx(address, 7, 90, 50) + require.NoError(t, txStore.Insert(pendingReplacement)) + assertState(txStoreStateForTest(nil, []*WrappedTx{pendingReplacement}), nil) + _, ok = txStore.ByHash(pending.Hash()) + require.False(t, ok) + got, ok = txStore.ByHash(pendingReplacement.Hash()) + require.True(t, ok) + require.Equal(t, pendingReplacement.Tx(), got) +} From b516f9483713afe7f57da8fdd43b3fef2aa72d95 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 00:04:52 +0200 Subject: [PATCH 037/100] applied codex comments --- sei-tendermint/internal/mempool/mempool.go | 5 ----- sei-tendermint/internal/mempool/tx.go | 6 +++--- sei-tendermint/internal/mempool/tx_test.go | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 584efa906d..6463130490 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -166,11 +166,6 @@ type TxMempool struct { // height defines the last block height process during Update() height int64 - // blockFailedTxs tracks tx hashes that have previously failed during - // block execution. Used to prevent infinite re-entry of txs that - // consistently fail before fee charging in DeliverTx. - blockFailedTxs *LRUTxCache - // A TTL cache which keeps all txs that we have seen before over the TTL window. // Currently, this can be used for tracking whether checkTx is always serving the same tx or not. duplicateTxsCache utils.Option[*DuplicateTxCache] diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index aa04273732..66e348eca9 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -155,8 +155,8 @@ type txStore struct { // but are eligible for reexecution (not added yet to cache) failedTxs *LRUTxCache - inner utils.RWMutex[*txStoreInner] - state utils.AtomicRecv[txStoreState] + inner utils.RWMutex[*txStoreInner] + state utils.AtomicRecv[txStoreState] // List of transactions that were ready now OR at some point in the past. // It is used for gossip and has to be stable - we cannot afford removing and reinserting transactions to this list, // because it would cause them to be regossiped. @@ -557,8 +557,8 @@ func (s *txStore) Reap(l ReapLimits, remove bool) (types.Txs, int64) { if el, ok := wtx.readyEl.Get(); ok { s.readyTxs.Remove(el) } - s.compact(inner, false) } + s.compact(inner, false) } } diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index 3b5c3ff45a..1041de1f0e 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -171,7 +171,7 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { rejected := makeTx(account.address, currentNonce-1) require.ErrorIs(t, txStore.Insert(rejected), errOldNonce) } - maxAdvance := max(0,int(account.lastNonce-currentNonce) + 4) + maxAdvance := max(0, int(account.lastNonce-currentNonce)+4) for range rng.Intn(maxAdvance + 1) { app.markMined(account.address) } From 94bbd6cb3fac6f5526dea3cb421659235976ac54 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 00:29:32 +0200 Subject: [PATCH 038/100] applied codex comments --- sei-tendermint/internal/mempool/tx.go | 7 +- sei-tendermint/internal/mempool/tx_test.go | 173 ++++++++++++++------- 2 files changed, 125 insertions(+), 55 deletions(-) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 66e348eca9..dde254b6ed 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -304,12 +304,13 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { delete(inner.byHash, old.Hash()) s.metrics.RemovedTxs.Add(1) state.total.Dec(old.Size()) + if el, ok := old.readyEl.Get(); ok { + s.readyTxs.Remove(el) + } if oldReady { state.ready.Dec(old.Size()) state.ready.Inc(wtx.Size()) - if !wtx.readyEl.IsPresent() { - wtx.readyEl = utils.Some(s.readyTxs.PushBack(wtx.Tx())) - } + wtx.readyEl = utils.Some(s.readyTxs.PushBack(wtx.Tx())) } } state.total.Inc(wtx.Size()) diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index 1041de1f0e..fcf2e51590 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -249,81 +249,150 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { } } -func TestTxStore_ReplacesSameNonceByHigherPriority(t *testing.T) { - rng := utils.TestRng() +type txStoreReplacementTestEnv struct { + address common.Address + app *evmNonceApp + txStore *txStore +} + +func newTxStoreReplacementTestEnv(t *testing.T, rng utils.Rng) txStoreReplacementTestEnv { + t.Helper() app := newEVMNonceApp() - txStore := NewTxStore(TestConfig(), proxy.New(app, proxy.NopMetrics()), NopMetrics()) + return txStoreReplacementTestEnv{ + address: common.BytesToAddress(utils.GenBytes(rng, common.AddressLength)), + app: app, + txStore: NewTxStore(TestConfig(), proxy.New(app, proxy.NopMetrics()), NopMetrics()), + } +} - makeTx := func(address common.Address, nonce uint64, priority int64, requiredBalance int) *WrappedTx { - return &WrappedTx{ - hashedTx: newHashedTx(utils.GenBytes(rng, rng.Intn(48)+16)), - timestamp: time.Now(), - priority: priority, - gasWanted: 1, - estimatedGas: 1, - evm: utils.Some(evmTx{ - address: address, - seiAddress: address.Bytes(), - nonce: nonce, - requiredBalance: big.NewInt(int64(requiredBalance)), - }), +func (e txStoreReplacementTestEnv) makeTx(rng utils.Rng, nonce uint64, priority int64, requiredBalance int) *WrappedTx { + return &WrappedTx{ + hashedTx: newHashedTx(utils.GenBytes(rng, rng.Intn(48)+16)), + timestamp: time.Now(), + priority: priority, + gasWanted: 1, + estimatedGas: 1, + evm: utils.Some(evmTx{ + address: e.address, + seiAddress: e.address.Bytes(), + nonce: nonce, + requiredBalance: big.NewInt(int64(requiredBalance)), + }), + } +} + +func (e txStoreReplacementTestEnv) assertState(t *testing.T, ready, pending []*WrappedTx) { + t.Helper() + expected := txStoreStateForTest(ready, pending) + require.Equal(t, expected, e.txStore.State()) + reaped, _ := e.txStore.Reap(ReapLimits{MaxTxs: utils.Some(uint64(expected.total.count))}, false) + var expectedReady types.Txs + if len(ready) > 0 { + expectedReady = make(types.Txs, 0, len(ready)) + for _, wtx := range ready { + expectedReady = append(expectedReady, wtx.Tx()) } } + require.Equal(t, expectedReady, reaped) +} - assertState := func(expected txStoreState, expectedReady types.Txs) { - t.Helper() - require.Equal(t, expected, txStore.State()) - reaped, _ := txStore.Reap(ReapLimits{MaxTxs: utils.Some(uint64(expected.total.count))}, false) - require.Equal(t, expectedReady, reaped) +func (e txStoreReplacementTestEnv) assertReadyList(t *testing.T, expected types.Txs) { + t.Helper() + var listed types.Txs + for el := e.txStore.readyTxs.Front(); el != nil; el = el.Next() { + listed = append(listed, el.Value()) } + require.Equal(t, expected, listed) +} - address := common.BytesToAddress(utils.GenBytes(rng, common.AddressLength)) - app.nextNonce[address] = 7 - app.setBalance(address, big.NewInt(100)) +func TestTxStore_ReplacesReadyTxByHigherPriority(t *testing.T) { + rng := utils.TestRng() + env := newTxStoreReplacementTestEnv(t, rng) + env.app.nextNonce[env.address] = 7 + env.app.setBalance(env.address, big.NewInt(100)) // Insert one ready transaction, then replace it with a higher-priority ready // transaction for the same nonce. - old := makeTx(address, 7, 10, 20) - require.NoError(t, txStore.Insert(old)) - assertState(txStoreStateForTest([]*WrappedTx{old}, nil), types.Txs{old.Tx()}) - - replacement := makeTx(address, 7, 20, 30) - require.NoError(t, txStore.Insert(replacement)) - assertState(txStoreStateForTest([]*WrappedTx{replacement}, nil), types.Txs{replacement.Tx()}) - _, ok := txStore.ByHash(old.Hash()) + old := env.makeTx(rng, 7, 10, 20) + require.NoError(t, env.txStore.Insert(old)) + env.assertState(t, []*WrappedTx{old}, nil) + env.assertReadyList(t, types.Txs{old.Tx()}) + + replacement := env.makeTx(rng, 7, 20, 30) + require.NoError(t, env.txStore.Insert(replacement)) + env.assertState(t, []*WrappedTx{replacement}, nil) + env.assertReadyList(t, types.Txs{replacement.Tx()}) + _, ok := env.txStore.ByHash(old.Hash()) require.False(t, ok) - got, ok := txStore.ByHash(replacement.Hash()) + got, ok := env.txStore.ByHash(replacement.Hash()) require.True(t, ok) require.Equal(t, replacement.Tx(), got) // A higher-priority transaction that would no longer be ready must not // replace the current ready transaction for the same nonce. - blocked := makeTx(address, 7, 30, 101) - require.ErrorIs(t, txStore.Insert(blocked), errSameNonce) + blocked := env.makeTx(rng, 7, 30, 101) + require.ErrorIs(t, env.txStore.Insert(blocked), errSameNonce) - assertState(txStoreStateForTest([]*WrappedTx{replacement}, nil), types.Txs{replacement.Tx()}) - got, ok = txStore.ByHash(replacement.Hash()) + env.assertState(t, []*WrappedTx{replacement}, nil) + env.assertReadyList(t, types.Txs{replacement.Tx()}) + got, ok = env.txStore.ByHash(replacement.Hash()) require.True(t, ok) require.Equal(t, replacement.Tx(), got) - _, ok = txStore.ByHash(blocked.Hash()) + _, ok = env.txStore.ByHash(blocked.Hash()) require.False(t, ok) +} - // If the existing transaction is pending, priority alone decides - // replacement for the same nonce. - txStore.Clear() - app.nextNonce[address] = 7 - app.setBalance(address, big.NewInt(0)) - - pending := makeTx(address, 7, 70, 40) - require.NoError(t, txStore.Insert(pending)) - assertState(txStoreStateForTest(nil, []*WrappedTx{pending}), nil) +func TestTxStore_ReplacesReadyThenPendingTxByHigherPriority(t *testing.T) { + rng := utils.TestRng() + env := newTxStoreReplacementTestEnv(t, rng) + env.app.nextNonce[env.address] = 7 + env.app.setBalance(env.address, big.NewInt(100)) + + becamePending := env.makeTx(rng, 7, 40, 60) + require.NoError(t, env.txStore.Insert(becamePending)) + env.assertState(t, []*WrappedTx{becamePending}, nil) + env.assertReadyList(t, types.Txs{becamePending.Tx()}) + + env.app.setBalance(env.address, big.NewInt(50)) + env.txStore.Update(updateSpec{ + Now: time.Now(), + Height: 1, + TxResults: map[types.TxHash]bool{}, + Constraints: NopTxConstraints(), + NewPriorities: map[types.TxHash]int64{}, + }) + env.assertState(t, nil, []*WrappedTx{becamePending}) + env.assertReadyList(t, types.Txs{becamePending.Tx()}) + + becamePendingReplacement := env.makeTx(rng, 7, 50, 70) + require.NoError(t, env.txStore.Insert(becamePendingReplacement)) + env.assertState(t, nil, []*WrappedTx{becamePendingReplacement}) + env.assertReadyList(t, nil) + _, ok := env.txStore.ByHash(becamePending.Hash()) + require.False(t, ok) + got, ok := env.txStore.ByHash(becamePendingReplacement.Hash()) + require.True(t, ok) + require.Equal(t, becamePendingReplacement.Tx(), got) +} - pendingReplacement := makeTx(address, 7, 90, 50) - require.NoError(t, txStore.Insert(pendingReplacement)) - assertState(txStoreStateForTest(nil, []*WrappedTx{pendingReplacement}), nil) - _, ok = txStore.ByHash(pending.Hash()) +func TestTxStore_ReplacesPendingTxByHigherPriority(t *testing.T) { + rng := utils.TestRng() + env := newTxStoreReplacementTestEnv(t, rng) + env.app.nextNonce[env.address] = 7 + env.app.setBalance(env.address, big.NewInt(0)) + + pending := env.makeTx(rng, 7, 70, 40) + require.NoError(t, env.txStore.Insert(pending)) + env.assertState(t, nil, []*WrappedTx{pending}) + env.assertReadyList(t, nil) + + pendingReplacement := env.makeTx(rng, 7, 90, 50) + require.NoError(t, env.txStore.Insert(pendingReplacement)) + env.assertState(t, nil, []*WrappedTx{pendingReplacement}) + env.assertReadyList(t, nil) + _, ok := env.txStore.ByHash(pending.Hash()) require.False(t, ok) - got, ok = txStore.ByHash(pendingReplacement.Hash()) + got, ok := env.txStore.ByHash(pendingReplacement.Hash()) require.True(t, ok) require.Equal(t, pendingReplacement.Tx(), got) } From b18320de8ec44eac4ef0db39d9ee4674840577c4 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 09:42:21 +0200 Subject: [PATCH 039/100] syntax --- .../internal/mempool/mempool_test.go | 72 ++++++------------- 1 file changed, 22 insertions(+), 50 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index c119601fe2..2b05dfc5b6 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -157,7 +157,7 @@ func checkTxs(ctx context.Context, t *testing.T, txmp *TxMempool, numTxs int) [] rng := rand.New(rand.NewSource(time.Now().UnixNano())) - for i := 0; i < numTxs; i++ { + for i := range numTxs { prefix := make([]byte, 20) _, err := rng.Read(prefix) require.NoError(t, err) @@ -175,16 +175,6 @@ func checkTxs(ctx context.Context, t *testing.T, txmp *TxMempool, numTxs int) [] return txs } -func convertTex(in []testTx) types.Txs { - out := make([]types.Tx, len(in)) - - for idx := range in { - out[idx] = in[idx].tx - } - - return out -} - func totalTxSizeBytes(txs []testTx) uint64 { var total uint64 for _, tx := range txs { @@ -255,7 +245,7 @@ func TestTxMempool_TxsAvailable(t *testing.T) { } responses := make([]*abci.ExecTxResult, len(rawTxs[:50])) - for i := 0; i < len(responses); i++ { + for i := range responses { responses[i] = &abci.ExecTxResult{Code: abci.CodeTypeOK} } @@ -289,7 +279,7 @@ func TestTxMempool_Size(t *testing.T) { } responses := make([]*abci.ExecTxResult, len(rawTxs[:50])) - for i := 0; i < len(responses); i++ { + for i := range responses { responses[i] = &abci.ExecTxResult{Code: abci.CodeTypeOK} } @@ -317,7 +307,7 @@ func TestTxMempool_Flush(t *testing.T) { } responses := make([]*abci.ExecTxResult, len(rawTxs[:50])) - for i := 0; i < len(responses); i++ { + for i := range responses { responses[i] = &abci.ExecTxResult{Code: abci.CodeTypeOK} } @@ -370,32 +360,26 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { var wg sync.WaitGroup // reap by gas capacity only - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxGasWanted: utils.Some(int64(50))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) require.Len(t, reapedTxs, 50) - }() + }) // reap by transaction bytes only - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxBytes: utils.Some(int64(1000))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) require.Len(t, reapedTxs, expectedReapCountByBytes(sortedTxs, 1000)) - }() + }) // Reap by both transaction bytes and gas, where the size yields 31 reaped // transactions and the gas limit reaps 25 transactions. - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { reapedTxs, _ := txmp.ReapTxs(ReapLimits{ MaxBytes: utils.Some(int64(1500)), MaxGasWanted: utils.Some(int64(30)), @@ -404,27 +388,23 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) require.Len(t, reapedTxs, min(expectedReapCountByBytes(sortedTxs, 1500), 30)) - }() + }) // Reap by min transactions in block regardless of gas limit. - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxGasWanted: utils.Some(int64(2))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Len(t, reapedTxs, 2) - }() + }) // Reap by max gas estimated - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(50))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Len(t, reapedTxs, 50) - }() + }) wg.Wait() } @@ -461,14 +441,12 @@ func TestTxMempool_ReapMaxBytesMaxGas_FallbackToGasWanted(t *testing.T) { var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxGasEstimated: utils.Some(int64(50))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Len(t, reapedTxs, 50) - }() + }) wg.Wait() } @@ -507,37 +485,31 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { var wg sync.WaitGroup // reap all transactions - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { reapedTxs, _ := txmp.ReapTxs(ReapLimits{}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) require.Len(t, reapedTxs, len(tTxs)) - }() + }) // reap a single transaction - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(1))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) require.Len(t, reapedTxs, 1) - }() + }) // reap half of the transactions - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { reapedTxs, _ := txmp.ReapTxs(ReapLimits{MaxTxs: utils.Some(uint64(len(tTxs) / 2))}, false) ensurePrioritized(reapedTxs) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) require.Len(t, reapedTxs, len(tTxs)/2) - }() + }) wg.Wait() } From ce65730eaefb1bd93a8fa48e3d2d05861109bdc2 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 09:48:29 +0200 Subject: [PATCH 040/100] monotone blockHeight check --- sei-tendermint/internal/mempool/mempool.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 6463130490..be11177d83 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -438,6 +438,9 @@ func (txmp *TxMempool) Update( txConstraints TxConstraints, recheck bool, ) error { + if blockHeight <= txmp.height { + return fmt.Errorf("blockHeight = %v, want > %v",blockHeight,txmp.height) + } txmp.height = blockHeight txmp.notifiedTxsAvailable.Store(false) txmp.txConstraintsFetcher = func() (TxConstraints, error) { From 41450a986441dafd52ac7c57031856d3a850b5aa Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 09:56:11 +0200 Subject: [PATCH 041/100] style --- sei-tendermint/internal/mempool/mempool.go | 2 +- .../internal/mempool/mempool_bench_test.go | 6 +- .../internal/mempool/mempool_test.go | 87 +++++++++++-------- .../internal/mempool/recheck_drain_test.go | 12 ++- 4 files changed, 66 insertions(+), 41 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index be11177d83..c450064f47 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -439,7 +439,7 @@ func (txmp *TxMempool) Update( recheck bool, ) error { if blockHeight <= txmp.height { - return fmt.Errorf("blockHeight = %v, want > %v",blockHeight,txmp.height) + return fmt.Errorf("blockHeight = %v, want > %v", blockHeight, txmp.height) } txmp.height = blockHeight txmp.notifiedTxsAvailable.Store(false) diff --git a/sei-tendermint/internal/mempool/mempool_bench_test.go b/sei-tendermint/internal/mempool/mempool_bench_test.go index 614a770b69..b0f1f302c0 100644 --- a/sei-tendermint/internal/mempool/mempool_bench_test.go +++ b/sei-tendermint/internal/mempool/mempool_bench_test.go @@ -19,8 +19,10 @@ func BenchmarkTxMempool_CheckTx(b *testing.B) { // setup the cache and the mempool number for hitting GetEvictableTxs during the // benchmark. 5000 is the current default mempool size in the TM config. - txmp := setup(b, proxyClient, 10000, NopTxConstraintsFetcher) - txmp.config.Size = 5000 + cfg := TestConfig() + cfg.CacheSize = 10000 + cfg.Size = 5000 + txmp := setup(cfg, proxyClient, NopTxConstraintsFetcher) rng := rand.New(rand.NewSource(time.Now().UnixNano())) const peerID = 1 diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index 2b05dfc5b6..13fc97daad 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -142,11 +142,7 @@ func (app *application) GetTxPriorityHint(context.Context, *abci.RequestGetTxPri }, nil } -func setup(t testing.TB, app *proxy.Proxy, cacheSize int, txConstraintsFetcher TxConstraintsFetcher) *TxMempool { - t.Helper() - - cfg := TestConfig() - cfg.CacheSize = cacheSize +func setup(cfg *Config, app *proxy.Proxy, txConstraintsFetcher TxConstraintsFetcher) *TxMempool { return NewTxMempool(cfg, app, NopMetrics(), txConstraintsFetcher) } @@ -209,8 +205,9 @@ func TestTxMempool_TxsAvailable(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 0 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) ensureNoTxFire := func() { timer := time.NewTimer(500 * time.Millisecond) @@ -266,8 +263,9 @@ func TestTxMempool_Size(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 0 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) txs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(txs), txmp.Size()) require.Equal(t, 0, txmp.PendingSize()) @@ -295,8 +293,9 @@ func TestTxMempool_Flush(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 0 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) txs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(txs), txmp.Size()) require.Equal(t, totalTxSizeBytes(txs), txmp.SizeBytes()) @@ -325,8 +324,9 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { gasEstimated := int64(1) // gas estimated set to 1 client := &application{Application: kvstore.NewApplication(), gasEstimated: &gasEstimated} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 0 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) tTxs := checkTxs(ctx, t, txmp, 100) // all txs request 1 gas unit require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) @@ -414,8 +414,9 @@ func TestTxMempool_ReapMaxBytesMaxGas_FallbackToGasWanted(t *testing.T) { gasEstimated := int64(0) // gas estimated not set so fallback to gas wanted client := &application{Application: kvstore.NewApplication(), gasEstimated: &gasEstimated} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 0 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) tTxs := checkTxs(ctx, t, txmp, 100) txMap := make(map[types.TxHash]testTx) @@ -455,8 +456,9 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 0 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) tTxs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, totalTxSizeBytes(tTxs), txmp.SizeBytes()) @@ -521,8 +523,9 @@ func TestTxMempool_ReapMaxBytesMaxGas_MinGasEVMTxThreshold(t *testing.T) { gasEstimated := int64(10000) gasWanted := int64(50000) client := &application{Application: kvstore.NewApplication(), gasEstimated: &gasEstimated, gasWanted: &gasWanted} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 0 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) address := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" // Insert a single EVM tx (format: evm-sender=account=priority=nonce) @@ -543,7 +546,9 @@ func TestTxMempool_CheckTxExceedsMaxSize(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 0 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) rng := rand.New(rand.NewSource(time.Now().UnixNano())) tx := make([]byte, txmp.config.MaxTxBytes+1) @@ -565,8 +570,9 @@ func TestTxMempool_Prioritization(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 100, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 100 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) address1 := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" address2 := "0xfD23B3A9DE15e92B9ef9540E587B3661E15A12fA" @@ -657,8 +663,9 @@ func TestTxMempool_CheckTxDuplicateRejected(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 100, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 100 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) rng := rand.New(rand.NewSource(time.Now().UnixNano())) prefix := make([]byte, 20) @@ -677,8 +684,9 @@ func TestTxMempool_ConcurrentTxs(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 100, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 100 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) rng := rand.New(rand.NewSource(time.Now().UnixNano())) checkTxDone := make(chan struct{}) @@ -746,9 +754,11 @@ func TestTxMempool_ExpiredTxs_NumBlocks(t *testing.T) { client := &application{Application: kvstore.NewApplication()} - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 500, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 500 + cfg.TTLNumBlocks = utils.Some(int64(10)) + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) txmp.height = 100 - txmp.config.TTLNumBlocks = utils.Some(int64(10)) tTxs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(tTxs), txmp.Size()) @@ -796,9 +806,11 @@ func TestMempoolExpiration(t *testing.T) { client := &application{Application: kvstore.NewApplication()} - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) - txmp.config.TTLDuration = utils.Some(time.Nanosecond) - txmp.config.RemoveExpiredTxsFromQueue = true + cfg := TestConfig() + cfg.CacheSize = 0 + cfg.TTLDuration = utils.Some(time.Nanosecond) + cfg.RemoveExpiredTxsFromQueue = true + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) txs := checkTxs(ctx, t, txmp, 100) require.Equal(t, len(txs), txmp.Size()) @@ -817,8 +829,9 @@ func TestTxMempool_ReapTxs_EVMFirst(t *testing.T) { ctx := t.Context() client := &application{Application: kvstore.NewApplication()} - - txmp := setup(t, proxy.New(client, proxy.NopMetrics()), 0, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 0 + txmp := setup(cfg, proxy.New(client, proxy.NopMetrics()), NopTxConstraintsFetcher) evmAddress1 := "0xeD23B3A9DE15e92B9ef9540E587B3661E15A12fA" evmAddress2 := "0xfD23B3A9DE15e92B9ef9540E587B3661E15A12fA" @@ -881,7 +894,9 @@ func TestBlockFailedTxNotReAdmittedAfterSecondFailure(t *testing.T) { defer cancel() app := &application{Application: kvstore.NewApplication()} - txmp := setup(t, proxy.New(app, proxy.NopMetrics()), 500, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 500 + txmp := setup(cfg, proxy.New(app, proxy.NopMetrics()), NopTxConstraintsFetcher) tx := types.Tx("sender-0-0=key=1000") @@ -931,7 +946,9 @@ func TestBlockFailedTxTrackerClearedOnSuccess(t *testing.T) { defer cancel() app := &application{Application: kvstore.NewApplication()} - txmp := setup(t, proxy.New(app, proxy.NopMetrics()), 500, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 500 + txmp := setup(cfg, proxy.New(app, proxy.NopMetrics()), NopTxConstraintsFetcher) tx := types.Tx("sender-0-0=key=1000") txHash := tx.Hash() diff --git a/sei-tendermint/internal/mempool/recheck_drain_test.go b/sei-tendermint/internal/mempool/recheck_drain_test.go index b54f4ad2a4..e34fb71d7d 100644 --- a/sei-tendermint/internal/mempool/recheck_drain_test.go +++ b/sei-tendermint/internal/mempool/recheck_drain_test.go @@ -146,7 +146,9 @@ func TestTxMempool_DescendingNonceDrain(t *testing.T) { ctx := t.Context() app := newEVMNonceApp() - txmp := setup(t, proxy.New(app, proxy.NopMetrics()), 5000, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 5000 + txmp := setup(cfg, proxy.New(app, proxy.NopMetrics()), NopTxConstraintsFetcher) // Submit nonces N-1, N-2, ..., 1, 0. Every tx except the last enters // pendingTxs because its nonce is ahead of the sender's expected nonce @@ -197,7 +199,9 @@ func TestTxMempool_EvmNextPendingNonceIncludesPendingTransactions(t *testing.T) app := newEVMNonceApp() app.nextNonce[sender] = 5 - txmp := setup(t, proxy.New(app, proxy.NopMetrics()), 5000, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 5000 + txmp := setup(cfg, proxy.New(app, proxy.NopMetrics()), NopTxConstraintsFetcher) for _, nonce := range []uint64{7, 5, 6} { tx := []byte(fmt.Sprintf("evm=%s=%d=1", sender.Hex(), nonce)) @@ -216,7 +220,9 @@ func TestTxMempool_EvmNextPendingNonceReplacesSameNonceByPriority(t *testing.T) app := newEVMNonceApp() app.nextNonce[sender] = 5 - txmp := setup(t, proxy.New(app, proxy.NopMetrics()), 5000, NopTxConstraintsFetcher) + cfg := TestConfig() + cfg.CacheSize = 5000 + txmp := setup(cfg, proxy.New(app, proxy.NopMetrics()), NopTxConstraintsFetcher) lowPriorityTx := []byte(fmt.Sprintf("evm=%s=%d=%d", sender.Hex(), 6, 1)) highPriorityTx := []byte(fmt.Sprintf("evm=%s=%d=%d", sender.Hex(), 6, 2)) From 616ba41351c142f56c168a01c72b6d5368e0aa37 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 11:18:05 +0200 Subject: [PATCH 042/100] wip --- .../internal/mempool/recheck_drain_test.go | 26 +- sei-tendermint/internal/mempool/tx_test.go | 371 ++++++++++++------ 2 files changed, 273 insertions(+), 124 deletions(-) diff --git a/sei-tendermint/internal/mempool/recheck_drain_test.go b/sei-tendermint/internal/mempool/recheck_drain_test.go index e34fb71d7d..dd914be05f 100644 --- a/sei-tendermint/internal/mempool/recheck_drain_test.go +++ b/sei-tendermint/internal/mempool/recheck_drain_test.go @@ -29,13 +29,13 @@ type evmNonceApp struct { mu sync.Mutex nextNonce map[common.Address]uint64 - balance map[common.Address]*big.Int + balance map[common.Address]int } func newEVMNonceApp() *evmNonceApp { return &evmNonceApp{ nextNonce: map[common.Address]uint64{}, - balance: map[common.Address]*big.Int{}, + balance: map[common.Address]int{}, } } @@ -47,19 +47,25 @@ func (a *evmNonceApp) markMined(sender common.Address) { a.mu.Unlock() } -func (a *evmNonceApp) setBalance(sender common.Address, balance *big.Int) { +func (a *evmNonceApp) setNonce(sender common.Address, nonce uint64) { a.mu.Lock() - a.balance[sender] = new(big.Int).Set(balance) + a.nextNonce[sender] = nonce a.mu.Unlock() } -func (a *evmNonceApp) balanceOf(sender common.Address) *big.Int { +func (a *evmNonceApp) setBalance(sender common.Address, balance int) { + a.mu.Lock() + a.balance[sender] = balance + a.mu.Unlock() +} + +func (a *evmNonceApp) balanceOf(sender common.Address) int { a.mu.Lock() defer a.mu.Unlock() if balance, ok := a.balance[sender]; ok { - return new(big.Int).Set(balance) + return balance } - return big.NewInt(0) + return 0 } func (a *evmNonceApp) parseTx(tx []byte) (sender string, nonce uint64, priority int64, ok bool) { @@ -124,7 +130,7 @@ func (a *evmNonceApp) EvmBalance(addr common.Address, _ []byte) *big.Int { a.mu.Lock() defer a.mu.Unlock() if balance, ok := a.balance[addr]; ok { - return new(big.Int).Set(balance) + return big.NewInt(int64(balance)) } return big.NewInt(0) } @@ -198,7 +204,7 @@ func TestTxMempool_EvmNextPendingNonceIncludesPendingTransactions(t *testing.T) sender := common.HexToAddress("0x00000000000000000000000000000000000000aa") app := newEVMNonceApp() - app.nextNonce[sender] = 5 + app.setNonce(sender, 5) cfg := TestConfig() cfg.CacheSize = 5000 txmp := setup(cfg, proxy.New(app, proxy.NopMetrics()), NopTxConstraintsFetcher) @@ -219,7 +225,7 @@ func TestTxMempool_EvmNextPendingNonceReplacesSameNonceByPriority(t *testing.T) sender := common.HexToAddress("0x00000000000000000000000000000000000000bb") app := newEVMNonceApp() - app.nextNonce[sender] = 5 + app.setNonce(sender, 5) cfg := TestConfig() cfg.CacheSize = 5000 txmp := setup(cfg, proxy.New(app, proxy.NopMetrics()), NopTxConstraintsFetcher) diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index fcf2e51590..0ee3c2f9f8 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -30,6 +30,162 @@ func txStoreStateForTest(ready, pending []*WrappedTx) txStoreState { return state } +type testAccount struct { + address common.Address + baseNonce uint64 + lastNonce uint64 +} + +type testEnv struct { + rng utils.Rng + txStore *txStore + app *evmNonceApp + accounts []testAccount + byHash map[types.TxHash]*WrappedTx + everReady map[types.TxHash]struct{} +} + +func newTestEnv( + rng utils.Rng, + txStore *txStore, + app *evmNonceApp, + numAccounts int, +) *testEnv { + env := &testEnv{ + rng: rng, + txStore: txStore, + app: app, + accounts: make([]testAccount, numAccounts), + byHash: map[types.TxHash]*WrappedTx{}, + everReady: map[types.TxHash]struct{}{}, + } + for i := range env.accounts { + env.accounts[i].address = common.BytesToAddress(utils.GenBytes(rng, common.AddressLength)) + env.accounts[i].baseNonce = uint64(rng.Intn(20) + 1) + rangeLen := rng.Intn(16) + 12 + env.accounts[i].lastNonce = env.accounts[i].baseNonce + uint64(rangeLen-1) + env.app.setNonce(env.accounts[i].address, env.accounts[i].baseNonce) + } + return env +} + +func (e *testEnv) insertTxs( + t *testing.T, + insertProbPercent int, + makeTx func(account *testAccount, nonce uint64) *WrappedTx, +) { + t.Helper() + + clear(e.byHash) + for i := range e.accounts { + account := &e.accounts[i] + rangeLen := int(account.lastNonce-account.baseNonce) + 1 + for offset := range rangeLen { + if e.rng.Intn(100) >= insertProbPercent { + continue + } + wtx := makeTx(account, account.baseNonce+uint64(offset)) + e.byHash[wtx.Hash()] = wtx + require.NoError(t, e.txStore.Insert(wtx)) + } + } +} + +func (e *testEnv) txs() []*WrappedTx { + txs := make([]*WrappedTx, 0, len(e.byHash)) + for _, wtx := range e.byHash { + txs = append(txs, wtx) + } + return txs +} + +func (e *testEnv) byNonce(account testAccount) map[uint64]*WrappedTx { + byNonce := map[uint64]*WrappedTx{} + for _, wtx := range e.byHash { + evm := wtx.evm.OrPanic("evm tx") + if evm.address == account.address { + byNonce[evm.nonce] = wtx + } + } + return byNonce +} + +func (e *testEnv) readyTxs() []*WrappedTx { + var ready []*WrappedTx + for _, account := range e.accounts { + byNonce := e.byNonce(account) + currentNonce := e.app.EvmNonce(account.address) + balance := e.app.balanceOf(account.address) + for nonce := currentNonce; ; nonce++ { + wtx, ok := byNonce[nonce] + if !ok { + break + } + if int(wtx.evm.OrPanic("").requiredBalance.Int64()) > balance { + break + } + ready = append(ready, wtx) + } + } + return ready +} + +func (e *testEnv) markReadyTxs() { + for _, wtx := range e.readyTxs() { + e.everReady[wtx.Hash()] = struct{}{} + } +} + +func (e *testEnv) stableReady() []*WrappedTx { + var stable []*WrappedTx + for _, wtx := range e.byHash { + if _, ok := e.everReady[wtx.Hash()]; ok { + stable = append(stable, wtx) + } + } + return stable +} + +func toTxs(wtxs []*WrappedTx) types.Txs { + var txs types.Txs + for _, wtx := range wtxs { + txs = append(txs, wtx.Tx()) + } + return txs +} + +func (e *testEnv) assertState(t *testing.T) { + t.Helper() + + expectedReady := e.readyTxs() + readySet := make(map[types.TxHash]struct{}, len(expectedReady)) + for _, wtx := range expectedReady { + readySet[wtx.Hash()] = struct{}{} + } + var expectedPending []*WrappedTx + for _, wtx := range e.txs() { + if _, ok := readySet[wtx.Hash()]; ok { + continue + } + expectedPending = append(expectedPending, wtx) + } + expectedStableReady := e.stableReady() + + require.Equal(t, txStoreStateForTest(expectedReady, expectedPending), e.txStore.State()) + + readyTxs := e.txStore.ReadyTxs() + require.ElementsMatch(t, toTxs(expectedReady), toTxs(readyTxs)) + + reaped, _ := e.txStore.Reap(ReapLimits{MaxTxs: utils.Some(uint64(len(e.byHash)))}, false) + require.ElementsMatch(t, toTxs(expectedReady), reaped) + + var listedTxs types.Txs + for el := e.txStore.readyTxs.Front(); el != nil; el = el.Next() { + listedTxs = append(listedTxs, el.Value()) + } + require.ElementsMatch(t, toTxs(expectedStableReady), listedTxs) +} + func TestTxStore_GetTxByHash(t *testing.T) { txs := newTxStoreForTest() wtx := &WrappedTx{ @@ -86,14 +242,6 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { app := newEVMNonceApp() txStore := NewTxStore(TestConfig(), proxy.New(app, proxy.NopMetrics()), NopMetrics()) - type accountCase struct { - address common.Address - baseNonce uint64 - lastNonce uint64 - byNonce map[uint64]*WrappedTx - txs []*WrappedTx - } - makeTx := func(address common.Address, nonce uint64) *WrappedTx { requiredBalance := big.NewInt(rng.Int63n(256)) return &WrappedTx{ @@ -114,59 +262,30 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { // Seed the store with sparse per-account nonce ranges so each account has a // mix of contiguous ready transactions and gaps that keep later transactions // pending. - accounts := make([]accountCase, 8) - everReady := map[types.TxHash]struct{}{} - expectedInserted := 0 - for i := range accounts { - accounts[i].address = common.BytesToAddress(utils.GenBytes(rng, common.AddressLength)) - accounts[i].baseNonce = uint64(rng.Intn(20) + 1) - accounts[i].byNonce = map[uint64]*WrappedTx{} - rangeLen := rng.Intn(16) + 12 - accounts[i].lastNonce = accounts[i].baseNonce + uint64(rangeLen-1) - app.nextNonce[accounts[i].address] = accounts[i].baseNonce - app.setBalance(accounts[i].address, big.NewInt(rng.Int63n(256))) - insertedForAccount := 0 - for offset := range rangeLen { - if rng.Intn(100) >= 80 { - continue - } - wtx := makeTx(accounts[i].address, accounts[i].baseNonce+uint64(offset)) - accounts[i].txs = append(accounts[i].txs, wtx) - accounts[i].byNonce[wtx.EVMNonce()] = wtx - require.NoError(t, txStore.Insert(wtx)) - expectedInserted++ - insertedForAccount++ - } - require.Positive(t, insertedForAccount) - - rejected := makeTx(accounts[i].address, accounts[i].baseNonce-1) + env := newTestEnv(rng, txStore, app, 8) + for _, account := range env.accounts { + app.setBalance(account.address, rng.Intn(256)) + } + env.insertTxs(t, 80, func(account *testAccount, nonce uint64) *WrappedTx { + return makeTx(account.address, nonce) + }) + for _, account := range env.accounts { + rejected := makeTx(account.address, account.baseNonce-1) require.ErrorIs(t, txStore.Insert(rejected), errOldNonce) } - require.Equal(t, expectedInserted, txStore.State().total.count) + require.Equal(t, len(env.byHash), txStore.State().total.count) // Seed the stable-ready history with transactions that are already ready // after the initial inserts. - for _, account := range accounts { - balance := app.balanceOf(account.address) - for nonce := account.baseNonce; ; nonce++ { - wtx, ok := account.byNonce[nonce] - if !ok { - break - } - if wtx.evm.OrPanic("evm tx").requiredBalance.Cmp(balance) > 0 { - break - } - everReady[wtx.Hash()] = struct{}{} - } - } + env.markReadyTxs() // Advance the per-account nonce frontier in several randomized rounds and // verify that Update removes every transaction that fell below the account // nonce while preserving the rest. for height := range int64(5) { - for _, account := range accounts { - currentNonce := app.nextNonce[account.address] + for _, account := range env.accounts { + currentNonce := app.EvmNonce(account.address) if currentNonce > 0 { rejected := makeTx(account.address, currentNonce-1) require.ErrorIs(t, txStore.Insert(rejected), errOldNonce) @@ -175,7 +294,7 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { for range rng.Intn(maxAdvance + 1) { app.markMined(account.address) } - app.setBalance(account.address, big.NewInt(rng.Int63n(256))) + app.setBalance(account.address, rng.Intn(256)) } txStore.Update(updateSpec{ @@ -186,66 +305,90 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { NewPriorities: map[types.TxHash]int64{}, }) - // Derive the expected remaining/ready sets from the test model: - // all txs at or above the current account nonce remain present, and the - // ready prefix is the contiguous run starting at the current nonce. - expectedRemaining := 0 - expectedReady := 0 - expectedStableReady := 0 - for _, account := range accounts { - currentNonce := app.nextNonce[account.address] - balance := app.balanceOf(account.address) - for nonce, wtx := range account.byNonce { - got, ok := txStore.ByHash(wtx.Hash()) - if nonce < currentNonce { - require.False(t, ok) - continue - } - require.True(t, ok) - require.Equal(t, wtx.Tx(), got) - expectedRemaining++ - if _, wasReady := everReady[wtx.Hash()]; wasReady { - expectedStableReady++ - } - } - for nonce := currentNonce; ; nonce++ { - wtx, ok := account.byNonce[nonce] - if !ok { - break - } - if wtx.evm.OrPanic("evm tx").requiredBalance.Cmp(balance) > 0 { - break - } - expectedReady++ - if _, wasReady := everReady[wtx.Hash()]; !wasReady { - everReady[wtx.Hash()] = struct{}{} - expectedStableReady++ - } + for txHash, wtx := range env.byHash { + if wtx.EVMNonce() < app.EvmNonce(wtx.evm.OrPanic("").address) { + delete(env.byHash, txHash) } } - state := txStore.State() - require.Equal(t, expectedRemaining, state.total.count) - require.Equal(t, expectedReady, state.ready.count) - - // Reap returns the currently ready transactions, while readyTxs is a - // stable list of transactions that have become ready at least once and - // have not been removed from the store. - reaped, _ := txStore.Reap(ReapLimits{ - MaxTxs: utils.Some(uint64(expectedRemaining)), - }, false) - listed := make(types.Txs, 0, expectedStableReady) - listedSet := make(map[types.TxHash]struct{}, expectedStableReady) - for el := txStore.readyTxs.Front(); el != nil; el = el.Next() { - tx := el.Value() - listed = append(listed, tx) - listedSet[tx.Hash()] = struct{}{} + env.markReadyTxs() + env.assertState(t) + } +} + +func TestTxStore_UpdateExpiresTransactions(t *testing.T) { + rng := utils.TestRng() + cfg := TestConfig() + cfg.CacheSize = 1_000 + cfg.TTLNumBlocks = utils.Some(int64(10)) + cfg.TTLDuration = utils.Some(10 * time.Second) + cfg.RemoveExpiredTxsFromQueue = true + + app := newEVMNonceApp() + txStore := NewTxStore(cfg, proxy.New(app, proxy.NopMetrics()), NopMetrics()) + baseTime := time.Unix(1_700_000_000, 0) + + makeTx := func(address common.Address, nonce uint64, height int64, timestamp time.Time) *WrappedTx { + return &WrappedTx{ + hashedTx: newHashedTx(utils.GenBytes(rng, 32)), + height: height, + timestamp: timestamp, + priority: rng.Int63n(1_000_000) + 1, + gasWanted: 1, + estimatedGas: 1, + evm: utils.Some(evmTx{ + address: address, + seiAddress: address.Bytes(), + nonce: nonce, + requiredBalance: big.NewInt(0), + }), + } + } + + // Seed the store with randomized timestamps, heights, and sparse nonce + // ranges across a bounded set of accounts. + env := newTestEnv(rng, txStore, app, 5) + for _, account := range env.accounts { + app.setBalance(account.address, 1_000_000) + } + env.insertTxs(t, 100, func(account *testAccount, nonce uint64) *WrappedTx { + return makeTx( + account.address, + nonce, + int64(rng.Intn(28)+1), + baseTime.Add(time.Duration(rng.Intn(31))*time.Second), + ) + }) + + // Record the transactions that are initially ready; the stable ready list + // keeps these entries until the transactions are removed. + env.markReadyTxs() + + updates := []updateSpec{ + {Now: baseTime.Add(16 * time.Second), Height: 14, TxResults: map[types.TxHash]bool{}, Constraints: NopTxConstraints(), NewPriorities: map[types.TxHash]int64{}}, + {Now: baseTime.Add(24 * time.Second), Height: 22, TxResults: map[types.TxHash]bool{}, Constraints: NopTxConstraints(), NewPriorities: map[types.TxHash]int64{}}, + {Now: baseTime.Add(36 * time.Second), Height: 34, TxResults: map[types.TxHash]bool{}, Constraints: NopTxConstraints(), NewPriorities: map[types.TxHash]int64{}}, + } + + for _, update := range updates { + txStore.Update(update) + minHeight := int64(-1) + if ttl, ok := cfg.TTLNumBlocks.Get(); ok && update.Height > ttl { + minHeight = update.Height - ttl } - require.Len(t, reaped, expectedReady) - require.Len(t, listed, expectedStableReady) - for _, tx := range reaped { - _, ok := listedSet[tx.Hash()] - require.True(t, ok) + minTime := time.Time{} + if ttl, ok := cfg.TTLDuration.Get(); ok { + minTime = update.Now.Add(-ttl) + } + + for txHash, wtx := range env.byHash { + expiredByHeight := minHeight >= 0 && wtx.height < minHeight + expiredByTime := !minTime.IsZero() && wtx.timestamp.Before(minTime) + if expiredByHeight || expiredByTime { + delete(env.byHash, txHash) + } } + env.markReadyTxs() + env.assertState(t) } } @@ -308,8 +451,8 @@ func (e txStoreReplacementTestEnv) assertReadyList(t *testing.T, expected types. func TestTxStore_ReplacesReadyTxByHigherPriority(t *testing.T) { rng := utils.TestRng() env := newTxStoreReplacementTestEnv(t, rng) - env.app.nextNonce[env.address] = 7 - env.app.setBalance(env.address, big.NewInt(100)) + env.app.setNonce(env.address, 7) + env.app.setBalance(env.address, 100) // Insert one ready transaction, then replace it with a higher-priority ready // transaction for the same nonce. @@ -345,15 +488,15 @@ func TestTxStore_ReplacesReadyTxByHigherPriority(t *testing.T) { func TestTxStore_ReplacesReadyThenPendingTxByHigherPriority(t *testing.T) { rng := utils.TestRng() env := newTxStoreReplacementTestEnv(t, rng) - env.app.nextNonce[env.address] = 7 - env.app.setBalance(env.address, big.NewInt(100)) + env.app.setNonce(env.address, 7) + env.app.setBalance(env.address, 100) becamePending := env.makeTx(rng, 7, 40, 60) require.NoError(t, env.txStore.Insert(becamePending)) env.assertState(t, []*WrappedTx{becamePending}, nil) env.assertReadyList(t, types.Txs{becamePending.Tx()}) - env.app.setBalance(env.address, big.NewInt(50)) + env.app.setBalance(env.address, 50) env.txStore.Update(updateSpec{ Now: time.Now(), Height: 1, @@ -378,8 +521,8 @@ func TestTxStore_ReplacesReadyThenPendingTxByHigherPriority(t *testing.T) { func TestTxStore_ReplacesPendingTxByHigherPriority(t *testing.T) { rng := utils.TestRng() env := newTxStoreReplacementTestEnv(t, rng) - env.app.nextNonce[env.address] = 7 - env.app.setBalance(env.address, big.NewInt(0)) + env.app.setNonce(env.address, 7) + env.app.setBalance(env.address, 0) pending := env.makeTx(rng, 7, 70, 40) require.NoError(t, env.txStore.Insert(pending)) From be86a25353e39eab7ee4fcd32262a09f72616507 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 11:22:35 +0200 Subject: [PATCH 043/100] merged helpers --- sei-tendermint/internal/mempool/tx_test.go | 159 +++++++++------------ 1 file changed, 69 insertions(+), 90 deletions(-) diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index 0ee3c2f9f8..c2b2a02478 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -154,6 +154,28 @@ func toTxs(wtxs []*WrappedTx) types.Txs { return txs } +func makeEvmTxForTest( + rng utils.Rng, + address common.Address, + nonce uint64, + priority int64, + requiredBalance int, +) *WrappedTx { + return &WrappedTx{ + hashedTx: newHashedTx(utils.GenBytes(rng, rng.Intn(48)+16)), + timestamp: time.Now(), + priority: priority, + gasWanted: 1, + estimatedGas: 1, + evm: utils.Some(evmTx{ + address: address, + seiAddress: address.Bytes(), + nonce: nonce, + requiredBalance: big.NewInt(int64(requiredBalance)), + }), + } +} + func (e *testEnv) assertState(t *testing.T) { t.Helper() @@ -388,83 +410,33 @@ func TestTxStore_UpdateExpiresTransactions(t *testing.T) { } } env.markReadyTxs() - env.assertState(t) - } -} - -type txStoreReplacementTestEnv struct { - address common.Address - app *evmNonceApp - txStore *txStore -} - -func newTxStoreReplacementTestEnv(t *testing.T, rng utils.Rng) txStoreReplacementTestEnv { - t.Helper() - app := newEVMNonceApp() - return txStoreReplacementTestEnv{ - address: common.BytesToAddress(utils.GenBytes(rng, common.AddressLength)), - app: app, - txStore: NewTxStore(TestConfig(), proxy.New(app, proxy.NopMetrics()), NopMetrics()), - } -} - -func (e txStoreReplacementTestEnv) makeTx(rng utils.Rng, nonce uint64, priority int64, requiredBalance int) *WrappedTx { - return &WrappedTx{ - hashedTx: newHashedTx(utils.GenBytes(rng, rng.Intn(48)+16)), - timestamp: time.Now(), - priority: priority, - gasWanted: 1, - estimatedGas: 1, - evm: utils.Some(evmTx{ - address: e.address, - seiAddress: e.address.Bytes(), - nonce: nonce, - requiredBalance: big.NewInt(int64(requiredBalance)), - }), + env.assertState(t) } } -func (e txStoreReplacementTestEnv) assertState(t *testing.T, ready, pending []*WrappedTx) { - t.Helper() - expected := txStoreStateForTest(ready, pending) - require.Equal(t, expected, e.txStore.State()) - reaped, _ := e.txStore.Reap(ReapLimits{MaxTxs: utils.Some(uint64(expected.total.count))}, false) - var expectedReady types.Txs - if len(ready) > 0 { - expectedReady = make(types.Txs, 0, len(ready)) - for _, wtx := range ready { - expectedReady = append(expectedReady, wtx.Tx()) - } - } - require.Equal(t, expectedReady, reaped) -} - -func (e txStoreReplacementTestEnv) assertReadyList(t *testing.T, expected types.Txs) { - t.Helper() - var listed types.Txs - for el := e.txStore.readyTxs.Front(); el != nil; el = el.Next() { - listed = append(listed, el.Value()) - } - require.Equal(t, expected, listed) -} - func TestTxStore_ReplacesReadyTxByHigherPriority(t *testing.T) { rng := utils.TestRng() - env := newTxStoreReplacementTestEnv(t, rng) - env.app.setNonce(env.address, 7) - env.app.setBalance(env.address, 100) + app := newEVMNonceApp() + txStore := NewTxStore(TestConfig(), proxy.New(app, proxy.NopMetrics()), NopMetrics()) + env := newTestEnv(rng, txStore, app, 1) + address := env.accounts[0].address + env.app.setNonce(address, 7) + env.app.setBalance(address, 100) // Insert one ready transaction, then replace it with a higher-priority ready // transaction for the same nonce. - old := env.makeTx(rng, 7, 10, 20) + old := makeEvmTxForTest(rng, address, 7, 10, 20) require.NoError(t, env.txStore.Insert(old)) - env.assertState(t, []*WrappedTx{old}, nil) - env.assertReadyList(t, types.Txs{old.Tx()}) + env.byHash = map[types.TxHash]*WrappedTx{old.Hash(): old} + env.markReadyTxs() + env.assertState(t) - replacement := env.makeTx(rng, 7, 20, 30) + replacement := makeEvmTxForTest(rng, address, 7, 20, 30) require.NoError(t, env.txStore.Insert(replacement)) - env.assertState(t, []*WrappedTx{replacement}, nil) - env.assertReadyList(t, types.Txs{replacement.Tx()}) + delete(env.byHash, old.Hash()) + env.byHash[replacement.Hash()] = replacement + env.markReadyTxs() + env.assertState(t) _, ok := env.txStore.ByHash(old.Hash()) require.False(t, ok) got, ok := env.txStore.ByHash(replacement.Hash()) @@ -473,11 +445,10 @@ func TestTxStore_ReplacesReadyTxByHigherPriority(t *testing.T) { // A higher-priority transaction that would no longer be ready must not // replace the current ready transaction for the same nonce. - blocked := env.makeTx(rng, 7, 30, 101) + blocked := makeEvmTxForTest(rng, address, 7, 30, 101) require.ErrorIs(t, env.txStore.Insert(blocked), errSameNonce) - env.assertState(t, []*WrappedTx{replacement}, nil) - env.assertReadyList(t, types.Txs{replacement.Tx()}) + env.assertState(t) got, ok = env.txStore.ByHash(replacement.Hash()) require.True(t, ok) require.Equal(t, replacement.Tx(), got) @@ -487,16 +458,20 @@ func TestTxStore_ReplacesReadyTxByHigherPriority(t *testing.T) { func TestTxStore_ReplacesReadyThenPendingTxByHigherPriority(t *testing.T) { rng := utils.TestRng() - env := newTxStoreReplacementTestEnv(t, rng) - env.app.setNonce(env.address, 7) - env.app.setBalance(env.address, 100) + app := newEVMNonceApp() + txStore := NewTxStore(TestConfig(), proxy.New(app, proxy.NopMetrics()), NopMetrics()) + env := newTestEnv(rng, txStore, app, 1) + address := env.accounts[0].address + env.app.setNonce(address, 7) + env.app.setBalance(address, 100) - becamePending := env.makeTx(rng, 7, 40, 60) + becamePending := makeEvmTxForTest(rng, address, 7, 40, 60) require.NoError(t, env.txStore.Insert(becamePending)) - env.assertState(t, []*WrappedTx{becamePending}, nil) - env.assertReadyList(t, types.Txs{becamePending.Tx()}) + env.byHash = map[types.TxHash]*WrappedTx{becamePending.Hash(): becamePending} + env.markReadyTxs() + env.assertState(t) - env.app.setBalance(env.address, 50) + env.app.setBalance(address, 50) env.txStore.Update(updateSpec{ Now: time.Now(), Height: 1, @@ -504,13 +479,13 @@ func TestTxStore_ReplacesReadyThenPendingTxByHigherPriority(t *testing.T) { Constraints: NopTxConstraints(), NewPriorities: map[types.TxHash]int64{}, }) - env.assertState(t, nil, []*WrappedTx{becamePending}) - env.assertReadyList(t, types.Txs{becamePending.Tx()}) + env.assertState(t) - becamePendingReplacement := env.makeTx(rng, 7, 50, 70) + becamePendingReplacement := makeEvmTxForTest(rng, address, 7, 50, 70) require.NoError(t, env.txStore.Insert(becamePendingReplacement)) - env.assertState(t, nil, []*WrappedTx{becamePendingReplacement}) - env.assertReadyList(t, nil) + delete(env.byHash, becamePending.Hash()) + env.byHash[becamePendingReplacement.Hash()] = becamePendingReplacement + env.assertState(t) _, ok := env.txStore.ByHash(becamePending.Hash()) require.False(t, ok) got, ok := env.txStore.ByHash(becamePendingReplacement.Hash()) @@ -520,19 +495,23 @@ func TestTxStore_ReplacesReadyThenPendingTxByHigherPriority(t *testing.T) { func TestTxStore_ReplacesPendingTxByHigherPriority(t *testing.T) { rng := utils.TestRng() - env := newTxStoreReplacementTestEnv(t, rng) - env.app.setNonce(env.address, 7) - env.app.setBalance(env.address, 0) + app := newEVMNonceApp() + txStore := NewTxStore(TestConfig(), proxy.New(app, proxy.NopMetrics()), NopMetrics()) + env := newTestEnv(rng, txStore, app, 1) + address := env.accounts[0].address + env.app.setNonce(address, 7) + env.app.setBalance(address, 0) - pending := env.makeTx(rng, 7, 70, 40) + pending := makeEvmTxForTest(rng, address, 7, 70, 40) require.NoError(t, env.txStore.Insert(pending)) - env.assertState(t, nil, []*WrappedTx{pending}) - env.assertReadyList(t, nil) + env.byHash = map[types.TxHash]*WrappedTx{pending.Hash(): pending} + env.assertState(t) - pendingReplacement := env.makeTx(rng, 7, 90, 50) + pendingReplacement := makeEvmTxForTest(rng, address, 7, 90, 50) require.NoError(t, env.txStore.Insert(pendingReplacement)) - env.assertState(t, nil, []*WrappedTx{pendingReplacement}) - env.assertReadyList(t, nil) + delete(env.byHash, pending.Hash()) + env.byHash[pendingReplacement.Hash()] = pendingReplacement + env.assertState(t) _, ok := env.txStore.ByHash(pending.Hash()) require.False(t, ok) got, ok := env.txStore.ByHash(pendingReplacement.Hash()) From 36b309df92be0b56f6b3a79486667644e6ed58bd Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 11:32:19 +0200 Subject: [PATCH 044/100] backward compatibility fix --- sei-tendermint/internal/mempool/tx.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index dde254b6ed..738dd9e36e 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -485,6 +485,11 @@ func (s *txStore) Update(spec updateSpec) { if remove { if expired { s.metrics.ExpiredTxs.Add(1) + // For some reason we treat expired txs as invalid here. + if !s.config.KeepInvalidTxsInCache { + s.cache.Remove(txHash) + s.metrics.CacheSize.Set(float64(s.cache.Size())) + } } delete(inner.byHash, txHash) s.metrics.RemovedTxs.Add(1) From 45e80c39981bb82cef8ae2534d5356103c4fa84e Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 11:50:37 +0200 Subject: [PATCH 045/100] compatibility fix --- sei-tendermint/internal/mempool/tx.go | 17 +- sei-tendermint/internal/mempool/tx_test.go | 197 ++++++++++++++++++++- 2 files changed, 205 insertions(+), 9 deletions(-) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 738dd9e36e..0acc4b5604 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -421,9 +421,11 @@ func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) { for _, wtx := range wtxs { total := inner.state.Load().total total.Inc(wtx.Size()) - if !total.LessEqual(&inner.softLimit) || s.insert(inner, wtx) != nil { - s.cache.Remove(wtx.Hash()) - s.metrics.CacheSize.Set(float64(s.cache.Size())) + limitOk := total.LessEqual(&inner.softLimit) + if !limitOk || s.insert(inner, wtx) != nil { + if !limitOk || !s.config.KeepInvalidTxsInCache { + s.cache.Remove(wtx.Hash()) + } s.metrics.RemovedTxs.Add(1) s.metrics.EvictedTxs.Add(1) if el, ok := wtx.readyEl.Get(); ok { @@ -431,6 +433,7 @@ func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) { } } } + s.metrics.CacheSize.Set(float64(s.cache.Size())) } type updateSpec struct { @@ -466,9 +469,12 @@ func (s *txStore) Update(spec updateSpec) { } for txHash, wtx := range inner.byHash { expired := isExpired(wtx) - remove := expired || wtx.check(spec.Constraints) != nil + invalid := wtx.check(spec.Constraints) != nil + remove := expired || invalid + executed := false if success, ok := spec.TxResults[wtx.Hash()]; ok { // Executed transactions should be removed. + executed = true remove = true if !s.config.KeepInvalidTxsInCache { if !success { @@ -490,6 +496,9 @@ func (s *txStore) Update(spec updateSpec) { s.cache.Remove(txHash) s.metrics.CacheSize.Set(float64(s.cache.Size())) } + } else if invalid && !executed && !s.config.KeepInvalidTxsInCache { + s.cache.Remove(txHash) + s.metrics.CacheSize.Set(float64(s.cache.Size())) } delete(inner.byHash, txHash) s.metrics.RemovedTxs.Add(1) diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index c2b2a02478..9a0b2fd6e0 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -337,13 +337,13 @@ func TestTxStore_RejectsAndEvictsTransactionsBelowAccountNonce(t *testing.T) { } } -func TestTxStore_UpdateExpiresTransactions(t *testing.T) { +func testTxStoreUpdateExpiresTransactions(t *testing.T, removeExpiredTxsFromQueue bool) { rng := utils.TestRng() cfg := TestConfig() cfg.CacheSize = 1_000 cfg.TTLNumBlocks = utils.Some(int64(10)) cfg.TTLDuration = utils.Some(10 * time.Second) - cfg.RemoveExpiredTxsFromQueue = true + cfg.RemoveExpiredTxsFromQueue = removeExpiredTxsFromQueue app := newEVMNonceApp() txStore := NewTxStore(cfg, proxy.New(app, proxy.NopMetrics()), NopMetrics()) @@ -392,6 +392,12 @@ func TestTxStore_UpdateExpiresTransactions(t *testing.T) { } for _, update := range updates { + readyBeforeUpdate := env.readyTxs() + readyBeforeUpdateSet := make(map[types.TxHash]struct{}, len(readyBeforeUpdate)) + for _, wtx := range readyBeforeUpdate { + readyBeforeUpdateSet[wtx.Hash()] = struct{}{} + } + txStore.Update(update) minHeight := int64(-1) if ttl, ok := cfg.TTLNumBlocks.Get(); ok && update.Height > ttl { @@ -405,12 +411,193 @@ func TestTxStore_UpdateExpiresTransactions(t *testing.T) { for txHash, wtx := range env.byHash { expiredByHeight := minHeight >= 0 && wtx.height < minHeight expiredByTime := !minTime.IsZero() && wtx.timestamp.Before(minTime) - if expiredByHeight || expiredByTime { - delete(env.byHash, txHash) + if !(expiredByHeight || expiredByTime) { + continue + } + if !cfg.RemoveExpiredTxsFromQueue { + if _, ok := readyBeforeUpdateSet[txHash]; ok { + continue + } } + delete(env.byHash, txHash) } env.markReadyTxs() - env.assertState(t) + env.assertState(t) + } +} + +func TestTxStore_UpdateExpiresTransactions(t *testing.T) { + testTxStoreUpdateExpiresTransactions(t, true) +} + +func TestTxStore_UpdateExpiresTransactionsKeepsReadyWhenConfigured(t *testing.T) { + testTxStoreUpdateExpiresTransactions(t, false) +} + +func TestTxStore_ExpiredTxCacheBehavior(t *testing.T) { + rng := utils.TestRng() + + for _, tc := range []struct { + name string + keepInvalidTxsInCache bool + removeExpiredFromQueue bool + wantReadyPresent bool + wantPendingPresent bool + wantReadyCached bool + wantPendingCached bool + }{ + { + name: "remove expired and drop from cache", + keepInvalidTxsInCache: false, + removeExpiredFromQueue: true, + wantReadyPresent: false, + wantPendingPresent: false, + wantReadyCached: false, + wantPendingCached: false, + }, + { + name: "remove expired and keep in cache", + keepInvalidTxsInCache: true, + removeExpiredFromQueue: true, + wantReadyPresent: false, + wantPendingPresent: false, + wantReadyCached: true, + wantPendingCached: true, + }, + { + name: "keep expired ready and drop expired pending from cache", + keepInvalidTxsInCache: false, + removeExpiredFromQueue: false, + wantReadyPresent: true, + wantPendingPresent: false, + wantReadyCached: true, + wantPendingCached: false, + }, + { + name: "keep expired ready and keep expired pending in cache", + keepInvalidTxsInCache: true, + removeExpiredFromQueue: false, + wantReadyPresent: true, + wantPendingPresent: false, + wantReadyCached: true, + wantPendingCached: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := TestConfig() + cfg.CacheSize = 10 + cfg.TTLDuration = utils.Some(time.Second) + cfg.TTLNumBlocks = utils.None[int64]() + cfg.KeepInvalidTxsInCache = tc.keepInvalidTxsInCache + cfg.RemoveExpiredTxsFromQueue = tc.removeExpiredFromQueue + + app := newEVMNonceApp() + txStore := NewTxStore(cfg, proxy.New(app, proxy.NopMetrics()), NopMetrics()) + env := newTestEnv(rng, txStore, app, 1) + address := env.accounts[0].address + env.app.setNonce(address, 7) + env.app.setBalance(address, 100) + + ready := &WrappedTx{ + hashedTx: newHashedTx(utils.GenBytes(rng, 32)), + timestamp: time.Unix(100, 0), + priority: 10, + gasWanted: 1, + estimatedGas: 1, + evm: utils.Some(evmTx{ + address: address, + seiAddress: address.Bytes(), + nonce: 7, + requiredBalance: big.NewInt(0), + }), + } + pending := &WrappedTx{ + hashedTx: newHashedTx(utils.GenBytes(rng, 32)), + timestamp: time.Unix(100, 0), + priority: 20, + gasWanted: 1, + estimatedGas: 1, + evm: utils.Some(evmTx{ + address: address, + seiAddress: address.Bytes(), + nonce: 8, + requiredBalance: big.NewInt(200), + }), + } + + require.NoError(t, txStore.Insert(ready)) + require.NoError(t, txStore.Insert(pending)) + + txStore.Update(updateSpec{ + Now: time.Unix(102, 0), + Height: 1, + TxResults: map[types.TxHash]bool{}, + Constraints: NopTxConstraints(), + NewPriorities: map[types.TxHash]int64{}, + }) + + _, readyPresent := txStore.ByHash(ready.Hash()) + _, pendingPresent := txStore.ByHash(pending.Hash()) + require.Equal(t, tc.wantReadyPresent, readyPresent) + require.Equal(t, tc.wantPendingPresent, pendingPresent) + require.Equal(t, tc.wantReadyCached, txStore.CacheHas(ready.Hash())) + require.Equal(t, tc.wantPendingCached, txStore.CacheHas(pending.Hash())) + }) + } +} + +func TestTxStore_NoncePrunedTxCacheBehavior(t *testing.T) { + rng := utils.TestRng() + + for _, tc := range []struct { + name string + keepInvalidTxsInCache bool + wantCached bool + }{ + { + name: "drop pruned txs from cache", + keepInvalidTxsInCache: false, + wantCached: false, + }, + { + name: "keep pruned txs in cache", + keepInvalidTxsInCache: true, + wantCached: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := TestConfig() + cfg.CacheSize = 10 + cfg.KeepInvalidTxsInCache = tc.keepInvalidTxsInCache + + app := newEVMNonceApp() + txStore := NewTxStore(cfg, proxy.New(app, proxy.NopMetrics()), NopMetrics()) + env := newTestEnv(rng, txStore, app, 1) + address := env.accounts[0].address + env.app.setNonce(address, 7) + env.app.setBalance(address, 100) + + prunedReady := makeEvmTxForTest(rng, address, 7, 10, 0) + prunedPending := makeEvmTxForTest(rng, address, 8, 20, 200) + require.NoError(t, txStore.Insert(prunedReady)) + require.NoError(t, txStore.Insert(prunedPending)) + + env.app.setNonce(address, 9) + txStore.Update(updateSpec{ + Now: time.Now(), + Height: 1, + TxResults: map[types.TxHash]bool{}, + Constraints: NopTxConstraints(), + NewPriorities: map[types.TxHash]int64{}, + }) + + _, readyPresent := txStore.ByHash(prunedReady.Hash()) + _, pendingPresent := txStore.ByHash(prunedPending.Hash()) + require.False(t, readyPresent) + require.False(t, pendingPresent) + require.Equal(t, tc.wantCached, txStore.CacheHas(prunedReady.Hash())) + require.Equal(t, tc.wantCached, txStore.CacheHas(prunedPending.Hash())) + }) } } From 0fd6a2deb8721975d2591ab15c468a6a0cce698d Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 12:23:07 +0200 Subject: [PATCH 046/100] updated caching logic --- sei-tendermint/internal/mempool/tx.go | 64 +++++++++++---------------- 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 0acc4b5604..6a9e6fa957 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -423,7 +423,7 @@ func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) { total.Inc(wtx.Size()) limitOk := total.LessEqual(&inner.softLimit) if !limitOk || s.insert(inner, wtx) != nil { - if !limitOk || !s.config.KeepInvalidTxsInCache { + if !s.config.KeepInvalidTxsInCache { s.cache.Remove(wtx.Hash()) } s.metrics.RemovedTxs.Add(1) @@ -454,51 +454,41 @@ func (s *txStore) Update(spec updateSpec) { if d, ok := s.config.TTLDuration.Get(); ok { minTime = utils.Some(spec.Now.Add(-d)) } - for inner := range s.inner.Lock() { - isExpired := func(wtx *WrappedTx) bool { - if !s.config.RemoveExpiredTxsFromQueue && inner.isReady(wtx) { - return false - } - if t, ok := minTime.Get(); ok && wtx.timestamp.Before(t) { - return true - } - if h, ok := minHeight.Get(); ok && wtx.height < h { - return true - } - return false + isExpired := func(wtx *WrappedTx) bool { + if t, ok := minTime.Get(); ok && wtx.timestamp.Before(t) { + return true } + if h, ok := minHeight.Get(); ok && wtx.height < h { + return true + } + return false + } + for inner := range s.inner.Lock() { for txHash, wtx := range inner.byHash { expired := isExpired(wtx) + if expired { + s.metrics.ExpiredTxs.Add(1) + } invalid := wtx.check(spec.Constraints) != nil - remove := expired || invalid - executed := false - if success, ok := spec.TxResults[wtx.Hash()]; ok { - // Executed transactions should be removed. - executed = true - remove = true + success, executed := spec.TxResults[wtx.Hash()] + remove := invalid || executed || (expired && (s.config.RemoveExpiredTxsFromQueue || !inner.isReady(wtx))) + if remove { + // KeepInvalidTxsInCache decides whether we give just 1 chance to each inserted transaction. + // In particular evicted/expired transactions caching depends on it. + // If not set, we just cache executed transactions (and txs invalidated pre-insertion) if !s.config.KeepInvalidTxsInCache { - if !success { - // Failed txs are eligible for reexection once. + // Cleanup the cache. + if !executed { + s.cache.Remove(txHash) + } else if !success { + // We keep executed txs in cache, unless they failed + // in which case we give them a second attempt. if s.failedTxs.Push(txHash) { s.cache.Remove(txHash) - s.metrics.CacheSize.Set(float64(s.cache.Size())) + } else { + s.failedTxs.Remove(txHash) } - } else { - s.failedTxs.Remove(txHash) - } - } - } - if remove { - if expired { - s.metrics.ExpiredTxs.Add(1) - // For some reason we treat expired txs as invalid here. - if !s.config.KeepInvalidTxsInCache { - s.cache.Remove(txHash) - s.metrics.CacheSize.Set(float64(s.cache.Size())) } - } else if invalid && !executed && !s.config.KeepInvalidTxsInCache { - s.cache.Remove(txHash) - s.metrics.CacheSize.Set(float64(s.cache.Size())) } delete(inner.byHash, txHash) s.metrics.RemovedTxs.Add(1) From 7208030d6c0ecf8ba9603d0e8d87d28759686d22 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 12:32:26 +0200 Subject: [PATCH 047/100] cache compatible --- sei-tendermint/internal/mempool/tx.go | 8 ++++--- sei-tendermint/internal/mempool/tx_test.go | 26 +++++++++++----------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 6a9e6fa957..599f9bbca7 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -299,7 +299,7 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { return errSameNonce } // Remove the old transaction. - s.cache.Remove(old.Hash()) + s.cache.Remove(old.Hash()) // evicted txs are not cached s.metrics.CacheSize.Set(float64(s.cache.Size())) delete(inner.byHash, old.Hash()) s.metrics.RemovedTxs.Add(1) @@ -422,8 +422,10 @@ func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) { total := inner.state.Load().total total.Inc(wtx.Size()) limitOk := total.LessEqual(&inner.softLimit) + // NOTE: insertion is lazily evaluated here. if !limitOk || s.insert(inner, wtx) != nil { - if !s.config.KeepInvalidTxsInCache { + // NOTE: evicted txs are not cached unconditionally + if !limitOk || !s.config.KeepInvalidTxsInCache { s.cache.Remove(wtx.Hash()) } s.metrics.RemovedTxs.Add(1) @@ -474,7 +476,7 @@ func (s *txStore) Update(spec updateSpec) { remove := invalid || executed || (expired && (s.config.RemoveExpiredTxsFromQueue || !inner.isReady(wtx))) if remove { // KeepInvalidTxsInCache decides whether we give just 1 chance to each inserted transaction. - // In particular evicted/expired transactions caching depends on it. + // In particular expired transactions caching depends on it. // If not set, we just cache executed transactions (and txs invalidated pre-insertion) if !s.config.KeepInvalidTxsInCache { // Cleanup the cache. diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index 9a0b2fd6e0..7fbc9d2722 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -373,12 +373,12 @@ func testTxStoreUpdateExpiresTransactions(t *testing.T, removeExpiredTxsFromQueu app.setBalance(account.address, 1_000_000) } env.insertTxs(t, 100, func(account *testAccount, nonce uint64) *WrappedTx { - return makeTx( - account.address, - nonce, - int64(rng.Intn(28)+1), - baseTime.Add(time.Duration(rng.Intn(31))*time.Second), - ) + return makeTx( + account.address, + nonce, + int64(rng.Intn(28)+1), + baseTime.Add(time.Duration(rng.Intn(31))*time.Second), + ) }) // Record the transactions that are initially ready; the stable ready list @@ -438,13 +438,13 @@ func TestTxStore_ExpiredTxCacheBehavior(t *testing.T) { rng := utils.TestRng() for _, tc := range []struct { - name string - keepInvalidTxsInCache bool - removeExpiredFromQueue bool - wantReadyPresent bool - wantPendingPresent bool - wantReadyCached bool - wantPendingCached bool + name string + keepInvalidTxsInCache bool + removeExpiredFromQueue bool + wantReadyPresent bool + wantPendingPresent bool + wantReadyCached bool + wantPendingCached bool }{ { name: "remove expired and drop from cache", From 5e67f49fef7c12dba17537ec0484b404f2de19ca Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 17:53:31 +0200 Subject: [PATCH 048/100] fixes --- .../internal/mempool/mempool_test.go | 6 ++-- .../internal/mempool/reactor/reactor_test.go | 28 +++++++++---------- sei-tendermint/internal/mempool/tx.go | 15 ++++------ 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index 13fc97daad..9c28578275 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -890,8 +890,7 @@ func TestTxMempool_ReapTxs_EVMFirst(t *testing.T) { } func TestBlockFailedTxNotReAdmittedAfterSecondFailure(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() app := &application{Application: kvstore.NewApplication()} cfg := TestConfig() @@ -942,8 +941,7 @@ func TestBlockFailedTxNotReAdmittedAfterSecondFailure(t *testing.T) { } func TestBlockFailedTxTrackerClearedOnSuccess(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() app := &application{Application: kvstore.NewApplication()} cfg := TestConfig() diff --git a/sei-tendermint/internal/mempool/reactor/reactor_test.go b/sei-tendermint/internal/mempool/reactor/reactor_test.go index 21a14c22e6..a866b91516 100644 --- a/sei-tendermint/internal/mempool/reactor/reactor_test.go +++ b/sei-tendermint/internal/mempool/reactor/reactor_test.go @@ -354,39 +354,39 @@ func TestReactorConcurrency(t *testing.T) { rts.start(t) var wg sync.WaitGroup + var primaryHeight int64 + var secondaryHeight int64 for range runtime.NumCPU() * 2 { - wg.Add(2) - - txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs) - go func() { - defer wg.Done() - + wg.Go(func() { + txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs) txmp := rts.mempools[primary] txmp.Lock() defer txmp.Unlock() + primaryHeight++ + height := primaryHeight deliverTxResponses := make([]*abci.ExecTxResult, len(txs)) for i := range txs { deliverTxResponses[i] = &abci.ExecTxResult{Code: 0} } - require.NoError(t, txmp.Update(ctx, 1, convertTex(txs), deliverTxResponses, mempool.NopTxConstraints(), true)) - }() - - _ = checkTxs(ctx, t, rts.reactors[secondary].mempool, numTxs) - go func() { - defer wg.Done() + require.NoError(t, txmp.Update(ctx, height, convertTex(txs), deliverTxResponses, mempool.NopTxConstraints(), true)) + }) + wg.Go(func() { + _ = checkTxs(ctx, t, rts.reactors[secondary].mempool, numTxs) txmp := rts.mempools[secondary] txmp.Lock() defer txmp.Unlock() + secondaryHeight++ + height := secondaryHeight - err := txmp.Update(ctx, 1, []types.Tx{}, make([]*abci.ExecTxResult, 0), mempool.NopTxConstraints(), true) + err := txmp.Update(ctx, height, []types.Tx{}, make([]*abci.ExecTxResult, 0), mempool.NopTxConstraints(), true) require.NoError(t, err) - }() + }) } wg.Wait() diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 599f9bbca7..721f5be0da 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -480,16 +480,13 @@ func (s *txStore) Update(spec updateSpec) { // If not set, we just cache executed transactions (and txs invalidated pre-insertion) if !s.config.KeepInvalidTxsInCache { // Cleanup the cache. - if !executed { + // We keep executed txs in cache, unless they failed + // in which case we give them a second attempt. + // NOTE: failedTxs.Push is executed lazily. + if !executed || (!success && s.failedTxs.Push(txHash)) { s.cache.Remove(txHash) - } else if !success { - // We keep executed txs in cache, unless they failed - // in which case we give them a second attempt. - if s.failedTxs.Push(txHash) { - s.cache.Remove(txHash) - } else { - s.failedTxs.Remove(txHash) - } + } else { + s.failedTxs.Remove(txHash) } } delete(inner.byHash, txHash) From 7be9604fa3a98ed3172eaadb841a26c84fce3889 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 18:53:00 +0200 Subject: [PATCH 049/100] more fixes --- sei-tendermint/internal/mempool/cache.go | 35 +++----- .../internal/mempool/cache_bench_test.go | 4 +- sei-tendermint/internal/mempool/cache_test.go | 34 ++++---- sei-tendermint/internal/mempool/mempool.go | 45 ++++++++-- sei-tendermint/internal/mempool/tx.go | 83 ++++++++++--------- 5 files changed, 110 insertions(+), 91 deletions(-) diff --git a/sei-tendermint/internal/mempool/cache.go b/sei-tendermint/internal/mempool/cache.go index a730fdf61d..c959ad2bd8 100644 --- a/sei-tendermint/internal/mempool/cache.go +++ b/sei-tendermint/internal/mempool/cache.go @@ -3,7 +3,6 @@ package mempool import ( "container/list" "context" - "sync" "time" "github.com/patrickmn/go-cache" @@ -11,10 +10,9 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) -// LRUTxCache maintains a thread-safe LRU cache of raw transactions. The cache +// lruTxCache maintains a NON-threadsafe lru cache of raw transactions. The cache // only stores the hash of the raw transaction. -type LRUTxCache struct { - mtx sync.Mutex +type lruTxCache struct { size int cacheMap map[cacheKey]*list.Element list *list.List @@ -23,7 +21,7 @@ type LRUTxCache struct { type cacheKey = string -// NewLRUTxCache creates an LRU (Least Recently Used) cache that stores +// newLRUTxCache creates an LRU (Least Recently Used) cache that stores // transactions by key. Keys are derived from the transaction key and trimmed to // at most maxKeyLen bytes for predictable and efficient storage. If maxKeyLen is // zero or negative, keys are not trimmed. When the cache exceeds cacheSize, the @@ -34,8 +32,8 @@ type cacheKey = string // positives in cache lookups. A larger value reduces collision risk but uses // more memory. A common choice is to use the full length of a cryptographic hash // (e.g., 32 bytes for SHA-256) to balance memory usage and collision risk. -func NewLRUTxCache(cacheSize int, maxKeyLen int) *LRUTxCache { - return &LRUTxCache{ +func newLRUTxCache(cacheSize int, maxKeyLen int) *lruTxCache { + return &lruTxCache{ size: cacheSize, cacheMap: make(map[cacheKey]*list.Element, cacheSize), list: list.New(), @@ -43,27 +41,20 @@ func NewLRUTxCache(cacheSize int, maxKeyLen int) *LRUTxCache { } } -func (c *LRUTxCache) Has(txHash types.TxHash) bool { - c.mtx.Lock() - defer c.mtx.Unlock() +func (c *lruTxCache) Has(txHash types.TxHash) bool { _, ok := c.cacheMap[c.toCacheKey(txHash)] return ok } -func (c *LRUTxCache) Reset() { - c.mtx.Lock() - defer c.mtx.Unlock() - +func (c *lruTxCache) Reset() { c.cacheMap = make(map[cacheKey]*list.Element, c.size) c.list.Init() } -func (c *LRUTxCache) Push(txHash types.TxHash) bool { +func (c *lruTxCache) Push(txHash types.TxHash) bool { if c.size <= 0 { return true } - c.mtx.Lock() - defer c.mtx.Unlock() key := c.toCacheKey(txHash) moved, ok := c.cacheMap[key] @@ -87,9 +78,7 @@ func (c *LRUTxCache) Push(txHash types.TxHash) bool { return true } -func (c *LRUTxCache) Remove(txHash types.TxHash) { - c.mtx.Lock() - defer c.mtx.Unlock() +func (c *lruTxCache) Remove(txHash types.TxHash) { key := c.toCacheKey(txHash) e := c.cacheMap[key] @@ -100,13 +89,11 @@ func (c *LRUTxCache) Remove(txHash types.TxHash) { } } -func (c *LRUTxCache) Size() int { - c.mtx.Lock() - defer c.mtx.Unlock() +func (c *lruTxCache) Size() int { return c.list.Len() } -func (c *LRUTxCache) toCacheKey(key types.TxHash) cacheKey { +func (c *lruTxCache) toCacheKey(key types.TxHash) cacheKey { return cacheKey(trimToSize(key, c.maxKeyLen)) } diff --git a/sei-tendermint/internal/mempool/cache_bench_test.go b/sei-tendermint/internal/mempool/cache_bench_test.go index 519d8bb2e9..d3be978c44 100644 --- a/sei-tendermint/internal/mempool/cache_bench_test.go +++ b/sei-tendermint/internal/mempool/cache_bench_test.go @@ -8,7 +8,7 @@ import ( ) func BenchmarkCacheInsertTime(b *testing.B) { - cache := NewLRUTxCache(b.N, 0) + cache := newLRUTxCache(b.N, 0) txs := make([]types.TxHash, b.N) for i := 0; i < b.N; i++ { @@ -27,7 +27,7 @@ func BenchmarkCacheInsertTime(b *testing.B) { // This benchmark is probably skewed, since we actually will be removing // txs in parallel, which may cause some overhead due to mutex locking. func BenchmarkCacheRemoveTime(b *testing.B) { - cache := NewLRUTxCache(b.N, 0) + cache := newLRUTxCache(b.N, 0) txs := make([]types.TxHash, b.N) for i := 0; i < b.N; i++ { diff --git a/sei-tendermint/internal/mempool/cache_test.go b/sei-tendermint/internal/mempool/cache_test.go index cd386259a6..5a9a915c4b 100644 --- a/sei-tendermint/internal/mempool/cache_test.go +++ b/sei-tendermint/internal/mempool/cache_test.go @@ -10,9 +10,9 @@ import ( "github.com/stretchr/testify/assert" ) -func TestLRUTxCache(t *testing.T) { - t.Run("NewLRUTxCache", func(t *testing.T) { - cache := NewLRUTxCache(10, 0) +func TestlruTxCache(t *testing.T) { + t.Run("newLRUTxCache", func(t *testing.T) { + cache := newLRUTxCache(10, 0) assert.NotNil(t, cache) assert.Equal(t, 10, cache.size) assert.NotNil(t, cache.cacheMap) @@ -20,7 +20,7 @@ func TestLRUTxCache(t *testing.T) { }) t.Run("Push_NewTransaction", func(t *testing.T) { - cache := NewLRUTxCache(3, 0) + cache := newLRUTxCache(3, 0) tx := types.Tx("test1").Hash() // First push should return true (newly added) @@ -30,7 +30,7 @@ func TestLRUTxCache(t *testing.T) { }) t.Run("Push_DuplicateTransaction", func(t *testing.T) { - cache := NewLRUTxCache(3, 0) + cache := newLRUTxCache(3, 0) tx := types.Tx("test1").Hash() // First push @@ -44,7 +44,7 @@ func TestLRUTxCache(t *testing.T) { }) t.Run("Push_CacheFull", func(t *testing.T) { - cache := NewLRUTxCache(2, 0) + cache := newLRUTxCache(2, 0) // Add two transactions tx1 := types.Tx("test1").Hash() @@ -54,7 +54,7 @@ func TestLRUTxCache(t *testing.T) { cache.Push(tx2) assert.Equal(t, 2, cache.Size()) - // Add third transaction, should evict the first one (LRU) + // Add third transaction, should evict the first one (lru) tx3 := types.Tx("test3").Hash() cache.Push(tx3) assert.Equal(t, 2, cache.Size()) @@ -64,7 +64,7 @@ func TestLRUTxCache(t *testing.T) { }) t.Run("Remove_ExistingTransaction", func(t *testing.T) { - cache := NewLRUTxCache(3, 0) + cache := newLRUTxCache(3, 0) tx := types.Tx("test1").Hash() cache.Push(tx) @@ -75,7 +75,7 @@ func TestLRUTxCache(t *testing.T) { }) t.Run("Remove_NonExistentTransaction", func(t *testing.T) { - cache := NewLRUTxCache(3, 0) + cache := newLRUTxCache(3, 0) tx := types.Tx("test1").Hash() // Remove non-existent transaction should not panic @@ -84,7 +84,7 @@ func TestLRUTxCache(t *testing.T) { }) t.Run("Reset", func(t *testing.T) { - cache := NewLRUTxCache(3, 0) + cache := newLRUTxCache(3, 0) // Add some transactions cache.Push(types.Tx("test1").Hash()) @@ -97,7 +97,7 @@ func TestLRUTxCache(t *testing.T) { }) t.Run("Size", func(t *testing.T) { - cache := NewLRUTxCache(3, 0) + cache := newLRUTxCache(3, 0) assert.Equal(t, 0, cache.Size()) cache.Push(types.Tx("test1").Hash()) @@ -325,8 +325,8 @@ func TestDuplicateTxCache(t *testing.T) { }) } -func TestLRUTxCache_ConcurrentAccess(t *testing.T) { - cache := NewLRUTxCache(100, 0) +func TestlruTxCache_ConcurrentAccess(t *testing.T) { + cache := newLRUTxCache(100, 0) // Test concurrent access const numGoroutines = 10 @@ -398,9 +398,9 @@ func TestDuplicateTxCache_ConcurrentAccess(t *testing.T) { assert.True(t, nonDuplicateCount >= 0) } -func TestLRUTxCache_EdgeCases(t *testing.T) { +func TestlruTxCache_EdgeCases(t *testing.T) { t.Run("ZeroSizeCache", func(t *testing.T) { - cache := NewLRUTxCache(0, 0) + cache := newLRUTxCache(0, 0) tx := types.Tx("test").Hash() // Zero-sized cache is effectively disabled. @@ -411,7 +411,7 @@ func TestLRUTxCache_EdgeCases(t *testing.T) { }) t.Run("NegativeSizeCache", func(t *testing.T) { - cache := NewLRUTxCache(-1, 0) + cache := newLRUTxCache(-1, 0) tx := types.Tx("test").Hash() // Negative-sized cache is effectively disabled. @@ -422,7 +422,7 @@ func TestLRUTxCache_EdgeCases(t *testing.T) { }) t.Run("NilTransaction", func(t *testing.T) { - cache := NewLRUTxCache(10, 0) + cache := newLRUTxCache(10, 0) var tx types.TxHash // Should handle nil transaction gracefully diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index c450064f47..71935f9789 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -150,6 +150,28 @@ func DefaultConfig() *Config { } } +type lockMap[K comparable] struct{ inner utils.Mutex[map[K]struct{}] } + +func newLockMap[K comparable]() *lockMap[K] { + return &lockMap[K]{inner: utils.NewMutex(map[K]struct{}{})} +} + +func (m *lockMap[K]) Lock(k K) bool { + for inner := range m.inner.Lock() { + if _, ok := inner[k]; ok { + return false + } + inner[k] = struct{}{} + } + return true +} + +func (m *lockMap[K]) Unlock(k K) { + for inner := range m.inner.Lock() { + delete(inner, k) + } +} + // TxMempool defines a prioritized mempool data structure used by the v1 mempool // reactor. It keeps a thread-safe priority queue of transactions that is used // when a block proposer constructs a block and a thread-safe linked-list that @@ -158,6 +180,7 @@ type TxMempool struct { metrics *Metrics config *Config app *proxy.Proxy + txLocks *lockMap[types.TxHash] // txsAvailable fires once for each height when the mempool is not empty txsAvailable chan struct{} @@ -201,6 +224,7 @@ func NewTxMempool( config: cfg, app: app, txsAvailable: make(chan struct{}, 1), + txLocks: newLockMap[types.TxHash](), height: -1, metrics: metrics, txStore: NewTxStore(cfg, app, metrics), @@ -282,6 +306,14 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response return nil, fmt.Errorf("%w: max size is %d, but got %d", ErrTxTooLarge, txmp.config.MaxTxBytes, txSize) } hTx := newHashedTx(tx) + + // Avoid processing same transaction in parallel. + if !txmp.txLocks.Lock(hTx.Hash()) { + // ErrTxInCache is returned for backward compatibility. + return nil, ErrTxInCache + } + defer txmp.txLocks.Unlock(hTx.Hash()) + constraints, err := txmp.txConstraintsFetcher() if err != nil { return nil, fmt.Errorf("txmp.txConstraintsFetcher(): %w", err) @@ -317,22 +349,19 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response return nil, ErrTxInCache } - // Check TTL cache to see if we've recently processed this transaction - // Only execute TTL cache logic if we're using a real TTL cache (not NOP) if c, ok := txmp.duplicateTxsCache.Get(); ok { c.Increment(hTx.Hash()) } res, err := txmp.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{Tx: tx}) if err != nil || !res.IsOK() { + txmp.txStore.CachePush(hTx.Hash()) txmp.metrics.NumberOfFailedCheckTxs.Add(1) txmp.metrics.observeCheckTxPriorityDistribution(0, false, "", true) } if err != nil { - txmp.txStore.CachePush(hTx.Hash()) return nil, err } if !res.IsOK() { - txmp.txStore.CachePush(hTx.Hash()) return res.ResponseCheckTx, nil } txmp.metrics.NumberOfSuccessfulCheckTxs.Add(1) @@ -452,6 +481,7 @@ func (txmp *TxMempool) Update( txResults[tx.Hash()] = execTxResult[i].Code == abci.CodeTypeOK } newPriorities := map[types.TxHash]int64{} + invalidTxs := map[types.TxHash]bool{} if recheck { for _, wtx := range txmp.txStore.ReadyTxs() { if _, ok := txResults[wtx.Hash()]; ok { @@ -463,11 +493,7 @@ func (txmp *TxMempool) Update( Type: abci.CheckTxTypeV2Recheck, }) if err != nil || !res.IsOK() { - // If recheck fails, just remove the tx. - // TODO(gprusak): we emulate the fact that we don't want this tx - // by saying that it was already executed - this way it is pushed to cache and removed from mempool. - // It deserves more explicit handling though. - txResults[wtx.Hash()] = true + invalidTxs[wtx.Hash()] = true } else { // If succeeds, we just care about the new priority. newPriorities[wtx.Hash()] = res.Priority @@ -479,6 +505,7 @@ func (txmp *TxMempool) Update( Height: blockHeight, TxResults: txResults, NewPriorities: newPriorities, + InvalidTxs: invalidTxs, Constraints: txConstraints, }) txmp.notifyTxsAvailable() diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 721f5be0da..5b45895a71 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -125,6 +125,19 @@ type txStoreInner struct { softLimit txCounter hardLimit txCounter state utils.AtomicSend[txStoreState] + + // Cache of already seen txs, reducess pressure on app. + // It is a superset of transactions in txStore. + // * successfully inserted transactions are automatically added to cache. + // * txs which fail Insert() are NOT added to cache and can be reattempted later. + // * invalid transactions can be recorded via CachePush. + // * txs dropped due to pruning are removed from cache. + // * txs successfully executed are kept in cache to avoid reinsert. + // * txs failed execution are eligible to be reexecuted once (iff config.KeepInvalidTxsInCache). + cache *lruTxCache + // Tracks transactions which already failed execution once + // but are eligible for reexecution (not added yet to cache) + failedTxs *lruTxCache } // Properties: @@ -142,19 +155,6 @@ type txStore struct { app *proxy.Proxy metrics *Metrics - // Cache of already seen txs, reducess pressure on app. - // It is a superset of transactions in txStore. - // * successfully inserted transactions are automatically added to cache. - // * txs which fail Insert() are NOT added to cache and can be reattempted later. - // * invalid transactions can be recorded via CachePush. - // * txs dropped due to pruning are removed from cache. - // * txs successfully executed are kept in cache to avoid reinsert. - // * txs failed execution are eligible to be reexecuted once (iff config.KeepInvalidTxsInCache). - cache *LRUTxCache - // Tracks transactions which already failed execution once - // but are eligible for reexecution (not added yet to cache) - failedTxs *LRUTxCache - inner utils.RWMutex[*txStoreInner] state utils.AtomicRecv[txStoreState] // List of transactions that were ready now OR at some point in the past. @@ -173,24 +173,24 @@ func NewTxStore(cfg *Config, app *proxy.Proxy, metrics *Metrics) *txStore { softLimit: softLimit, hardLimit: hardLimit, state: utils.NewAtomicSend(txStoreState{}), + cache: newLRUTxCache(cfg.CacheSize, maxCacheKeySize), + failedTxs: newLRUTxCache(cfg.CacheSize, maxCacheKeySize), } return &txStore{ - config: cfg, - cache: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), - failedTxs: NewLRUTxCache(cfg.CacheSize, maxCacheKeySize), - app: app, - metrics: metrics, - inner: utils.NewRWMutex(inner), - readyTxs: clist.New[types.Tx](), - state: inner.state.Subscribe(), + config: cfg, + app: app, + metrics: metrics, + inner: utils.NewRWMutex(inner), + readyTxs: clist.New[types.Tx](), + state: inner.state.Subscribe(), } } func (s *txStore) Clear() { for inner := range s.inner.Lock() { - s.cache.Reset() - s.metrics.CacheSize.Set(float64(s.cache.Size())) - s.failedTxs.Reset() + inner.cache.Reset() + s.metrics.CacheSize.Set(float64(inner.cache.Size())) + inner.failedTxs.Reset() inner.byHash = map[types.TxHash]*WrappedTx{} inner.byNonce = map[evmAddrNonce]*WrappedTx{} inner.accounts = map[common.Address]*evmAccount{} @@ -201,13 +201,18 @@ func (s *txStore) Clear() { // Checks if cache contains a given hash. func (s *txStore) CacheHas(txHash types.TxHash) bool { - return s.cache.Has(txHash) + for inner := range s.inner.RLock() { + return inner.cache.Has(txHash) + } + panic("unreachable") } // Pushes a tx to cache, effectively blocking it from being inserted. func (s *txStore) CachePush(txHash types.TxHash) { - s.cache.Push(txHash) - s.metrics.CacheSize.Set(float64(s.cache.Size())) + for inner := range s.inner.Lock() { + inner.cache.Push(txHash) + s.metrics.CacheSize.Set(float64(inner.cache.Size())) + } } // Size returns the total number of transactions in the store. @@ -299,8 +304,8 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { return errSameNonce } // Remove the old transaction. - s.cache.Remove(old.Hash()) // evicted txs are not cached - s.metrics.CacheSize.Set(float64(s.cache.Size())) + inner.cache.Remove(old.Hash()) // evicted txs are not cached + s.metrics.CacheSize.Set(float64(inner.cache.Size())) delete(inner.byHash, old.Hash()) s.metrics.RemovedTxs.Add(1) state.total.Dec(old.Size()) @@ -398,9 +403,9 @@ func (s *txStore) Insert(wtx *WrappedTx) error { return errMempoolFull } } + inner.cache.Push(wtx.Hash()) + s.metrics.CacheSize.Set(float64(inner.cache.Size())) } - s.cache.Push(wtx.Hash()) - s.metrics.CacheSize.Set(float64(s.cache.Size())) return nil } @@ -426,7 +431,7 @@ func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) { if !limitOk || s.insert(inner, wtx) != nil { // NOTE: evicted txs are not cached unconditionally if !limitOk || !s.config.KeepInvalidTxsInCache { - s.cache.Remove(wtx.Hash()) + inner.cache.Remove(wtx.Hash()) } s.metrics.RemovedTxs.Add(1) s.metrics.EvictedTxs.Add(1) @@ -435,16 +440,16 @@ func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) { } } } - s.metrics.CacheSize.Set(float64(s.cache.Size())) + s.metrics.CacheSize.Set(float64(inner.cache.Size())) } type updateSpec struct { Now time.Time Height int64 - // Indicates whether tx succeeded. - TxResults map[types.TxHash]bool + TxResults map[types.TxHash]bool // true - success, false - failed, missing - not executed Constraints TxConstraints NewPriorities map[types.TxHash]int64 + InvalidTxs map[types.TxHash]bool } func (s *txStore) Update(spec updateSpec) { @@ -471,7 +476,7 @@ func (s *txStore) Update(spec updateSpec) { if expired { s.metrics.ExpiredTxs.Add(1) } - invalid := wtx.check(spec.Constraints) != nil + invalid := spec.InvalidTxs[wtx.Hash()] || wtx.check(spec.Constraints) != nil success, executed := spec.TxResults[wtx.Hash()] remove := invalid || executed || (expired && (s.config.RemoveExpiredTxsFromQueue || !inner.isReady(wtx))) if remove { @@ -483,10 +488,10 @@ func (s *txStore) Update(spec updateSpec) { // We keep executed txs in cache, unless they failed // in which case we give them a second attempt. // NOTE: failedTxs.Push is executed lazily. - if !executed || (!success && s.failedTxs.Push(txHash)) { - s.cache.Remove(txHash) + if !executed || (!success && inner.failedTxs.Push(txHash)) { + inner.cache.Remove(txHash) } else { - s.failedTxs.Remove(txHash) + inner.failedTxs.Remove(txHash) } } delete(inner.byHash, txHash) From d9b1814ae662b835a3f1b5774f284fe0e093c495 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 19:02:21 +0200 Subject: [PATCH 050/100] removed irrelevant test --- sei-tendermint/internal/mempool/cache_test.go | 37 +------------------ sei-tendermint/internal/mempool/mempool.go | 2 +- .../internal/mempool/mempool_test.go | 2 +- sei-tendermint/internal/mempool/tx.go | 14 +++++-- 4 files changed, 15 insertions(+), 40 deletions(-) diff --git a/sei-tendermint/internal/mempool/cache_test.go b/sei-tendermint/internal/mempool/cache_test.go index 5a9a915c4b..9e8be7c9df 100644 --- a/sei-tendermint/internal/mempool/cache_test.go +++ b/sei-tendermint/internal/mempool/cache_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestlruTxCache(t *testing.T) { +func TestLRUTxCache(t *testing.T) { t.Run("newLRUTxCache", func(t *testing.T) { cache := newLRUTxCache(10, 0) assert.NotNil(t, cache) @@ -325,39 +325,6 @@ func TestDuplicateTxCache(t *testing.T) { }) } -func TestlruTxCache_ConcurrentAccess(t *testing.T) { - cache := newLRUTxCache(100, 0) - - // Test concurrent access - const numGoroutines = 10 - const operationsPerGoroutine = 100 - - var wg sync.WaitGroup - wg.Add(numGoroutines) - - for i := 0; i < numGoroutines; i++ { - go func(id int) { - defer wg.Done() - - for j := 0; j < operationsPerGoroutine; j++ { - tx := types.Tx(fmt.Sprintf("goroutine_%d_tx_%d", id, j)).Hash() - cache.Push(tx) - - if j%10 == 0 { - cache.Size() // Read operation - } - } - }(i) - } - - wg.Wait() - - // Verify final state is reasonable - size := cache.Size() - assert.True(t, size > 0) - assert.True(t, size <= 100) // Should not exceed cache size -} - func TestDuplicateTxCache_ConcurrentAccess(t *testing.T) { cache := NewDuplicateTxCache(100, 100*time.Millisecond, 0) @@ -398,7 +365,7 @@ func TestDuplicateTxCache_ConcurrentAccess(t *testing.T) { assert.True(t, nonDuplicateCount >= 0) } -func TestlruTxCache_EdgeCases(t *testing.T) { +func TestLRUTxCache_EdgeCases(t *testing.T) { t.Run("ZeroSizeCache", func(t *testing.T) { cache := newLRUTxCache(0, 0) tx := types.Tx("test").Hash() diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 71935f9789..c42e090749 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -505,7 +505,7 @@ func (txmp *TxMempool) Update( Height: blockHeight, TxResults: txResults, NewPriorities: newPriorities, - InvalidTxs: invalidTxs, + InvalidTxs: invalidTxs, Constraints: txConstraints, }) txmp.notifyTxsAvailable() diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index 9c28578275..eddf66c679 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -973,7 +973,7 @@ func TestBlockFailedTxTrackerClearedOnSuccess(t *testing.T) { // Success clears the failure tracker. Simulate LRU eviction of the // main cache entry so we can verify the tracker was actually reset. - txmp.txStore.cache.Remove(txHash) + txmp.txStore.CacheRemove(txHash) // Tx should now be re-admittable _, err = txmp.CheckTx(ctx, tx) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 5b45895a71..c776bec06c 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -215,6 +215,14 @@ func (s *txStore) CachePush(txHash types.TxHash) { } } +// Removes a tx from cache. +func (s *txStore) CacheRemove(txHash types.TxHash) { + for inner := range s.inner.Lock() { + inner.cache.Remove(txHash) + s.metrics.CacheSize.Set(float64(inner.cache.Size())) + } +} + // Size returns the total number of transactions in the store. func (s *txStore) State() txStoreState { return s.state.Load() } @@ -444,12 +452,12 @@ func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) { } type updateSpec struct { - Now time.Time - Height int64 + Now time.Time + Height int64 TxResults map[types.TxHash]bool // true - success, false - failed, missing - not executed Constraints TxConstraints NewPriorities map[types.TxHash]int64 - InvalidTxs map[types.TxHash]bool + InvalidTxs map[types.TxHash]bool } func (s *txStore) Update(spec updateSpec) { From b4d042bd77cde2f7531d5ea1566c97a5f1caae74 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 19:29:48 +0200 Subject: [PATCH 051/100] test fix --- sei-tendermint/internal/consensus/replay_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sei-tendermint/internal/consensus/replay_test.go b/sei-tendermint/internal/consensus/replay_test.go index a9eb4e6a56..c0d1107981 100644 --- a/sei-tendermint/internal/consensus/replay_test.go +++ b/sei-tendermint/internal/consensus/replay_test.go @@ -655,11 +655,12 @@ func testHandshakeReplay( store.commits = commits state := genesisState.Copy() + replayMempool := newReplayTxMempool(kvstore.NewProxy()) // run the chain through state.ApplyBlock to build up the tendermint state state = buildTMStateFromChain( ctx, t, - sim.Mempool, + replayMempool, sim.Evpool, stateStore, state, @@ -681,7 +682,7 @@ func testHandshakeReplay( stateStore := sm.NewStore(stateDB1) err := stateStore.Save(genesisState) require.NoError(t, err) - buildAppStateFromChain(ctx, t, app, stateStore, sim.Mempool, sim.Evpool, genesisState, chain, eventBus, nBlocks, mode, store) + buildAppStateFromChain(ctx, t, app, stateStore, sim.Evpool, genesisState, chain, eventBus, nBlocks, mode, store) } // Prune block store if requested @@ -759,7 +760,6 @@ func buildAppStateFromChain( t *testing.T, appClient *kvstore.Application, stateStore sm.Store, - mempool *mempool.TxMempool, evpool sm.EvidencePool, state sm.State, chain []*types.Block, @@ -771,6 +771,7 @@ func buildAppStateFromChain( t.Helper() // start a new app without handshake, play nBlocks blocks proxyApp := proxy.New(appClient, proxy.NopMetrics()) + mempool := newReplayTxMempool(proxyApp) state.Version.Consensus.App = kvstore.ProtocolVersion // simulate handshake, receive app version _, err := appClient.InitChain(ctx, &abci.RequestInitChain{}) require.NoError(t, err) From b072f49f805759fe254ec6975de07fe2ee4d05ec Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 20:03:37 +0200 Subject: [PATCH 052/100] adjusted priorityReservoir usage --- sei-tendermint/internal/mempool/mempool.go | 30 +++------------------- sei-tendermint/internal/mempool/tx.go | 20 ++++++++++----- 2 files changed, 17 insertions(+), 33 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index c42e090749..837186a753 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -12,7 +12,6 @@ import ( "github.com/ethereum/go-ethereum/common" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/libs/clist" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/libs/reservoir" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" @@ -203,10 +202,9 @@ type TxMempool struct { // the mempool via Update(). mtx sync.RWMutex txConstraintsFetcher TxConstraintsFetcher - - priorityReservoir *reservoir.Sampler[int64] } +func (txmp *TxMempool) Size() int { return txmp.txStore.State().total.count } func (txmp *TxMempool) SizeBytes() uint64 { return txmp.txStore.State().total.bytes } func (txmp *TxMempool) NumTxsNotPending() int { return txmp.txStore.State().ready.count } func (txmp *TxMempool) BytesNotPending() uint64 { return txmp.txStore.State().ready.bytes } @@ -229,7 +227,6 @@ func NewTxMempool( metrics: metrics, txStore: NewTxStore(cfg, app, metrics), txConstraintsFetcher: txConstraintsFetcher, - priorityReservoir: reservoir.New[int64](cfg.DropPriorityReservoirSize, cfg.DropPriorityThreshold, nil), // Use non-deterministic RNG } if cfg.DuplicateTxsCacheSize > 0 { @@ -256,14 +253,8 @@ func (txmp *TxMempool) Lock() { txmp.mtx.Lock() } // Unlock releases a write-lock on the mempool. func (txmp *TxMempool) Unlock() { txmp.mtx.Unlock() } -// Size returns the number of valid transactions in the mempool. It is -// thread-safe. -func (txmp *TxMempool) Size() int { - return txmp.txStore.State().total.count -} - func (txmp *TxMempool) utilisation() float64 { - return float64(txmp.NumTxsNotPending()) / float64(txmp.config.Size) + return float64(txmp.NumTxsNotPending()) / float64(txmp.config.Size+txmp.config.PendingSize) } // WaitForNextTx waits until the next transaction is available for gossip. @@ -335,7 +326,7 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response } txmp.metrics.observeCheckTxPriorityDistribution(hint.Priority, true, "", false) - cutoff, found := txmp.priorityReservoir.Percentile() + cutoff, found := txmp.txStore.priorityReservoir.Percentile() if found && hint.Priority <= cutoff { txmp.metrics.CheckTxDroppedByPriorityHint.Add(1) return nil, errors.New("priority not high enough for mempool") @@ -388,21 +379,6 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response requiredBalance: res.EVMRequiredBalance, }) } - // Update transaction priority reservoir with the true Tx priority - // as determined by the application. - // - // NOTE: This is done before potentially rejecting the transaction due to - // mempool being full. This is to ensure that the reservoir contains a - // representative sample of all transactions that have been processed by - // CheckTx. - // - // However, this is NOT done if the tx is pending, since a spammer could - // throw off the correct priority percentiles otherwise. - // - // We do not use the priority hint here as it may be misleading and - // inaccurate. The true priority as determined by the application is the - // most accurate. - txmp.priorityReservoir.Add(wtx.priority) if err := wtx.check(constraints); err != nil { // ignore bad transactions diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index c776bec06c..a80e6dd7cd 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/libs/clist" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/libs/reservoir" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/types" @@ -161,6 +162,9 @@ type txStore struct { // It is used for gossip and has to be stable - we cannot afford removing and reinserting transactions to this list, // because it would cause them to be regossiped. readyTxs *clist.CList[types.Tx] + // Sampler of priorites of all inserted READY txs. + // Used by TxMempool to damp re-gossiping of transactions. + priorityReservoir *reservoir.Sampler[int64] } func NewTxStore(cfg *Config, app *proxy.Proxy, metrics *Metrics) *txStore { @@ -177,12 +181,13 @@ func NewTxStore(cfg *Config, app *proxy.Proxy, metrics *Metrics) *txStore { failedTxs: newLRUTxCache(cfg.CacheSize, maxCacheKeySize), } return &txStore{ - config: cfg, - app: app, - metrics: metrics, - inner: utils.NewRWMutex(inner), - readyTxs: clist.New[types.Tx](), - state: inner.state.Subscribe(), + config: cfg, + app: app, + metrics: metrics, + inner: utils.NewRWMutex(inner), + state: inner.state.Subscribe(), + readyTxs: clist.New[types.Tx](), + priorityReservoir: reservoir.New[int64](cfg.DropPriorityReservoirSize, cfg.DropPriorityThreshold, nil), // Use non-deterministic RNG } } @@ -405,6 +410,9 @@ func (s *txStore) Insert(wtx *WrappedTx) error { if err := s.insert(inner, wtx); err != nil { return err } + if inner.isReady(wtx) { + s.priorityReservoir.Add(wtx.priority) + } if total := inner.state.Load().total; !total.LessEqual(&inner.hardLimit) { s.compact(inner, false) if _, ok := inner.byHash[wtx.Hash()]; !ok { From 91eb256232b9516d0e1069461f85e42f5c675e3b Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 20:14:41 +0200 Subject: [PATCH 053/100] eliminated deviation from main --- sei-tendermint/internal/mempool/mempool.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 837186a753..6e6988f3c0 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -345,7 +345,6 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response } res, err := txmp.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{Tx: tx}) if err != nil || !res.IsOK() { - txmp.txStore.CachePush(hTx.Hash()) txmp.metrics.NumberOfFailedCheckTxs.Add(1) txmp.metrics.observeCheckTxPriorityDistribution(0, false, "", true) } From 38e5109e5983a5c9d6080fa981f2bf82d4b6859e Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 20:27:26 +0200 Subject: [PATCH 054/100] test triggering compaction in insert --- sei-tendermint/internal/mempool/tx.go | 8 +++-- sei-tendermint/internal/mempool/tx_test.go | 39 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index a80e6dd7cd..d4551ce655 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -214,9 +214,11 @@ func (s *txStore) CacheHas(txHash types.TxHash) bool { // Pushes a tx to cache, effectively blocking it from being inserted. func (s *txStore) CachePush(txHash types.TxHash) { - for inner := range s.inner.Lock() { - inner.cache.Push(txHash) - s.metrics.CacheSize.Set(float64(inner.cache.Size())) + if s.config.KeepInvalidTxsInCache { + for inner := range s.inner.Lock() { + inner.cache.Push(txHash) + s.metrics.CacheSize.Set(float64(inner.cache.Size())) + } } } diff --git a/sei-tendermint/internal/mempool/tx_test.go b/sei-tendermint/internal/mempool/tx_test.go index 7fbc9d2722..064d9a2aac 100644 --- a/sei-tendermint/internal/mempool/tx_test.go +++ b/sei-tendermint/internal/mempool/tx_test.go @@ -1,6 +1,7 @@ package mempool import ( + "errors" "fmt" "math/big" "testing" @@ -705,3 +706,41 @@ func TestTxStore_ReplacesPendingTxByHigherPriority(t *testing.T) { require.True(t, ok) require.Equal(t, pendingReplacement.Tx(), got) } + +func TestTxStore_InsertCompactionKeepsReadyListInSync(t *testing.T) { + rng := utils.TestRng() + cfg := TestConfig() + cfg.Size = 50 + cfg.PendingSize = 0 + + app := newEVMNonceApp() + txStore := NewTxStore(cfg, proxy.New(app, proxy.NopMetrics()), NopMetrics()) + inserted := map[types.TxHash]*WrappedTx{} + + for range 20 * cfg.Size { + address := common.BytesToAddress(utils.GenBytes(rng, common.AddressLength)) + wtx := makeEvmTxForTest(rng, address, 0, rng.Int63(), 0) + inserted[wtx.Hash()] = wtx + + err := txStore.Insert(wtx) + require.True(t, err == nil || errors.Is(err, errMempoolFull), "unexpected insert error: %v", err) + + expected := make([]*WrappedTx, 0, txStore.State().total.count) + for txHash, candidate := range inserted { + if tx, ok := txStore.ByHash(txHash); ok { + require.Equal(t, candidate.Tx(), tx) + expected = append(expected, candidate) + } + } + + ready := txStore.ReadyTxs() + require.Equal(t, txStore.State().total.count, txStore.State().ready.count) + require.ElementsMatch(t, toTxs(expected), toTxs(ready)) + + var listed types.Txs + for el := txStore.readyTxs.Front(); el != nil; el = el.Next() { + listed = append(listed, el.Value()) + } + require.ElementsMatch(t, toTxs(expected), listed) + } +} From c56c2cca4a46d388020b8d6b34b861f890f3c120 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 20:31:04 +0200 Subject: [PATCH 055/100] comments --- sei-tendermint/internal/libs/clist/clist.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/sei-tendermint/internal/libs/clist/clist.go b/sei-tendermint/internal/libs/clist/clist.go index f74c586d37..80417f618f 100644 --- a/sei-tendermint/internal/libs/clist/clist.go +++ b/sei-tendermint/internal/libs/clist/clist.go @@ -6,7 +6,6 @@ The purpose of CList is to provide a goroutine-safe linked-list. This list can be traversed concurrently by any number of goroutines. However, removed CElements cannot be added back. NOTE: Not all methods of container/list are (yet) implemented. -NOTE: Removed elements need to DetachPrev or DetachNext consistently to ensure garbage collection of removed elements. */ @@ -241,7 +240,6 @@ func (l *CList[T]) PushBack(v T) *CElement[T] { return e } -// CONTRACT: Caller must call e.DetachPrev() and/or e.DetachNext() to avoid memory leaks. // NOTE: As per the contract of CList, removed elements cannot be added back. func (l *CList[T]) Remove(e *CElement[T]) T { l.mtx.Lock() From 7a8da9476ae4b6911b436af066c4d2be865ab4f4 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 21 May 2026 20:56:43 +0200 Subject: [PATCH 056/100] changed utilisation to account all transactions, because gossip dampening should be enabled as soon as we start evicting transactions, no matter what kind --- sei-tendermint/internal/mempool/mempool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 6e6988f3c0..f9ec04da6b 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -254,7 +254,7 @@ func (txmp *TxMempool) Lock() { txmp.mtx.Lock() } func (txmp *TxMempool) Unlock() { txmp.mtx.Unlock() } func (txmp *TxMempool) utilisation() float64 { - return float64(txmp.NumTxsNotPending()) / float64(txmp.config.Size+txmp.config.PendingSize) + return float64(txmp.Size()) / float64(txmp.config.Size+txmp.config.PendingSize) } // WaitForNextTx waits until the next transaction is available for gossip. From ca0dc587dc4a9d61e5deed616072363d69710b61 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 25 May 2026 10:22:12 +0200 Subject: [PATCH 057/100] autobahn mempool draft --- .../internal/autobahn/producer/state.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index 07fff9e134..0534c1fddb 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -50,6 +50,23 @@ type State struct { consensus *consensus.State } +type Mempool struct { + // (addr,nonce) -> txs + // tracking of what is in progress + // on startup + // * read data.State and avail.State from executed until the end (even across gaps) + // * parse all of these transactions + // * consider only our lane blocks (we are guaranteed to have all of our lane blocks) + // * we are interested only in evm nonces - ignore txs with nonces after a gap + // every time execution progresses + // * we check if nonces progressed as expected. + // * if not - just drop all the non-included txs of the given address + // for testnet + // * accept only ready txs + // * don't drop ready txs (unless some tx was unexpectedly dropped) + // * drop over capacity. +} + // NewState constructs a new block producer state. // Returns an error if the current node is NOT a producer. func NewState(cfg *Config, txMempool *mempool.TxMempool, consensus *consensus.State) *State { From de2eda1c4f71552e3dd7b8d381cbc03f32ee1b3f Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 25 May 2026 10:40:02 +0200 Subject: [PATCH 058/100] applied comments --- sei-tendermint/internal/consensus/replay_test.go | 8 ++------ sei-tendermint/internal/mempool/mempool.go | 2 +- sei-tendermint/internal/mempool/tx.go | 6 +++--- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/sei-tendermint/internal/consensus/replay_test.go b/sei-tendermint/internal/consensus/replay_test.go index c0d1107981..c4ff2f4804 100644 --- a/sei-tendermint/internal/consensus/replay_test.go +++ b/sei-tendermint/internal/consensus/replay_test.go @@ -271,8 +271,7 @@ type simulatorTestSuite struct { Commits []*types.Commit CleanupFunc cleanupFunc - Mempool *mempool.TxMempool - Evpool sm.EvidencePool + Evpool sm.EvidencePool } const ( @@ -292,11 +291,8 @@ var modes = []uint{0, 1, 2, 3} func setupSimulator(ctx context.Context, t *testing.T) *simulatorTestSuite { t.Helper() cfg := configSetup(t) - proxyApp := kvstore.NewProxy() - sim := &simulatorTestSuite{ - Mempool: newReplayTxMempool(proxyApp), - Evpool: sm.EmptyEvidencePool{}, + Evpool: sm.EmptyEvidencePool{}, } nPeers := 7 diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index f9ec04da6b..2a46b0ee5a 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -205,7 +205,7 @@ type TxMempool struct { } func (txmp *TxMempool) Size() int { return txmp.txStore.State().total.count } -func (txmp *TxMempool) SizeBytes() uint64 { return txmp.txStore.State().total.bytes } +func (txmp *TxMempool) SizeBytes() uint64 { return txmp.txStore.State().ready.bytes } func (txmp *TxMempool) NumTxsNotPending() int { return txmp.txStore.State().ready.count } func (txmp *TxMempool) BytesNotPending() uint64 { return txmp.txStore.State().ready.bytes } func (txmp *TxMempool) TotalTxsBytesSize() uint64 { return txmp.txStore.State().total.bytes } diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index d4551ce655..264bb6c98e 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -382,12 +382,12 @@ func (inner *txStoreInner) inInclusionOrder() []*WrappedTx { pending = append(pending, wtx) } } + // Cap priority to obtain a linear order of txs per account by nonce. + // NOTE: this precisely emulates the heap behavior described in this functions docstring. + accPrio := make(map[common.Address]int64, len(inner.accounts)) for _, txs := range utils.Slice(ready, pending) { // Sort by nonce. slices.SortFunc(txs, func(a, b *WrappedTx) int { return cmp.Compare(a.EVMNonce(), b.EVMNonce()) }) - // Cap priority to obtain a linear order of txs per account by nonce. - // NOTE: this precisely emulates the heap behavior described in this functions docstring. - accPrio := make(map[common.Address]int64, len(inner.accounts)) txPrio := make(map[*WrappedTx]int64, len(txs)) for _, tx := range txs { if evm, ok := tx.evm.Get(); ok { From a68002a4c905bb7c1a213f38ae0e75e873972712 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 25 May 2026 17:50:27 +0200 Subject: [PATCH 059/100] mempool WIP --- .../internal/autobahn/avail/state.go | 7 +- .../internal/autobahn/consensus/state.go | 12 -- .../internal/autobahn/data/state.go | 34 ++++++ .../internal/autobahn/producer/mempool.go | 104 +++++++++++++++++ .../internal/autobahn/producer/state.go | 109 +++++++----------- sei-tendermint/internal/p2p/giga_router.go | 23 +--- 6 files changed, 190 insertions(+), 99 deletions(-) create mode 100644 sei-tendermint/internal/autobahn/producer/mempool.go diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index b4c122c063..ae08c23657 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -40,6 +40,10 @@ type State struct { persisters persisters } +func (s *State) PublicKey() types.PublicKey { + return s.key.Public() +} + // persisters holds all disk persistence components. Either all are present // (real I/O) or all are no-op (testing). It is a pure I/O struct — all inner // state access goes through State methods. @@ -521,7 +525,8 @@ func (s *State) fullCommitQC(ctx context.Context, n types.RoadIndex) (*types.Ful } // WaitForCapacity waits until the given lane has enough capacity for a new block. -func (s *State) WaitForCapacity(ctx context.Context, lane types.LaneID) error { +func (s *State) WaitForCapacity(ctx context.Context) error { + lane := s.key.Public() for inner, ctrl := range s.inner.Lock() { q := inner.blocks[lane] if err := ctrl.WaitUntil(ctx, func() bool { diff --git a/sei-tendermint/internal/autobahn/consensus/state.go b/sei-tendermint/internal/autobahn/consensus/state.go index 0790ee9548..ed92d9ff03 100644 --- a/sei-tendermint/internal/autobahn/consensus/state.go +++ b/sei-tendermint/internal/autobahn/consensus/state.go @@ -154,18 +154,6 @@ func (s *State) commitQC() utils.AtomicRecv[utils.Option[*types.CommitQC]] { panic("unreachable") } -// WaitForCapacity waits until a new block can be produced by this node. -func (s *State) WaitForCapacity(ctx context.Context) error { - return s.avail.WaitForCapacity(ctx, s.cfg.Key.Public()) -} - -// ProduceBlock produces a new block with the given payload. -// Returns ErrNoCapacity if there is currently no capacity for the next block. -// Run WaitForCapacity before calling ProduceBlock. -func (s *State) ProduceBlock(ctx context.Context, payload *types.Payload) (*types.Signed[*types.LaneProposal], error) { - return s.avail.ProduceBlock(ctx, payload) -} - // PushProposal processes an unverified FullProposal message. func (s *State) PushProposal(ctx context.Context, proposal *types.FullProposal) error { return s.pushProposal(ctx, proposal) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 89b3b537bb..04b0c6a1e7 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -147,6 +147,7 @@ type appProposalWithTimestamp struct { type inner struct { qcs map[types.GlobalBlockNumber]*types.FullCommitQC // [first,nextQC) blocks map[types.GlobalBlockNumber]*types.Block // [first,nextBlock) + subset of [nextBlock,nextQC) + // appProposal[n] contains appProposal block >=n. appProposals map[types.GlobalBlockNumber]appProposalWithTimestamp // [first,nextAppProposal) // blockHashes is a hash → height index mirroring blocks. Maintained @@ -565,6 +566,9 @@ func (s *State) PushAppHash(ctx context.Context, n types.GlobalBlockNumber, hash proposal: proposal, timestamp: t, } + // TODO(gprusak): this will be problematic on restart, + // nextAppProposal should be initiated wrt current application height, + // so that we don't iterate over all blocks in storage on startup. for inner.nextAppProposal <= n { b := inner.blocks[inner.nextAppProposal] latency := t.Sub(b.Payload().CreatedAt()).Seconds() @@ -594,6 +598,36 @@ func (s *State) AppProposal(ctx context.Context, n types.GlobalBlockNumber) (*ty panic("unreachable") } +func (i *inner) nextToExecute(lane types.LaneID) types.BlockNumber { + // TODO(gprusak): decide whether 0 is a good result in this case in general. + if i.first == i.nextAppProposal { return 0 } + n := i.nextAppProposal-1 + r := i.qcs[n].QC().LaneRange(lane) + // TODO: this header can be actually extracted from FullCommitQC, so consider moving all this logic there. + h := i.blocks[n].Header() + x := lane.Compare(h.Lane()) + // NOTE: here we assume the specific ordering of lane blocks in the CommitQC: + // TODO(gprusak): move this logic closer to CommitQC + switch { + case x<0: return r.Next() + case x>0: return r.First() + default: return h.BlockNumber()+1 + } +} + +// Waits until lane block n is executed, returns the next block of this lane to be exectued (>n) +func (s *State) WaitUntilExecuted(ctx context.Context, lane types.LaneID, n types.BlockNumber) (types.BlockNumber,error) { + for inner,ctrl := range s.inner.Lock() { + for { + if next := inner.nextToExecute(lane); n < next { + return next,nil + } + if err:=ctrl.Wait(ctx); err!=nil { return 0,err } + } + } + panic("unreachable") +} + // PruneBefore removes blocks, QCs, and AppProposals before retainFrom. // Blocks at retainFrom and above are kept. Per-block pruning may split // a QC range; this is handled on recovery (NewState skips partial QC prefixes). diff --git a/sei-tendermint/internal/autobahn/producer/mempool.go b/sei-tendermint/internal/autobahn/producer/mempool.go new file mode 100644 index 0000000000..0d3f188a57 --- /dev/null +++ b/sei-tendermint/internal/autobahn/producer/mempool.go @@ -0,0 +1,104 @@ +package producer + +import ( + "context" + "fmt" + "time" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" + "github.com/ethereum/go-ethereum/common" + ttypes "github.com/sei-protocol/sei-chain/sei-tendermint/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" + abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" +) + +type tx struct { + tx ttypes.Tx + hash ttypes.TxHash + gasEstimated int + gasWanted int +} + +type evmTx struct { + *tx + evmNonce uint64 + evmAddress common.Address + seiAddress []byte +} + +type evmAccount struct { + nonce uint64 + txs []*evmTx +} + +// (addr,nonce) -> tx +// tracking of what is in progress +// on startup +// * read data.State and avail.State from executed until the end (even across gaps) +// * parse all of these transactions +// * consider only our lane blocks (we are guaranteed to have all of our lane blocks) +// * we are interested only in evm nonces - ignore txs with nonces after a gap +// every time execution progresses +// * we check if nonces progressed as expected. +// * if not - just drop all the non-included txs of the given address +// for testnet +// * accept only ready txs +// * don't drop ready txs (unless some tx was unexpectedly dropped) +// * drop over capacity. +// TODO: limit the lag between lane head and local execution +// TODO: make sure that we query nonce at height > expected height +// this way our check will be an approximation from below +type Mempool struct { + app *proxy.Proxy + capacity uint64 + size uint64 + cosmosTxs []*tx + // expected evm account states after the given block + // used to ev + blocks queue[types.BlockNumber, map[common.Address]uint64] + evmAccounts map[common.Address]*evmAccount +} + +func NewMempool(app *proxy.Proxy, capacity uint64) *Mempool { + return &Mempool { + capacity: capacity, + evmAccounts: map[common.Address]*evmAccount{}, + } +} + +type ReapLimits struct { + MaxTxs utils.Option[uint64] + MaxBytes utils.Option[int64] // Max total bytes in proto representation. + MaxGasWanted utils.Option[int64] + MaxGasEstimated utils.Option[int64] +} + +func (m *Mempool) EvmNextPendingNonce(addr common.Address) uint64 { + panic("TODO") +} + +func (m *Mempool) Insert(ctx context.Context, tx ttypes.Tx) (*abci.ResponseCheckTx, error) { + panic("TODO") +} + +// Reaps a non-empty set of ready txs. +func (m *Mempool) ReapTxs(ctx context.Context, limits ReapLimits) (*types.Payload, error) { + payloadTxs := make([][]byte, 0, len(txs)) + for _, tx := range txs { + payloadTxs = append(payloadTxs, tx) + } + payload, err := types.PayloadBuilder{ + CreatedAt: time.Now(), + TotalGas: uint64(gasEstimated), // nolint:gosec // always non-negative + Txs: payloadTxs, + }.Build() + if err != nil { + // This should never happen: we construct the payload from correctly sized data. + panic(fmt.Errorf("PayloadBuilder{}.Build(): %w", err)) + } + return payload, nil +} + +func (m *Mempool) MarkExecuted(ctx context.Context, n types.BlockNumber) error { + panic("TODO") +} diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index 0534c1fddb..bef449e44e 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -6,8 +6,8 @@ import ( "time" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/avail" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" "golang.org/x/time/rate" @@ -45,85 +45,45 @@ func (c *Config) MaxGasPerBlockI64() int64 { // State is the block producer state. type State struct { cfg *Config - txMempool *mempool.TxMempool + mempool *Mempool // consensus state to which published blocks will be reported. consensus *consensus.State } - -type Mempool struct { - // (addr,nonce) -> txs - // tracking of what is in progress - // on startup - // * read data.State and avail.State from executed until the end (even across gaps) - // * parse all of these transactions - // * consider only our lane blocks (we are guaranteed to have all of our lane blocks) - // * we are interested only in evm nonces - ignore txs with nonces after a gap - // every time execution progresses - // * we check if nonces progressed as expected. - // * if not - just drop all the non-included txs of the given address - // for testnet - // * accept only ready txs - // * don't drop ready txs (unless some tx was unexpectedly dropped) - // * drop over capacity. -} - // NewState constructs a new block producer state. // Returns an error if the current node is NOT a producer. -func NewState(cfg *Config, txMempool *mempool.TxMempool, consensus *consensus.State) *State { +func NewState(cfg *Config, consensus *consensus.State) *State { return &State{ cfg: cfg, - txMempool: txMempool, + mempool: NewMempool(nil/*TODO*/,cfg.MempoolSize), consensus: consensus, } } -// makePayload constructs payload for the next produced block. +func (s *State) MarkExecuted(ctx context.Context, h *types.BlockHeader) error { + // Producer only cares about executed lane blocks, + // at which it verifies nonce progress. + if h.Lane()!=s.consensus.Avail().PublicKey() { + return nil + } + return s.mempool.MarkExecuted(ctx, h.BlockNumber()) +} + +// nextPayload constructs payload for the next produced block. // It waits for any transactions OR until `cfg.BlockInterval` passes. -func (s *State) makePayload(ctx context.Context) (*types.Payload, error) { +func (s *State) nextPayload(ctx context.Context) (*types.Payload, error) { // Wait for transactions. We give up and produce an empty block if mempool is empty for // cfg.BlockInterval. - _ = utils.WithTimeout(ctx, s.cfg.BlockInterval, func(ctx context.Context) error { - return s.txMempool.WaitForTxs(ctx) - }) - // If the context has been cancelled though, we just fail. - if err := ctx.Err(); err != nil { - return nil, err + if s.cfg.AllowEmptyBlocks { + var cancel context.CancelFunc + ctx,cancel = context.WithTimeout(ctx, s.cfg.BlockInterval) + defer cancel() } - - txs, gasEstimated := s.txMempool.ReapTxs(mempool.ReapLimits{ + return s.mempool.ReapTxs(ctx, ReapLimits{ MaxTxs: utils.Some(min(types.MaxTxsPerBlock, s.cfg.maxTxsPerBlock())), MaxBytes: utils.Some(utils.Clamp[int64](types.MaxTxsBytesPerBlock)), MaxGasWanted: utils.Some(s.cfg.MaxGasPerBlockI64()), MaxGasEstimated: utils.Some(s.cfg.MaxGasPerBlockI64()), - }, true) - payloadTxs := make([][]byte, 0, len(txs)) - for _, tx := range txs { - payloadTxs = append(payloadTxs, tx) - } - payload, err := types.PayloadBuilder{ - CreatedAt: time.Now(), - TotalGas: uint64(gasEstimated), // nolint:gosec // always non-negative - Txs: payloadTxs, - }.Build() - // This should never happen: we construct the payload from correctly sized data. - if err != nil { - panic(fmt.Errorf("PayloadBuilder{}.Build(): %w", err)) - } - return payload, nil -} - -// nextPayload constructs the payload for the next block. -// Wrapper of makePayload which ensures that the block is not empty (if required). -func (s *State) nextPayload(ctx context.Context) (*types.Payload, error) { - for { - payload, err := s.makePayload(ctx) - if err != nil { - return nil, err - } - if len(payload.Txs()) > 0 || s.cfg.AllowEmptyBlocks { - return payload, nil - } - } + }) } // Run runs the background tasks of the producer state. @@ -137,15 +97,34 @@ func (s *State) Run(ctx context.Context) error { burst = int(l + s.cfg.MaxTxsPerBlock) // nolint:gosec } limiter := rate.NewLimiter(limit, burst) - for { - if err := s.consensus.WaitForCapacity(ctx); err != nil { - return fmt.Errorf("s.Data().WaitForCapacity(): %w", err) + dataState := s.consensus.Data() + availState := s.consensus.Avail() + lane := availState.PublicKey() + + // TODO: this variable can be property of the mempool - initial cutoff of the evm snapshot queue + nextToExecute := utils.NewAtomicSend(types.BlockNumber(0)) + scope.Spawn(func() error { + for { + n,err := dataState.WaitUntilExecuted(ctx,lane,nextToExecute.Load()) + if err!=nil { return err } + s.mempool.Update(n) + nextToExecute.Store(n) + } + }) + for { + n := availState.NextBlock(lane) + if _,err := nextToExecute.Wait(ctx, func(next types.BlockNumber) bool { return next + avail.BlocksPerLane > n }); err!=nil { + return err + } + // TODO: we should block pruning of dataState on AppQC as well, in which case WaitForCapacity and previous check would be both based on dataState. + if err := availState.WaitForCapacity(ctx); err != nil { + return fmt.Errorf("s.consensus.Avail().WaitForCapacity(): %w", err) } payload, err := s.nextPayload(ctx) if err != nil { return fmt.Errorf("s.nextPayload(): %w", err) } - if _, err := s.consensus.ProduceBlock(ctx, payload); err != nil { + if _, err := availState.ProduceBlock(ctx, payload); err != nil { return fmt.Errorf("s.Data().PushBlock(): %w", err) } if err := limiter.WaitN(ctx, len(payload.Txs())); err != nil { diff --git a/sei-tendermint/internal/p2p/giga_router.go b/sei-tendermint/internal/p2p/giga_router.go index 1eed9d2bb4..6ec36b0027 100644 --- a/sei-tendermint/internal/p2p/giga_router.go +++ b/sei-tendermint/internal/p2p/giga_router.go @@ -283,31 +283,12 @@ func (r *GigaRouter) executeBlock(ctx context.Context, b *atypes.GlobalBlock) (* if err != nil { return nil, fmt.Errorf("r.cfg.App.FinalizeBlock(): %w", err) } - if err := r.data.PushAppHash(ctx, b.GlobalNumber, resp.AppHash); err != nil { - return nil, fmt.Errorf("r.data.PushAppHash(%v): %w", b.GlobalNumber, err) - } commitResp, err := app.Commit(ctx) if err != nil { return nil, fmt.Errorf("r.cfg.App.Commit(): %w", err) } - blockTxs := make(types.Txs, len(b.Payload.Txs())) - for i, tx := range b.Payload.Txs() { - blockTxs[i] = tx - } - err = r.cfg.TxMempool.Update( - ctx, - int64(b.GlobalNumber), // nolint:gosec // autobahn block numbers fit in int64. - blockTxs, - resp.TxResults, - // TODO: We need the constraints to be fixed per epoch, because we don't know where the lane blocks will be sequenced. - // Therefore we disable constraints for now, until epochs are supported AND - // chain state understands that consensus parameters can change only at the epoch boundary. - mempool.NopTxConstraints(), - // recheck=false; see TxMempool.Update doc for why. - false, - ) - if err != nil { - return nil, fmt.Errorf("r.cfg.TxMempool.Update(%v): %w", b.GlobalNumber, err) + if err := r.data.PushAppHash(ctx, b.GlobalNumber, resp.AppHash); err != nil { + return nil, fmt.Errorf("r.data.PushAppHash(%v): %w", b.GlobalNumber, err) } return commitResp, nil } From 41a6a18d685a61377202976ad6ca4647109c39e5 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 26 May 2026 14:48:52 +0200 Subject: [PATCH 060/100] applied comments --- sei-tendermint/internal/mempool/mempool.go | 2 +- sei-tendermint/internal/mempool/tx.go | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 2a46b0ee5a..7173689064 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -254,7 +254,7 @@ func (txmp *TxMempool) Lock() { txmp.mtx.Lock() } func (txmp *TxMempool) Unlock() { txmp.mtx.Unlock() } func (txmp *TxMempool) utilisation() float64 { - return float64(txmp.Size()) / float64(txmp.config.Size+txmp.config.PendingSize) + return float64(txmp.Size()) / float64(max(txmp.config.Size+txmp.config.PendingSize, 1)) } // WaitForNextTx waits until the next transaction is available for gossip. diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 264bb6c98e..c47d253291 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -27,8 +27,10 @@ type evmAddrNonce struct { Nonce uint64 } +// hashedTx is used to avoid recomputing values derived from tx. type hashedTx struct { - tx types.Tx + tx types.Tx + // derived hash types.TxHash protoSize int64 } @@ -133,8 +135,8 @@ type txStoreInner struct { // * txs which fail Insert() are NOT added to cache and can be reattempted later. // * invalid transactions can be recorded via CachePush. // * txs dropped due to pruning are removed from cache. - // * txs successfully executed are kept in cache to avoid reinsert. - // * txs failed execution are eligible to be reexecuted once (iff config.KeepInvalidTxsInCache). + // * txs successfully executed are kept in cache to avoid reinsert + // * txs failed execution are eligible to be reexecuted once (iff !config.KeepInvalidTxsInCache). cache *lruTxCache // Tracks transactions which already failed execution once // but are eligible for reexecution (not added yet to cache) @@ -382,11 +384,13 @@ func (inner *txStoreInner) inInclusionOrder() []*WrappedTx { pending = append(pending, wtx) } } - // Cap priority to obtain a linear order of txs per account by nonce. - // NOTE: this precisely emulates the heap behavior described in this functions docstring. + // To achieve the desired txs order, we assign a new priority to each transaction: + // new priority of tx = min over old priorities of all txs with lower or equal nonces of this account + // If we just sort all txs by (new priority, nonce) we will obtain the desired ordering. + // To compute the new priority we first sort all txs by nonce. + // We use accPrio to accumulate min priority of all txs of each account occured so far. accPrio := make(map[common.Address]int64, len(inner.accounts)) for _, txs := range utils.Slice(ready, pending) { - // Sort by nonce. slices.SortFunc(txs, func(a, b *WrappedTx) int { return cmp.Compare(a.EVMNonce(), b.EVMNonce()) }) txPrio := make(map[*WrappedTx]int64, len(txs)) for _, tx := range txs { From 6bdb4051a93581c8cf6898a3da7080be20cb3c78 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 26 May 2026 16:48:51 +0200 Subject: [PATCH 061/100] WIP --- sei-tendermint/abci/types/types.go | 2 +- .../internal/autobahn/avail/inner.go | 16 +- .../internal/autobahn/avail/queue.go | 17 +- .../internal/autobahn/avail/state.go | 10 +- .../internal/autobahn/producer/mempool.go | 180 ++++++++++++++---- .../internal/autobahn/producer/state.go | 21 +- 6 files changed, 172 insertions(+), 74 deletions(-) diff --git a/sei-tendermint/abci/types/types.go b/sei-tendermint/abci/types/types.go index d4411ce7b0..3d5125dc77 100644 --- a/sei-tendermint/abci/types/types.go +++ b/sei-tendermint/abci/types/types.go @@ -232,12 +232,12 @@ type ResponseCheckTxV2 struct { *ResponseCheckTx // helper properties for prioritization in mempool + IsEVM bool EVMNonce uint64 // EVM and sei addresses are both derived from the sender's public key. // TODO(gprusak): include just the secp256k1 public key and let the CheckTx caller derive evm/sei address on their own. EVMSenderAddress common.Address SeiSenderAddress []byte - IsEVM bool EVMRequiredBalance *big.Int } diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index fc22dac206..9722315f6b 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -73,7 +73,7 @@ func newInner(c *types.Committee, loaded utils.Option[*loadedAvailState]) (*inne nextBlockToPersist: make(map[types.LaneID]types.BlockNumber, c.Lanes().Len()), persistedBlockStart: make(map[types.LaneID]types.BlockNumber, c.Lanes().Len()), } - i.appVotes.prune(c.FirstBlock()) + i.appVotes.Prune(c.FirstBlock()) l, ok := loaded.Get() if !ok { @@ -105,7 +105,7 @@ func newInner(c *types.Committee, loaded utils.Option[*loadedAvailState]) (*inne if lqc.Index != i.commitQCs.next { return nil, fmt.Errorf("non-contiguous persisted commitQCs: expected %d, got %d", i.commitQCs.next, lqc.Index) } - i.commitQCs.pushBack(lqc.QC) + i.commitQCs.PushBack(lqc.QC) } if i.commitQCs.next > i.commitQCs.first { i.latestCommitQC.Store(utils.Some(i.commitQCs.q[i.commitQCs.next-1])) @@ -133,7 +133,7 @@ func newInner(c *types.Committee, loaded utils.Option[*loadedAvailState]) (*inne } } lastHash = b.Proposal.Msg().Block().Header().Hash() - q.pushBack(b.Proposal) + q.PushBack(b.Proposal) } if q.next > q.first { i.nextBlockToPersist[lane] = q.next @@ -163,15 +163,15 @@ func (i *inner) prune(c *types.Committee, appQC *types.AppQC, commitQC *types.Co return false, nil } i.latestAppQC = utils.Some(appQC) - i.commitQCs.prune(idx) + i.commitQCs.Prune(idx) if i.commitQCs.next == idx { - i.commitQCs.pushBack(commitQC) + i.commitQCs.PushBack(commitQC) } - i.appVotes.prune(commitQC.GlobalRange(c).First) + i.appVotes.Prune(commitQC.GlobalRange(c).First) for lane := range i.votes { lr := commitQC.LaneRange(lane) - i.votes[lr.Lane()].prune(lr.First()) - i.blocks[lr.Lane()].prune(lr.First()) + i.votes[lr.Lane()].Prune(lr.First()) + i.blocks[lr.Lane()].Prune(lr.First()) if i.nextBlockToPersist[lr.Lane()] < lr.First() { i.nextBlockToPersist[lr.Lane()] = lr.First() } diff --git a/sei-tendermint/internal/autobahn/avail/queue.go b/sei-tendermint/internal/autobahn/avail/queue.go index 1b4eaaded5..072a7f5540 100644 --- a/sei-tendermint/internal/autobahn/avail/queue.go +++ b/sei-tendermint/internal/autobahn/avail/queue.go @@ -1,8 +1,12 @@ package avail +import ( + "golang.org/x/exp/constraints" +) + // queue is a collection of objects of type T, indexed by type I in range [first,next). // Supports pushing new items to the back and popping items from the front. -type queue[I ~uint64, T any] struct { +type queue[I constraints.Integer, T any] struct { q map[I]T first I next I @@ -12,16 +16,17 @@ func newQueue[I ~uint64, T any]() *queue[I, T] { return &queue[I, T]{q: map[I]T{}, first: 0, next: 0} } -func (q *queue[I, T]) Len() uint64 { - return uint64(q.next) - uint64(q.first) -} +func (q *queue[I, T]) First() I { return q.first } +func (q *queue[I, T]) Next() I { return q.next } +func (q *queue[I, T]) Get(i I) T { return q.q[i] } +func (q *queue[I, T]) Len() I { return q.next - q.first } -func (q *queue[I, T]) pushBack(t T) { +func (q *queue[I, T]) PushBack(t T) { q.q[q.next] = t q.next += 1 } -func (q *queue[I, T]) prune(newFirst I) { +func (q *queue[I, T]) Prune(newFirst I) { if newFirst <= q.first { return } diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index ae08c23657..51cc473c84 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -268,7 +268,7 @@ func (s *State) PushCommitQC(ctx context.Context, qc *types.CommitQC) error { if idx != inner.commitQCs.next { return nil } - inner.commitQCs.pushBack(qc) + inner.commitQCs.PushBack(qc) // The persist goroutine publishes latestCommitQC after writing to disk // (or immediately for no-op persisters), so consensus won't advance // until the CommitQC is durable. @@ -302,7 +302,7 @@ func (s *State) PushAppVote(ctx context.Context, v *types.Signed[*types.AppVote] n := v.Msg().Proposal().GlobalNumber() q := inner.appVotes for q.next <= n { - q.pushBack(newAppVotes()) + q.PushBack(newAppVotes()) } appQC, ok := q.q[n].pushVote(s.data.Committee(), v) if !ok { @@ -432,7 +432,7 @@ func (s *State) PushBlock(ctx context.Context, p *types.Signed[*types.LanePropos return nil } } - q.pushBack(p) + q.PushBack(p) ctrl.Updated() } return nil @@ -463,7 +463,7 @@ func (s *State) PushVote(ctx context.Context, vote *types.Signed[*types.LaneVote return nil } for q.next <= h.BlockNumber() { - q.pushBack(newBlockVotes()) + q.PushBack(newBlockVotes()) } if _, ok := q.q[h.BlockNumber()].pushVote(s.data.Committee(), vote); ok { ctrl.Updated() @@ -592,7 +592,7 @@ func (s *State) produceBlock(ctx context.Context, key types.SecretKey, payload * parent = q.q[q.next-1].Msg().Block().Header().Hash() } result = types.Sign(key, types.NewLaneProposal(types.NewBlock(lane, q.next, parent, payload))) - q.pushBack(result) + q.PushBack(result) ctrl.Updated() } return result, nil diff --git a/sei-tendermint/internal/autobahn/producer/mempool.go b/sei-tendermint/internal/autobahn/producer/mempool.go index 0d3f188a57..76557515d9 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool.go +++ b/sei-tendermint/internal/autobahn/producer/mempool.go @@ -2,33 +2,86 @@ package producer import ( "context" + "errors" "fmt" "time" + "golang.org/x/exp/constraints" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" "github.com/ethereum/go-ethereum/common" - ttypes "github.com/sei-protocol/sei-chain/sei-tendermint/types" + tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/types" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" ) -type tx struct { - tx ttypes.Tx - hash ttypes.TxHash - gasEstimated int - gasWanted int +// queue is a collection of objects of type T, indexed by type I in range [first,next). +// Supports pushing new items to the back and popping items from the front. +type queue[I constraints.Integer, T any] struct { + q map[I]T + first I + next I } -type evmTx struct { - *tx - evmNonce uint64 - evmAddress common.Address - seiAddress []byte +func newQueue[I ~uint64, T any]() *queue[I, T] { + return &queue[I, T]{q: map[I]T{}, first: 0, next: 0} +} + +func (q *queue[I, T]) First() I { return q.first } +func (q *queue[I, T]) Next() I { return q.next } +func (q *queue[I, T]) Get(i I) T { return q.q[i] } +func (q *queue[I, T]) Len() I { return q.next - q.first } + +func (q *queue[I, T]) PushBack(i I, t T) { + if q.next <= i { + q.q[i] = t + if q.first == q.next { + q.first = i + } + q.next = i+1 + } +} + +func (q *queue[I, T]) PopFront(i I) (res T, ok bool) { + for q.first < min(i+1,q.next) { + res,ok = q.q[q.first] + delete(q.q,q.first) + q.first += 1 + } + q.next = max(i+1,q.next) + return +} + +type extTx struct { + tx tmtypes.Tx + hash tmtypes.TxHash + gasEstimated int64 + gasWanted int64 } type evmAccount struct { - nonce uint64 - txs []*evmTx + // List of the txs ready to be sequenced. + readyTxs []*extTx + // nonce that the account will have after the readyTxs are executed. + nextNonce uint64 + // Nonces that this account is expected to be at after executing the given block. + // Since autobahn has asynchronous execution, there is no guarantee that the account nonce will + // be incremented at the time of constructing the lane block. + // On the other hand we need to be able to sequence many account txs before the first one is executed. + // To achieve that we track the expected per-account nonces after each block of the local lane. + // If after execution the nonce is below the expectation, it means that execution failed and the + // same will happen with all the subsequent txs (because of the nonce gap). In such a case we + // drop whole account from the mempool, because user needs to submit the txs again. + nonceByBlock queue[types.BlockNumber,uint64] +} + +func (a *evmAccount) IsEmpty() bool { + return len(a.readyTxs)==0 && a.nonceByBlock.Len()==0 +} + +type mempoolInner struct { + count uint64 + cosmosTxs []*extTx + evmAccounts map[common.Address]*evmAccount } // (addr,nonce) -> tx @@ -45,44 +98,95 @@ type evmAccount struct { // * accept only ready txs // * don't drop ready txs (unless some tx was unexpectedly dropped) // * drop over capacity. -// TODO: limit the lag between lane head and local execution // TODO: make sure that we query nonce at height > expected height // this way our check will be an approximation from below type Mempool struct { app *proxy.Proxy - capacity uint64 - size uint64 - cosmosTxs []*tx - // expected evm account states after the given block - // used to ev - blocks queue[types.BlockNumber, map[common.Address]uint64] - evmAccounts map[common.Address]*evmAccount + cfg *Config + maxCount uint64 + maxBytes uint64 + inner utils.Watch[*mempoolInner] } -func NewMempool(app *proxy.Proxy, capacity uint64) *Mempool { +func NewMempool(cfg *Config, app *proxy.Proxy) *Mempool { return &Mempool { - capacity: capacity, - evmAccounts: map[common.Address]*evmAccount{}, + app: app, + cfg: cfg, + inner: utils.NewWatch(&mempoolInner { + evmAccounts: map[common.Address]*evmAccount{}, + }), } } type ReapLimits struct { - MaxTxs utils.Option[uint64] - MaxBytes utils.Option[int64] // Max total bytes in proto representation. - MaxGasWanted utils.Option[int64] - MaxGasEstimated utils.Option[int64] + MaxTxs uint64 + MaxBytes uint64 + MaxGasWanted uint64 } func (m *Mempool) EvmNextPendingNonce(addr common.Address) uint64 { - panic("TODO") + for inner := range m.inner.Lock() { + if acc,ok := inner.evmAccounts[addr]; ok { + return acc.nextNonce + } + } + return m.app.EvmNonce(addr) } -func (m *Mempool) Insert(ctx context.Context, tx ttypes.Tx) (*abci.ResponseCheckTx, error) { - panic("TODO") +var errTooLarge = errors.New("transaction too large") +var errFull = errors.New("mempool is full") + +func (m *Mempool) Insert(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseCheckTx, error) { + if uint64(len(tx)) > types.MaxTxsBytesPerBlock { + return nil, errTooLarge + } + resp, err := m.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{Tx: tx}) + if err!=nil { return nil, err } + if !resp.IsOK() { return resp.ResponseCheckTx, nil } + etx := &extTx { + tx: tx, + hash: tx.Hash(), + gasEstimated: resp.GasEstimated, + gasWanted: resp.GasWanted, + } + if etx.gasEstimated < minTxGas || etx.gasEstimated > etx.gasWanted { + etx.gasEstimated = etx.gasWanted + } + for inner,ctrl := range m.inner.Lock() { + if inner.count+1 > m.cfg.MempoolSize { return nil, errFull } + // TODO: byte capacity + if resp.IsEVM { + addr := resp.EVMSenderAddress + acc,ok := inner.evmAccounts[addr] + if !ok { + acc = &evmAccount { nextNonce: m.app.EvmNonce(addr) } + } + if acc.nextNonce != resp.EVMNonce { + return nil, fmt.Errorf("bad nonce: got %v, want %v", resp.EVMNonce, acc.nextNonce) + } + acc.readyTxs = append(acc.readyTxs,etx) + acc.nextNonce += 1 + inner.evmAccounts[addr] = acc + } else { + inner.cosmosTxs = append(inner.cosmosTxs,etx) + } + inner.count += 1 + ctrl.Updated() + return resp.ResponseCheckTx,nil + } + panic("unreachable") } -// Reaps a non-empty set of ready txs. -func (m *Mempool) ReapTxs(ctx context.Context, limits ReapLimits) (*types.Payload, error) { +// Reaps a non-empty set of ready txs for constructing block n. +func (m *Mempool) ReapTxs(ctx context.Context, n types.BlockNumber) (*types.Payload, error) { + limits := ReapLimits{ + MaxTxs: m.cfg.maxTxsPerBlock(), + MaxBytes: types.MaxTxsBytesPerBlock, + MaxGasWanted: m.cfg.MaxGasPerBlock, + } + for inner,ctrl := range m.inner.Lock() { + if err := ctrl.WaitUntil(ctx, func() bool { return inner.count > 0 }); err!=nil { return nil,err } + } payloadTxs := make([][]byte, 0, len(txs)) for _, tx := range txs { payloadTxs = append(payloadTxs, tx) @@ -99,6 +203,12 @@ func (m *Mempool) ReapTxs(ctx context.Context, limits ReapLimits) (*types.Payloa return payload, nil } -func (m *Mempool) MarkExecuted(ctx context.Context, n types.BlockNumber) error { - panic("TODO") +func (m *Mempool) MarkExecuted(n types.BlockNumber) { + for inner := range m.inner.Lock() { + for addr,acc := range inner.evmAccounts { + if wantMin,ok := acc.nonceByBlock.PopFront(n); acc.IsEmpty() || (ok && m.app.EvmNonce(addr) < wantMin) { + delete(inner.evmAccounts,addr) + } + } + } } diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index bef449e44e..566f7c1e2f 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -27,19 +27,7 @@ type Config struct { const minTxGas = 21000 func (c *Config) maxTxsPerBlock() uint64 { - return min(c.MaxTxsPerBlock, c.MaxGasPerBlock/minTxGas) -} - -// MaxGasPerBlockI64 returns MaxGasPerBlock clamped to the int64 range. -// Config validation only enforces > 0 (sei-tendermint/config/autobahn.go), -// so a misconfigured chain with a value above math.MaxInt64 can't silently -// overflow when consumed by APIs that take int64 (the mempool's ReapLimits, -// the RPC layer's ConsensusParamUpdates.Block.MaxGas). Centralizing the -// clamp here means callers pick this up by name instead of repeating -// utils.Clamp[int64] at every site, and any future change to the clamp -// rule (or the underlying field type) lives in one place. -func (c *Config) MaxGasPerBlockI64() int64 { - return utils.Clamp[int64](c.MaxGasPerBlock) + return min(types.MaxTxsPerBlock, c.MaxTxsPerBlock, c.MaxGasPerBlock/minTxGas) } // State is the block producer state. @@ -78,12 +66,7 @@ func (s *State) nextPayload(ctx context.Context) (*types.Payload, error) { ctx,cancel = context.WithTimeout(ctx, s.cfg.BlockInterval) defer cancel() } - return s.mempool.ReapTxs(ctx, ReapLimits{ - MaxTxs: utils.Some(min(types.MaxTxsPerBlock, s.cfg.maxTxsPerBlock())), - MaxBytes: utils.Some(utils.Clamp[int64](types.MaxTxsBytesPerBlock)), - MaxGasWanted: utils.Some(s.cfg.MaxGasPerBlockI64()), - MaxGasEstimated: utils.Some(s.cfg.MaxGasPerBlockI64()), - }) + return s.mempool.ReapTxs(ctx) } // Run runs the background tasks of the producer state. From abd463ab9de0e1effcef91b00f28407f91d8e061 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 26 May 2026 16:57:47 +0200 Subject: [PATCH 062/100] flake fix: --- sei-tendermint/internal/mempool/tx.go | 2 +- sei-tendermint/internal/p2p/router.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index c47d253291..446a673c32 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -388,7 +388,7 @@ func (inner *txStoreInner) inInclusionOrder() []*WrappedTx { // new priority of tx = min over old priorities of all txs with lower or equal nonces of this account // If we just sort all txs by (new priority, nonce) we will obtain the desired ordering. // To compute the new priority we first sort all txs by nonce. - // We use accPrio to accumulate min priority of all txs of each account occured so far. + // We use accPrio to accumulate min priority of all txs of each account occurred so far. accPrio := make(map[common.Address]int64, len(inner.accounts)) for _, txs := range utils.Slice(ready, pending) { slices.SortFunc(txs, func(a, b *WrappedTx) int { return cmp.Compare(a.EVMNonce(), b.EVMNonce()) }) diff --git a/sei-tendermint/internal/p2p/router.go b/sei-tendermint/internal/p2p/router.go index 7c42e3f0a1..5f6a1797d9 100644 --- a/sei-tendermint/internal/p2p/router.go +++ b/sei-tendermint/internal/p2p/router.go @@ -174,6 +174,7 @@ func (r *Router) acceptPeersRoutine(ctx context.Context) error { if err != nil { return fmt.Errorf("net.Listen(): %w", err) } + defer listener.Close() close(r.started) // signal that we are listening connTracker := newConnTracker( From 2465a33f5abe06e9c63db7f82fea43aed5a80378 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 26 May 2026 17:07:13 +0200 Subject: [PATCH 063/100] lint --- sei-tendermint/internal/p2p/router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sei-tendermint/internal/p2p/router.go b/sei-tendermint/internal/p2p/router.go index 5f6a1797d9..fba795598a 100644 --- a/sei-tendermint/internal/p2p/router.go +++ b/sei-tendermint/internal/p2p/router.go @@ -174,7 +174,7 @@ func (r *Router) acceptPeersRoutine(ctx context.Context) error { if err != nil { return fmt.Errorf("net.Listen(): %w", err) } - defer listener.Close() + defer func() { _ = listener.Close() }() close(r.started) // signal that we are listening connTracker := newConnTracker( From 353af1925f863b4586e5ad403a27a48c45a7c79c Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 27 May 2026 10:00:04 +0200 Subject: [PATCH 064/100] before async mutex --- .../internal/autobahn/autobahn.proto | 5 +- .../internal/autobahn/producer/mempool.go | 183 +++++++----------- .../internal/autobahn/producer/state.go | 93 +++++---- .../internal/autobahn/types/block.go | 24 --- 4 files changed, 132 insertions(+), 173 deletions(-) diff --git a/sei-tendermint/internal/autobahn/autobahn.proto b/sei-tendermint/internal/autobahn/autobahn.proto index c64fdc119a..0145ce0b3c 100644 --- a/sei-tendermint/internal/autobahn/autobahn.proto +++ b/sei-tendermint/internal/autobahn/autobahn.proto @@ -74,12 +74,11 @@ message BlockHeader { } message Payload { + reserved "edge_count ", "coinbase", "basefee"; + reserved 3,4,5; option (hashable.hashable) = true; optional Timestamp created_at = 1; // required optional uint64 total_gas = 2; // required - optional int64 edge_count = 3; // required - optional bytes coinbase = 4; // required - optional int64 basefee = 5; // required repeated bytes txs = 6; } diff --git a/sei-tendermint/internal/autobahn/producer/mempool.go b/sei-tendermint/internal/autobahn/producer/mempool.go index 76557515d9..491622c3bf 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool.go +++ b/sei-tendermint/internal/autobahn/producer/mempool.go @@ -10,7 +10,6 @@ import ( "github.com/ethereum/go-ethereum/common" tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/types" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" ) @@ -26,9 +25,6 @@ func newQueue[I ~uint64, T any]() *queue[I, T] { return &queue[I, T]{q: map[I]T{}, first: 0, next: 0} } -func (q *queue[I, T]) First() I { return q.first } -func (q *queue[I, T]) Next() I { return q.next } -func (q *queue[I, T]) Get(i I) T { return q.q[i] } func (q *queue[I, T]) Len() I { return q.next - q.first } func (q *queue[I, T]) PushBack(i I, t T) { @@ -41,26 +37,18 @@ func (q *queue[I, T]) PushBack(i I, t T) { } } -func (q *queue[I, T]) PopFront(i I) (res T, ok bool) { - for q.first < min(i+1,q.next) { +func (q *queue[I, T]) Prune(i I) (res T, ok bool) { + for q.first < min(i,q.next) { res,ok = q.q[q.first] delete(q.q,q.first) q.first += 1 } - q.next = max(i+1,q.next) + q.first = max(q.first,i) + q.next = max(q.next,i) return } -type extTx struct { - tx tmtypes.Tx - hash tmtypes.TxHash - gasEstimated int64 - gasWanted int64 -} - type evmAccount struct { - // List of the txs ready to be sequenced. - readyTxs []*extTx // nonce that the account will have after the readyTxs are executed. nextNonce uint64 // Nonces that this account is expected to be at after executing the given block. @@ -74,14 +62,43 @@ type evmAccount struct { nonceByBlock queue[types.BlockNumber,uint64] } -func (a *evmAccount) IsEmpty() bool { - return len(a.readyTxs)==0 && a.nonceByBlock.Len()==0 +type mempool struct { + gasEstimated uint64 + gasWanted uint64 + sizeBytes uint64 + txs [][]byte + nextPayload utils.Option[*types.Payload] + + nextToProduce types.BlockNumber + nextToExecute types.BlockNumber + evmAccounts map[common.Address]*evmAccount } -type mempoolInner struct { - count uint64 - cosmosTxs []*extTx - evmAccounts map[common.Address]*evmAccount +func (m *mempool) buildPayload() { + if m.nextPayload.IsPresent() { + return + } + // Snapshot evm state. + for _,acc := range m.evmAccounts { + acc.nonceByBlock.PushBack(m.nextToProduce,acc.nextNonce) + } + m.nextToProduce += 1 + // Construct a payload. + payload, err := types.PayloadBuilder{ + CreatedAt: time.Now(), + TotalGas: uint64(m.gasEstimated), // nolint:gosec // always non-negative + Txs: m.txs, + }.Build() + if err != nil { + // This should never happen: we construct the payload from correctly sized data. + panic(fmt.Errorf("PayloadBuilder{}.Build(): %w", err)) + } + m.nextPayload = utils.Some(payload) + // Clear the mempool. + m.txs = nil + m.gasEstimated = 0 + m.gasWanted = 0 + m.sizeBytes = 0 } // (addr,nonce) -> tx @@ -100,115 +117,65 @@ type mempoolInner struct { // * drop over capacity. // TODO: make sure that we query nonce at height > expected height // this way our check will be an approximation from below -type Mempool struct { - app *proxy.Proxy - cfg *Config - maxCount uint64 - maxBytes uint64 - inner utils.Watch[*mempoolInner] -} - -func NewMempool(cfg *Config, app *proxy.Proxy) *Mempool { - return &Mempool { - app: app, - cfg: cfg, - inner: utils.NewWatch(&mempoolInner { - evmAccounts: map[common.Address]*evmAccount{}, - }), - } -} -type ReapLimits struct { - MaxTxs uint64 - MaxBytes uint64 - MaxGasWanted uint64 -} - -func (m *Mempool) EvmNextPendingNonce(addr common.Address) uint64 { - for inner := range m.inner.Lock() { - if acc,ok := inner.evmAccounts[addr]; ok { +func (s *State) EvmNextPendingNonce(addr common.Address) uint64 { + for m := range s.mempool.Lock() { + if acc,ok := m.evmAccounts[addr]; ok { return acc.nextNonce } } - return m.app.EvmNonce(addr) + return s.app.EvmNonce(addr) } var errTooLarge = errors.New("transaction too large") var errFull = errors.New("mempool is full") -func (m *Mempool) Insert(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseCheckTx, error) { +// Blocking insert. +func (s *State) Insert(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseCheckTx, error) { if uint64(len(tx)) > types.MaxTxsBytesPerBlock { return nil, errTooLarge } - resp, err := m.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{Tx: tx}) + resp, err := s.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{Tx: tx}) if err!=nil { return nil, err } - if !resp.IsOK() { return resp.ResponseCheckTx, nil } - etx := &extTx { - tx: tx, - hash: tx.Hash(), - gasEstimated: resp.GasEstimated, - gasWanted: resp.GasWanted, - } - if etx.gasEstimated < minTxGas || etx.gasEstimated > etx.gasWanted { - etx.gasEstimated = etx.gasWanted - } - for inner,ctrl := range m.inner.Lock() { - if inner.count+1 > m.cfg.MempoolSize { return nil, errFull } - // TODO: byte capacity + if !resp.IsOK() { return resp.ResponseCheckTx, nil } + gasWanted := utils.Clamp[uint64](resp.GasWanted) + if gasWanted > s.cfg.MaxGasPerBlock { return nil, errTooLarge } + + for m,ctrl := range s.mempool.Lock() { + if err:=ctrl.WaitUntil(ctx, func() bool { return !m.nextPayload.IsPresent() }); err!=nil { + return nil, err + } if resp.IsEVM { addr := resp.EVMSenderAddress - acc,ok := inner.evmAccounts[addr] + acc,ok := m.evmAccounts[addr] if !ok { - acc = &evmAccount { nextNonce: m.app.EvmNonce(addr) } + acc = &evmAccount { nextNonce: s.app.EvmNonce(addr) } } if acc.nextNonce != resp.EVMNonce { return nil, fmt.Errorf("bad nonce: got %v, want %v", resp.EVMNonce, acc.nextNonce) } - acc.readyTxs = append(acc.readyTxs,etx) acc.nextNonce += 1 - inner.evmAccounts[addr] = acc - } else { - inner.cosmosTxs = append(inner.cosmosTxs,etx) + m.evmAccounts[addr] = acc + } + // If any limit would be exceeded, then construct a payload. + ok := uint64(len(m.txs)) + 1 <= s.cfg.maxTxsPerBlock() + ok = ok && m.sizeBytes + uint64(len(tx)) <= types.MaxTxsBytesPerBlock + ok = ok && m.gasWanted + gasWanted <= s.cfg.MaxGasPerBlock + if !ok { + m.buildPayload() + ctrl.Updated() } - inner.count += 1 - ctrl.Updated() - return resp.ResponseCheckTx,nil - } - panic("unreachable") -} - -// Reaps a non-empty set of ready txs for constructing block n. -func (m *Mempool) ReapTxs(ctx context.Context, n types.BlockNumber) (*types.Payload, error) { - limits := ReapLimits{ - MaxTxs: m.cfg.maxTxsPerBlock(), - MaxBytes: types.MaxTxsBytesPerBlock, - MaxGasWanted: m.cfg.MaxGasPerBlock, - } - for inner,ctrl := range m.inner.Lock() { - if err := ctrl.WaitUntil(ctx, func() bool { return inner.count > 0 }); err!=nil { return nil,err } - } - payloadTxs := make([][]byte, 0, len(txs)) - for _, tx := range txs { - payloadTxs = append(payloadTxs, tx) - } - payload, err := types.PayloadBuilder{ - CreatedAt: time.Now(), - TotalGas: uint64(gasEstimated), // nolint:gosec // always non-negative - Txs: payloadTxs, - }.Build() - if err != nil { - // This should never happen: we construct the payload from correctly sized data. - panic(fmt.Errorf("PayloadBuilder{}.Build(): %w", err)) - } - return payload, nil -} -func (m *Mempool) MarkExecuted(n types.BlockNumber) { - for inner := range m.inner.Lock() { - for addr,acc := range inner.evmAccounts { - if wantMin,ok := acc.nonceByBlock.PopFront(n); acc.IsEmpty() || (ok && m.app.EvmNonce(addr) < wantMin) { - delete(inner.evmAccounts,addr) - } + // Normalize the gas estimate. + gasEstimated := resp.GasEstimated + if gasEstimated < minTxGas || gasEstimated > resp.GasWanted { + gasEstimated = resp.GasWanted } + m.gasEstimated += utils.Clamp[uint64](gasEstimated) + m.gasWanted += utils.Clamp[uint64](resp.GasWanted) + m.sizeBytes += uint64(len(tx)) + m.txs = append(m.txs,tx) + return resp.ResponseCheckTx,nil } + panic("unreachable") } diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index 566f7c1e2f..9bce172093 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -5,11 +5,13 @@ import ( "fmt" "time" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/avail" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" + "github.com/ethereum/go-ethereum/common" "golang.org/x/time/rate" ) @@ -17,23 +19,26 @@ import ( type Config struct { MaxGasPerBlock uint64 MaxTxsPerBlock uint64 - MaxTxsPerSecond utils.Option[uint64] - MempoolSize uint64 + // Delay after which a non-full block can be produced. BlockInterval time.Duration AllowEmptyBlocks bool + // TESTONLY: max rate at which lane is produced. It can be used to do + // benchmarks with stable throughput, in case execution performance degrades + // when overloaded. + MaxTxsPerSecond utils.Option[uint64] } -// minTxGas is the minimum gas cost of an evm tx. const minTxGas = 21000 func (c *Config) maxTxsPerBlock() uint64 { - return min(types.MaxTxsPerBlock, c.MaxTxsPerBlock, c.MaxGasPerBlock/minTxGas) + return min(types.MaxTxsPerBlock, c.MaxTxsPerBlock) } // State is the block producer state. type State struct { - cfg *Config - mempool *Mempool + app *proxy.Proxy + cfg *Config + mempool utils.Watch[*mempool] // consensus state to which published blocks will be reported. consensus *consensus.State } @@ -42,31 +47,24 @@ type State struct { func NewState(cfg *Config, consensus *consensus.State) *State { return &State{ cfg: cfg, - mempool: NewMempool(nil/*TODO*/,cfg.MempoolSize), + mempool: utils.NewWatch(&mempool { + evmAccounts: map[common.Address]*evmAccount{}, + }), consensus: consensus, } } -func (s *State) MarkExecuted(ctx context.Context, h *types.BlockHeader) error { - // Producer only cares about executed lane blocks, - // at which it verifies nonce progress. - if h.Lane()!=s.consensus.Avail().PublicKey() { - return nil - } - return s.mempool.MarkExecuted(ctx, h.BlockNumber()) -} - -// nextPayload constructs payload for the next produced block. -// It waits for any transactions OR until `cfg.BlockInterval` passes. -func (s *State) nextPayload(ctx context.Context) (*types.Payload, error) { - // Wait for transactions. We give up and produce an empty block if mempool is empty for - // cfg.BlockInterval. - if s.cfg.AllowEmptyBlocks { - var cancel context.CancelFunc - ctx,cancel = context.WithTimeout(ctx, s.cfg.BlockInterval) - defer cancel() +func (s *State) setNextToExecute(n types.BlockNumber) { + for m,ctrl := range s.mempool.Lock() { + if n < m.nextToExecute { return } + ctrl.Updated() + m.nextToExecute = n + for addr,acc := range m.evmAccounts { + if wantMin,ok := acc.nonceByBlock.Prune(n); acc.nonceByBlock.Len()==0 || (ok && s.app.EvmNonce(addr) < wantMin) { + delete(m.evmAccounts,addr) + } + } } - return s.mempool.ReapTxs(ctx) } // Run runs the background tasks of the producer state. @@ -83,32 +81,51 @@ func (s *State) Run(ctx context.Context) error { dataState := s.consensus.Data() availState := s.consensus.Avail() lane := availState.PublicKey() + + for m := range s.mempool.Lock() { + m.nextToExecute = 0 + m.nextToProduce = availState.NextBlock(lane) + } - // TODO: this variable can be property of the mempool - initial cutoff of the evm snapshot queue - nextToExecute := utils.NewAtomicSend(types.BlockNumber(0)) scope.Spawn(func() error { + next := types.BlockNumber(0) + var err error for { - n,err := dataState.WaitUntilExecuted(ctx,lane,nextToExecute.Load()) - if err!=nil { return err } - s.mempool.Update(n) - nextToExecute.Store(n) + if next,err = dataState.WaitUntilExecuted(ctx,lane,next); err!=nil { return err } + s.setNextToExecute(next) } }) + lastBlockTime := time.Now() for { - n := availState.NextBlock(lane) if _,err := nextToExecute.Wait(ctx, func(next types.BlockNumber) bool { return next + avail.BlocksPerLane > n }); err!=nil { return err } - // TODO: we should block pruning of dataState on AppQC as well, in which case WaitForCapacity and previous check would be both based on dataState. if err := availState.WaitForCapacity(ctx); err != nil { return fmt.Errorf("s.consensus.Avail().WaitForCapacity(): %w", err) } - payload, err := s.nextPayload(ctx) - if err != nil { - return fmt.Errorf("s.nextPayload(): %w", err) + // Wait until either + // * there is a full proposal in mempool + // * BlockInterval since the last block passed AND (AllowEmptyBlocks OR mempool is non-empty) + for m,ctrl := range s.mempool.Lock() { + // First just wait for full proposal with timeout (first condition) + _ = utils.WithDeadline(ctx, utils.Some(lastBlockTime.Add(s.cfg.BlockInterval)), func(ctx context.Context) error { + return ctrl.WaitUntil(ctx, func() bool { return m.nextPayload.IsPresent() }) + }) + if ctx.Err()!=nil { + return ctx.Err() + } + // Then wait for ANY condition. + if err:=ctrl.WaitUntil(ctx, func() bool { + return m.nextPayload.IsPresent() || s.cfg.AllowEmptyBlocks || len(m.txs) > 0 + }); err!=nil { + return err + } + // Construct the payload unconditionally. + m.buildPayload() } + if _, err := availState.ProduceBlock(ctx, payload); err != nil { - return fmt.Errorf("s.Data().PushBlock(): %w", err) + return fmt.Errorf("availState.ProduceBlock(): %w", err) } if err := limiter.WaitN(ctx, len(payload.Txs())); err != nil { return fmt.Errorf("limiter(): %w", err) diff --git a/sei-tendermint/internal/autobahn/types/block.go b/sei-tendermint/internal/autobahn/types/block.go index 8f39301bb9..58a6eb2041 100644 --- a/sei-tendermint/internal/autobahn/types/block.go +++ b/sei-tendermint/internal/autobahn/types/block.go @@ -160,9 +160,6 @@ type PayloadHash hashable.Hash[*pb.Payload] type PayloadBuilder struct { CreatedAt time.Time TotalGas uint64 - EdgeCount int64 - Coinbase []byte - Basefee int64 Txs [][]byte } @@ -196,15 +193,6 @@ func (p *Payload) CreatedAt() time.Time { return p.p.CreatedAt } // TotalGas . func (p *Payload) TotalGas() uint64 { return p.p.TotalGas } -// EdgeCount . -func (p *Payload) EdgeCount() int64 { return p.p.EdgeCount } - -// Coinbase . -func (p *Payload) Coinbase() []byte { return p.p.Coinbase } - -// Basefee . -func (p *Payload) Basefee() int64 { return p.p.Basefee } - // Txs . func (p *Payload) Txs() [][]byte { return p.p.Txs } @@ -254,9 +242,6 @@ var PayloadConv = protoutils.Conv[*Payload, *pb.Payload]{ return &pb.Payload{ CreatedAt: TimeConv.Encode(p.p.CreatedAt), TotalGas: utils.Alloc(p.p.TotalGas), - EdgeCount: utils.Alloc(p.p.EdgeCount), - Coinbase: p.p.Coinbase, - Basefee: utils.Alloc(p.p.Basefee), Txs: p.p.Txs, } }, @@ -268,18 +253,9 @@ var PayloadConv = protoutils.Conv[*Payload, *pb.Payload]{ if p.TotalGas == nil { return nil, fmt.Errorf("TotalGas: missing") } - if p.EdgeCount == nil { - return nil, fmt.Errorf("EdgeCount: missing") - } - if p.Basefee == nil { - return nil, fmt.Errorf("Basefee: missing") - } return PayloadBuilder{ CreatedAt: createdAt, TotalGas: *p.TotalGas, - EdgeCount: *p.EdgeCount, - Coinbase: p.Coinbase, - Basefee: *p.Basefee, Txs: p.Txs, }.Build() }, From 7b1eed587f70a21b2764d881fa47aeca75ddcd69 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 27 May 2026 12:59:54 +0200 Subject: [PATCH 065/100] code complete --- .../internal/autobahn/autobahn.proto | 2 +- .../internal/autobahn/avail/state.go | 20 +-- .../internal/autobahn/pb/autobahn.pb.go | 43 +---- .../internal/autobahn/producer/mempool.go | 170 +++++++++--------- .../internal/autobahn/producer/state.go | 150 +++++++++------- .../internal/autobahn/types/testonly.go | 3 - .../internal/p2p/conn/secret_connection.go | 23 +-- sei-tendermint/libs/utils/mutex.go | 18 ++ 8 files changed, 202 insertions(+), 227 deletions(-) diff --git a/sei-tendermint/internal/autobahn/autobahn.proto b/sei-tendermint/internal/autobahn/autobahn.proto index 0145ce0b3c..6db13dfe78 100644 --- a/sei-tendermint/internal/autobahn/autobahn.proto +++ b/sei-tendermint/internal/autobahn/autobahn.proto @@ -74,7 +74,7 @@ message BlockHeader { } message Payload { - reserved "edge_count ", "coinbase", "basefee"; + reserved "edge_count", "coinbase", "basefee"; reserved 3,4,5; option (hashable.hashable) = true; optional Timestamp created_at = 1; // required diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 51cc473c84..1c50a7a943 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -525,12 +525,11 @@ func (s *State) fullCommitQC(ctx context.Context, n types.RoadIndex) (*types.Ful } // WaitForCapacity waits until the given lane has enough capacity for a new block. -func (s *State) WaitForCapacity(ctx context.Context) error { +func (s *State) WaitForCapacity(ctx context.Context, toProduce types.BlockNumber) error { lane := s.key.Public() for inner, ctrl := range s.inner.Lock() { - q := inner.blocks[lane] if err := ctrl.WaitUntil(ctx, func() bool { - return q.next < inner.persistedBlockStart[lane]+BlocksPerLane + return toProduce < inner.persistedBlockStart[lane]+BlocksPerLane }); err != nil { return err } @@ -569,12 +568,12 @@ func (s *State) WaitForLaneQCs( // ProduceBlock appends a new block to the producers lane. // Blocks until the lane has enough capacity for the new block. -func (s *State) ProduceBlock(ctx context.Context, payload *types.Payload) (*types.Signed[*types.LaneProposal], error) { - return s.produceBlock(ctx, s.key, payload) +func (s *State) ProduceBlock(n types.BlockNumber, payload *types.Payload) (*types.Signed[*types.LaneProposal], error) { + return s.produceBlock(n, s.key, payload) } // TODO: produceBlock is a separate function for testing - consider improving the tests to use ProduceBlock only. -func (s *State) produceBlock(ctx context.Context, key types.SecretKey, payload *types.Payload) (*types.Signed[*types.LaneProposal], error) { +func (s *State) produceBlock(n types.BlockNumber, key types.SecretKey, payload *types.Payload) (*types.Signed[*types.LaneProposal], error) { lane := key.Public() var result *types.Signed[*types.LaneProposal] for inner, ctrl := range s.inner.Lock() { @@ -582,10 +581,11 @@ func (s *State) produceBlock(ctx context.Context, key types.SecretKey, payload * if !ok { return nil, ErrBadLane } - if err := ctrl.WaitUntil(ctx, func() bool { - return q.next < inner.persistedBlockStart[lane]+BlocksPerLane - }); err != nil { - return nil, err + if n >= inner.persistedBlockStart[lane]+BlocksPerLane { + return nil, fmt.Errorf("lane full") + } + if q.next != n { + return nil, fmt.Errorf("unexpected block number: got %v, want %v",n,q.next) } var parent types.BlockHeaderHash if q.first < q.next { diff --git a/sei-tendermint/internal/autobahn/pb/autobahn.pb.go b/sei-tendermint/internal/autobahn/pb/autobahn.pb.go index 6085e0b2bb..aa7d59194c 100644 --- a/sei-tendermint/internal/autobahn/pb/autobahn.pb.go +++ b/sei-tendermint/internal/autobahn/pb/autobahn.pb.go @@ -542,11 +542,8 @@ func (x *BlockHeader) GetPayloadHash() []byte { type Payload struct { state protoimpl.MessageState `protogen:"open.v1"` - CreatedAt *Timestamp `protobuf:"bytes,1,opt,name=created_at,json=createdAt,proto3,oneof" json:"created_at,omitempty"` // required - TotalGas *uint64 `protobuf:"varint,2,opt,name=total_gas,json=totalGas,proto3,oneof" json:"total_gas,omitempty"` // required - EdgeCount *int64 `protobuf:"varint,3,opt,name=edge_count,json=edgeCount,proto3,oneof" json:"edge_count,omitempty"` // required - Coinbase []byte `protobuf:"bytes,4,opt,name=coinbase,proto3,oneof" json:"coinbase,omitempty"` // required - Basefee *int64 `protobuf:"varint,5,opt,name=basefee,proto3,oneof" json:"basefee,omitempty"` // required + CreatedAt *Timestamp `protobuf:"bytes,1,opt,name=created_at,json=createdAt,proto3,oneof" json:"created_at,omitempty"` // required + TotalGas *uint64 `protobuf:"varint,2,opt,name=total_gas,json=totalGas,proto3,oneof" json:"total_gas,omitempty"` // required Txs [][]byte `protobuf:"bytes,6,rep,name=txs,proto3" json:"txs,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -596,27 +593,6 @@ func (x *Payload) GetTotalGas() uint64 { return 0 } -func (x *Payload) GetEdgeCount() int64 { - if x != nil && x.EdgeCount != nil { - return *x.EdgeCount - } - return 0 -} - -func (x *Payload) GetCoinbase() []byte { - if x != nil { - return x.Coinbase - } - return nil -} - -func (x *Payload) GetBasefee() int64 { - if x != nil && x.Basefee != nil { - return *x.Basefee - } - return 0 -} - func (x *Payload) GetTxs() [][]byte { if x != nil { return x.Txs @@ -1963,23 +1939,16 @@ const file_autobahn_autobahn_proto_rawDesc = "" + "\x05_laneB\x0f\n" + "\r_block_numberB\x0e\n" + "\f_parent_hashB\x0f\n" + - "\r_payload_hash\"\xa7\x02\n" + + "\r_payload_hash\"\xcc\x01\n" + "\aPayload\x127\n" + "\n" + "created_at\x18\x01 \x01(\v2\x13.autobahn.TimestampH\x00R\tcreatedAt\x88\x01\x01\x12 \n" + - "\ttotal_gas\x18\x02 \x01(\x04H\x01R\btotalGas\x88\x01\x01\x12\"\n" + - "\n" + - "edge_count\x18\x03 \x01(\x03H\x02R\tedgeCount\x88\x01\x01\x12\x1f\n" + - "\bcoinbase\x18\x04 \x01(\fH\x03R\bcoinbase\x88\x01\x01\x12\x1d\n" + - "\abasefee\x18\x05 \x01(\x03H\x04R\abasefee\x88\x01\x01\x12\x10\n" + + "\ttotal_gas\x18\x02 \x01(\x04H\x01R\btotalGas\x88\x01\x01\x12\x10\n" + "\x03txs\x18\x06 \x03(\fR\x03txs:\x06Ȉ\xe2\xab\f\x01B\r\n" + "\v_created_atB\f\n" + "\n" + - "_total_gasB\r\n" + - "\v_edge_countB\v\n" + - "\t_coinbaseB\n" + - "\n" + - "\b_basefee\"\x8c\x01\n" + + "_total_gasJ\x04\b\x03\x10\x04J\x04\b\x04\x10\x05J\x04\b\x05\x10\x06R\n" + + "edge_countR\bcoinbaseR\abasefee\"\x8c\x01\n" + "\x05Block\x122\n" + "\x06header\x18\x01 \x01(\v2\x15.autobahn.BlockHeaderH\x00R\x06header\x88\x01\x01\x120\n" + "\apayload\x18\x02 \x01(\v2\x11.autobahn.PayloadH\x01R\apayload\x88\x01\x01:\x06Ȉ\xe2\xab\f\x01B\t\n" + diff --git a/sei-tendermint/internal/autobahn/producer/mempool.go b/sei-tendermint/internal/autobahn/producer/mempool.go index 491622c3bf..266f299a99 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool.go +++ b/sei-tendermint/internal/autobahn/producer/mempool.go @@ -4,8 +4,6 @@ import ( "context" "errors" "fmt" - "time" - "golang.org/x/exp/constraints" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" "github.com/ethereum/go-ethereum/common" tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/types" @@ -13,41 +11,6 @@ import ( abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" ) -// queue is a collection of objects of type T, indexed by type I in range [first,next). -// Supports pushing new items to the back and popping items from the front. -type queue[I constraints.Integer, T any] struct { - q map[I]T - first I - next I -} - -func newQueue[I ~uint64, T any]() *queue[I, T] { - return &queue[I, T]{q: map[I]T{}, first: 0, next: 0} -} - -func (q *queue[I, T]) Len() I { return q.next - q.first } - -func (q *queue[I, T]) PushBack(i I, t T) { - if q.next <= i { - q.q[i] = t - if q.first == q.next { - q.first = i - } - q.next = i+1 - } -} - -func (q *queue[I, T]) Prune(i I) (res T, ok bool) { - for q.first < min(i,q.next) { - res,ok = q.q[q.first] - delete(q.q,q.first) - q.first += 1 - } - q.first = max(q.first,i) - q.next = max(q.next,i) - return -} - type evmAccount struct { // nonce that the account will have after the readyTxs are executed. nextNonce uint64 @@ -59,48 +22,42 @@ type evmAccount struct { // If after execution the nonce is below the expectation, it means that execution failed and the // same will happen with all the subsequent txs (because of the nonce gap). In such a case we // drop whole account from the mempool, because user needs to submit the txs again. - nonceByBlock queue[types.BlockNumber,uint64] + nonceByBlock map[types.BlockNumber]uint64 } -type mempool struct { +type blockSpec struct { gasEstimated uint64 gasWanted uint64 sizeBytes uint64 txs [][]byte - nextPayload utils.Option[*types.Payload] + evmNonces map[common.Address]uint64 +} - nextToProduce types.BlockNumber - nextToExecute types.BlockNumber - evmAccounts map[common.Address]*evmAccount +type mempool struct { + capacity uint64 + first types.BlockNumber + next types.BlockNumber + blocks map[types.BlockNumber]*blockSpec + nextBlock *blockSpec + evmNonces map[common.Address]uint64 } -func (m *mempool) buildPayload() { - if m.nextPayload.IsPresent() { - return - } - // Snapshot evm state. - for _,acc := range m.evmAccounts { - acc.nonceByBlock.PushBack(m.nextToProduce,acc.nextNonce) - } - m.nextToProduce += 1 - // Construct a payload. - payload, err := types.PayloadBuilder{ - CreatedAt: time.Now(), - TotalGas: uint64(m.gasEstimated), // nolint:gosec // always non-negative - Txs: m.txs, - }.Build() - if err != nil { - // This should never happen: we construct the payload from correctly sized data. - panic(fmt.Errorf("PayloadBuilder{}.Build(): %w", err)) +func (m *mempool) IsFull() bool { + return uint64(m.next-m.first) >= m.capacity && len(m.nextBlock.txs) > 0 +} + +func (m *mempool) CanPushBlock() bool { + return uint64(m.next-m.first) < m.capacity && len(m.nextBlock.txs) > 0 +} + +func (m *mempool) PushBlock() { + m.blocks[m.next] = m.nextBlock + m.nextBlock = &blockSpec{ + evmNonces: map[common.Address]uint64{}, } - m.nextPayload = utils.Some(payload) - // Clear the mempool. - m.txs = nil - m.gasEstimated = 0 - m.gasWanted = 0 - m.sizeBytes = 0 } + // (addr,nonce) -> tx // tracking of what is in progress // on startup @@ -115,13 +72,10 @@ func (m *mempool) buildPayload() { // * accept only ready txs // * don't drop ready txs (unless some tx was unexpectedly dropped) // * drop over capacity. -// TODO: make sure that we query nonce at height > expected height -// this way our check will be an approximation from below - func (s *State) EvmNextPendingNonce(addr common.Address) uint64 { for m := range s.mempool.Lock() { - if acc,ok := m.evmAccounts[addr]; ok { - return acc.nextNonce + if nonce,ok := m.evmNonces[addr]; ok { + return nonce } } return s.app.EvmNonce(addr) @@ -129,6 +83,44 @@ func (s *State) EvmNextPendingNonce(addr common.Address) uint64 { var errTooLarge = errors.New("transaction too large") var errFull = errors.New("mempool is full") +var errBadNonce = errors.New("bad nonce") + +func (s *State) mempoolFirst() types.BlockNumber { + for m := range s.mempool.Lock() { + return m.first + } + panic("unreachable") +} + +func (s *State) pruneMempool(n types.BlockNumber) { + for m,ctrl := range s.mempool.Lock() { + if n < m.first { return } + ctrl.Updated() + for m.first < min(n,m.next) { + b := m.blocks[m.first] + delete(m.blocks,m.first) + m.first += 1 + for addr,wantNonce := range b.evmNonces { + if wantNonce == m.evmNonces[addr] { + // Happy path: all account's txs got executed. + delete(m.evmNonces,addr) + } else if gotNonce := s.app.EvmNonce(addr); gotNonce < wantNonce { + // Some txs have not been executed - reset account tracking. + // NOTE: app execution is not synchronized with mempool, so nonce could have already + // proceeded past wantNonce and that is expected. + delete(m.evmNonces,addr) + for _, x := range m.blocks { + delete(x.evmNonces,addr) + } + } + } + } + // n > m.next shouldn't really happen, + // because local mempool is the only source of local lane blocks, + // but we handle it gracefully anyway. + m.next = max(m.next,n) + } +} // Blocking insert. func (s *State) Insert(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseCheckTx, error) { @@ -142,27 +134,27 @@ func (s *State) Insert(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseCheckT if gasWanted > s.cfg.MaxGasPerBlock { return nil, errTooLarge } for m,ctrl := range s.mempool.Lock() { - if err:=ctrl.WaitUntil(ctx, func() bool { return !m.nextPayload.IsPresent() }); err!=nil { + if err:=ctrl.WaitUntil(ctx, func() bool { return !m.IsFull() }); err!=nil { return nil, err } if resp.IsEVM { addr := resp.EVMSenderAddress - acc,ok := m.evmAccounts[addr] + nonce,ok := m.evmNonces[addr] if !ok { - acc = &evmAccount { nextNonce: s.app.EvmNonce(addr) } + nonce = s.app.EvmNonce(addr) } - if acc.nextNonce != resp.EVMNonce { - return nil, fmt.Errorf("bad nonce: got %v, want %v", resp.EVMNonce, acc.nextNonce) + if nonce != resp.EVMNonce { + return nil, fmt.Errorf("%w: got %v, want %v", errBadNonce, resp.EVMNonce, nonce) } - acc.nextNonce += 1 - m.evmAccounts[addr] = acc + m.nextBlock.evmNonces[addr] = nonce + 1 + m.evmNonces[addr] = nonce + 1 } // If any limit would be exceeded, then construct a payload. - ok := uint64(len(m.txs)) + 1 <= s.cfg.maxTxsPerBlock() - ok = ok && m.sizeBytes + uint64(len(tx)) <= types.MaxTxsBytesPerBlock - ok = ok && m.gasWanted + gasWanted <= s.cfg.MaxGasPerBlock + ok := uint64(len(m.nextBlock.txs)) + 1 <= s.cfg.maxTxsPerBlock() + ok = ok && m.nextBlock.sizeBytes + uint64(len(tx)) <= types.MaxTxsBytesPerBlock + ok = ok && m.nextBlock.gasWanted + gasWanted <= s.cfg.MaxGasPerBlock if !ok { - m.buildPayload() + m.PushBlock() ctrl.Updated() } @@ -171,11 +163,11 @@ func (s *State) Insert(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseCheckT if gasEstimated < minTxGas || gasEstimated > resp.GasWanted { gasEstimated = resp.GasWanted } - m.gasEstimated += utils.Clamp[uint64](gasEstimated) - m.gasWanted += utils.Clamp[uint64](resp.GasWanted) - m.sizeBytes += uint64(len(tx)) - m.txs = append(m.txs,tx) - return resp.ResponseCheckTx,nil + b := m.nextBlock + b.gasEstimated += utils.Clamp[uint64](gasEstimated) + b.gasWanted += utils.Clamp[uint64](resp.GasWanted) + b.sizeBytes += uint64(len(tx)) + b.txs = append(b.txs,tx) } - panic("unreachable") + return resp.ResponseCheckTx,nil } diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index 9bce172093..813d64aee6 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -21,7 +21,6 @@ type Config struct { MaxTxsPerBlock uint64 // Delay after which a non-full block can be produced. BlockInterval time.Duration - AllowEmptyBlocks bool // TESTONLY: max rate at which lane is produced. It can be used to do // benchmarks with stable throughput, in case execution performance degrades // when overloaded. @@ -45,91 +44,106 @@ type State struct { // NewState constructs a new block producer state. // Returns an error if the current node is NOT a producer. func NewState(cfg *Config, consensus *consensus.State) *State { + lane := consensus.Avail().PublicKey() + n := consensus.Avail().NextBlock(lane) return &State{ cfg: cfg, mempool: utils.NewWatch(&mempool { - evmAccounts: map[common.Address]*evmAccount{}, + capacity: avail.BlocksPerLane, + first: n, + next: n, + evmNonces: map[common.Address]uint64{}, }), consensus: consensus, } } -func (s *State) setNextToExecute(n types.BlockNumber) { - for m,ctrl := range s.mempool.Lock() { - if n < m.nextToExecute { return } - ctrl.Updated() - m.nextToExecute = n - for addr,acc := range m.evmAccounts { - if wantMin,ok := acc.nonceByBlock.Prune(n); acc.nonceByBlock.Len()==0 || (ok && s.app.EvmNonce(addr) < wantMin) { - delete(m.evmAccounts,addr) - } - } - } -} - -// Run runs the background tasks of the producer state. +// Run runs the background tasks of the producer state: +// * prunes executed lane blocks from mempool +// * pushes new lane blocks from mempool to avail state +// Note that mempool capacity bounds the number of unexecuted blocks of the local lane. +// This is needed so that we can track the evm nonces of sequenced txs - mempool admits txs +// sequentially in the nonce order. func (s *State) Run(ctx context.Context) error { - return scope.Run(ctx, func(ctx context.Context, scope scope.Scope) error { - // Construct blocks from mempool. - limit := rate.Inf - burst := 1 - if l, ok := s.cfg.MaxTxsPerSecond.Get(); ok { - limit = rate.Limit(l) - burst = int(l + s.cfg.MaxTxsPerBlock) // nolint:gosec - } - limiter := rate.NewLimiter(limit, burst) - dataState := s.consensus.Data() + return scope.Run(ctx, func(ctx context.Context, scope scope.Scope) error { availState := s.consensus.Avail() lane := availState.PublicKey() - - for m := range s.mempool.Lock() { - m.nextToExecute = 0 - m.nextToProduce = availState.NextBlock(lane) - } - + firstBlock := s.mempoolFirst() scope.Spawn(func() error { - next := types.BlockNumber(0) + // Task pruning executed lane blocks from the mempool + dataState := s.consensus.Data() var err error - for { - if next,err = dataState.WaitUntilExecuted(ctx,lane,next); err!=nil { return err } - s.setNextToExecute(next) + for toExecute := firstBlock ;; { + if toExecute,err = dataState.WaitUntilExecuted(ctx,lane,toExecute); err!=nil { + return err + } + s.pruneMempool(toExecute) } }) - lastBlockTime := time.Now() - for { - if _,err := nextToExecute.Wait(ctx, func(next types.BlockNumber) bool { return next + avail.BlocksPerLane > n }); err!=nil { - return err - } - if err := availState.WaitForCapacity(ctx); err != nil { - return fmt.Errorf("s.consensus.Avail().WaitForCapacity(): %w", err) + scope.Spawn(func() error { + // Task pushing blocks from mempool to avail state. + limit := rate.Inf + burst := 1 + if l, ok := s.cfg.MaxTxsPerSecond.Get(); ok { + limit = rate.Limit(l) + burst = int(l + s.cfg.MaxTxsPerBlock) // nolint:gosec } - // Wait until either - // * there is a full proposal in mempool - // * BlockInterval since the last block passed AND (AllowEmptyBlocks OR mempool is non-empty) - for m,ctrl := range s.mempool.Lock() { - // First just wait for full proposal with timeout (first condition) - _ = utils.WithDeadline(ctx, utils.Some(lastBlockTime.Add(s.cfg.BlockInterval)), func(ctx context.Context) error { - return ctrl.WaitUntil(ctx, func() bool { return m.nextPayload.IsPresent() }) - }) - if ctx.Err()!=nil { - return ctx.Err() + limiter := rate.NewLimiter(limit, burst) + lastBlockTime := time.Now() + for toProduce:=firstBlock;; firstBlock += 1 { + if err := availState.WaitForCapacity(ctx,toProduce); err != nil { + return fmt.Errorf("availState.WaitForCapacity(): %w", err) } - // Then wait for ANY condition. - if err:=ctrl.WaitUntil(ctx, func() bool { - return m.nextPayload.IsPresent() || s.cfg.AllowEmptyBlocks || len(m.txs) > 0 - }); err!=nil { - return err + var payload *types.Payload + // Wait until either + // * there is a full proposal in mempool + // * BlockInterval since the last block passed AND mempool is non-empty + for m,ctrl := range s.mempool.Lock() { + // Wait for full payload with timeout. + if err := utils.WithDeadline(ctx, utils.Some(lastBlockTime.Add(s.cfg.BlockInterval)), func(ctx context.Context) error { + return ctrl.WaitUntil(ctx, func() bool { return toProduce < m.next }) + }); err!=nil { + if ctx.Err()!=nil { + return ctx.Err() + } + // Wait for non-empty payload. + if err:=ctrl.WaitUntil(ctx, func() bool { + return toProduce < m.next || (toProduce==m.next && m.CanPushBlock()) + }); err!=nil { + return err + } + // Seal the payload if needed. + if toProduce==m.next { + m.PushBlock() + } + } + b,ok := m.blocks[toProduce] + if !ok { + // Block number tracking should always be in sync between avail state and mempool: + // * mempool keeps blocks until they are executed. + // * blocks can be executed only after they are included in the lane. + // * lane is populated from the mempool. + return fmt.Errorf("mempool mismatched block production") + } + var err error + payload, err = types.PayloadBuilder{ + CreatedAt: time.Now(), + TotalGas: b.gasEstimated, + Txs: b.txs, + }.Build() + if err != nil { + // This should never happen: we construct the payload from correctly sized data. + panic(fmt.Errorf("PayloadBuilder{}.Build(): %w", err)) + } + } + if _, err := availState.ProduceBlock(toProduce, payload); err != nil { + return fmt.Errorf("availState.ProduceBlock(): %w", err) + } + if err := limiter.WaitN(ctx, len(payload.Txs())); err != nil { + return fmt.Errorf("limiter(): %w", err) } - // Construct the payload unconditionally. - m.buildPayload() - } - - if _, err := availState.ProduceBlock(ctx, payload); err != nil { - return fmt.Errorf("availState.ProduceBlock(): %w", err) - } - if err := limiter.WaitN(ctx, len(payload.Txs())); err != nil { - return fmt.Errorf("limiter(): %w", err) } - } + }) + return nil }) } diff --git a/sei-tendermint/internal/autobahn/types/testonly.go b/sei-tendermint/internal/autobahn/types/testonly.go index 11a126d659..5044fac52e 100644 --- a/sei-tendermint/internal/autobahn/types/testonly.go +++ b/sei-tendermint/internal/autobahn/types/testonly.go @@ -93,9 +93,6 @@ func GenPayload(rng utils.Rng) *Payload { return utils.OrPanic1(PayloadBuilder{ CreatedAt: utils.GenTimestamp(rng), TotalGas: rng.Uint64(), - EdgeCount: rng.Int63(), - Coinbase: utils.GenBytes(rng, 10), - Basefee: rng.Int63(), Txs: utils.GenSlice(rng, func(rng utils.Rng) []byte { return utils.GenBytes(rng, 10) }), }.Build()) } diff --git a/sei-tendermint/internal/p2p/conn/secret_connection.go b/sei-tendermint/internal/p2p/conn/secret_connection.go index dff6272f0f..c5d970e03f 100644 --- a/sei-tendermint/internal/p2p/conn/secret_connection.go +++ b/sei-tendermint/internal/p2p/conn/secret_connection.go @@ -42,22 +42,7 @@ const ( labelSecretConnectionMac = "SECRET_CONNECTION_MAC" ) -type asyncMutex[T any] struct { - mu chan struct{} - v T -} - -func newAsyncMutex[T any](v T) asyncMutex[T] { - return asyncMutex[T]{make(chan struct{}, 1), v} -} -func (m *asyncMutex[T]) Lock(ctx context.Context, yield func(T) error) error { - if err := utils.Send(ctx, m.mu, struct{}{}); err != nil { - return err - } - defer func() { <-m.mu }() - return yield(m.v) -} var secretConnKeyAndChallengeGen = []byte("TENDERMINT_SECRET_CONNECTION_KEY_AND_CHALLENGE_GEN") @@ -107,8 +92,8 @@ type Challenge [32]byte type SecretConnection struct { conn Conn challenge Challenge - recvState asyncMutex[*recvState] - sendState asyncMutex[*sendState] + recvState utils.AsyncMutex[*recvState] + sendState utils.AsyncMutex[*sendState] } func (sc *SecretConnection) Challenge() Challenge { return sc.challenge } @@ -136,8 +121,8 @@ func newSecretConnection(conn Conn, loc ephSecret, rem ephPublic) (*SecretConnec return &SecretConnection{ conn: conn, challenge: challenge, - recvState: newAsyncMutex(newRecvState(aead.recv.Cipher())), - sendState: newAsyncMutex(newSendState(aead.send.Cipher())), + recvState: utils.NewAsyncMutex(newRecvState(aead.recv.Cipher())), + sendState: utils.NewAsyncMutex(newSendState(aead.send.Cipher())), }, nil } diff --git a/sei-tendermint/libs/utils/mutex.go b/sei-tendermint/libs/utils/mutex.go index ffd5952e85..f4416f8434 100644 --- a/sei-tendermint/libs/utils/mutex.go +++ b/sei-tendermint/libs/utils/mutex.go @@ -232,3 +232,21 @@ func MonitorWatchUpdates[T any](w *Watch[T], f func()) bool { return false } } + +type AsyncMutex[T any] struct { + _ NoCopy + mu chan struct{} + v T +} + +func NewAsyncMutex[T any](v T) AsyncMutex[T] { + return AsyncMutex[T]{mu:make(chan struct{}, 1), v:v} +} + +func (m *AsyncMutex[T]) Lock(ctx context.Context, yield func(T) error) error { + if err := Send(ctx, m.mu, struct{}{}); err != nil { + return err + } + defer func() { <-m.mu }() + return yield(m.v) +} From c59c3d8b6c6b3482369074c8bd01ff0cfc037fe0 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 27 May 2026 13:10:09 +0200 Subject: [PATCH 066/100] reverted unrelated changes --- .../internal/autobahn/avail/inner.go | 16 ++++++------- .../internal/autobahn/avail/queue.go | 17 +++++--------- .../internal/autobahn/avail/state.go | 10 ++++---- .../internal/p2p/conn/secret_connection.go | 23 +++++++++++++++---- sei-tendermint/libs/utils/mutex.go | 18 --------------- 5 files changed, 38 insertions(+), 46 deletions(-) diff --git a/sei-tendermint/internal/autobahn/avail/inner.go b/sei-tendermint/internal/autobahn/avail/inner.go index 9722315f6b..fc22dac206 100644 --- a/sei-tendermint/internal/autobahn/avail/inner.go +++ b/sei-tendermint/internal/autobahn/avail/inner.go @@ -73,7 +73,7 @@ func newInner(c *types.Committee, loaded utils.Option[*loadedAvailState]) (*inne nextBlockToPersist: make(map[types.LaneID]types.BlockNumber, c.Lanes().Len()), persistedBlockStart: make(map[types.LaneID]types.BlockNumber, c.Lanes().Len()), } - i.appVotes.Prune(c.FirstBlock()) + i.appVotes.prune(c.FirstBlock()) l, ok := loaded.Get() if !ok { @@ -105,7 +105,7 @@ func newInner(c *types.Committee, loaded utils.Option[*loadedAvailState]) (*inne if lqc.Index != i.commitQCs.next { return nil, fmt.Errorf("non-contiguous persisted commitQCs: expected %d, got %d", i.commitQCs.next, lqc.Index) } - i.commitQCs.PushBack(lqc.QC) + i.commitQCs.pushBack(lqc.QC) } if i.commitQCs.next > i.commitQCs.first { i.latestCommitQC.Store(utils.Some(i.commitQCs.q[i.commitQCs.next-1])) @@ -133,7 +133,7 @@ func newInner(c *types.Committee, loaded utils.Option[*loadedAvailState]) (*inne } } lastHash = b.Proposal.Msg().Block().Header().Hash() - q.PushBack(b.Proposal) + q.pushBack(b.Proposal) } if q.next > q.first { i.nextBlockToPersist[lane] = q.next @@ -163,15 +163,15 @@ func (i *inner) prune(c *types.Committee, appQC *types.AppQC, commitQC *types.Co return false, nil } i.latestAppQC = utils.Some(appQC) - i.commitQCs.Prune(idx) + i.commitQCs.prune(idx) if i.commitQCs.next == idx { - i.commitQCs.PushBack(commitQC) + i.commitQCs.pushBack(commitQC) } - i.appVotes.Prune(commitQC.GlobalRange(c).First) + i.appVotes.prune(commitQC.GlobalRange(c).First) for lane := range i.votes { lr := commitQC.LaneRange(lane) - i.votes[lr.Lane()].Prune(lr.First()) - i.blocks[lr.Lane()].Prune(lr.First()) + i.votes[lr.Lane()].prune(lr.First()) + i.blocks[lr.Lane()].prune(lr.First()) if i.nextBlockToPersist[lr.Lane()] < lr.First() { i.nextBlockToPersist[lr.Lane()] = lr.First() } diff --git a/sei-tendermint/internal/autobahn/avail/queue.go b/sei-tendermint/internal/autobahn/avail/queue.go index 072a7f5540..1b4eaaded5 100644 --- a/sei-tendermint/internal/autobahn/avail/queue.go +++ b/sei-tendermint/internal/autobahn/avail/queue.go @@ -1,12 +1,8 @@ package avail -import ( - "golang.org/x/exp/constraints" -) - // queue is a collection of objects of type T, indexed by type I in range [first,next). // Supports pushing new items to the back and popping items from the front. -type queue[I constraints.Integer, T any] struct { +type queue[I ~uint64, T any] struct { q map[I]T first I next I @@ -16,17 +12,16 @@ func newQueue[I ~uint64, T any]() *queue[I, T] { return &queue[I, T]{q: map[I]T{}, first: 0, next: 0} } -func (q *queue[I, T]) First() I { return q.first } -func (q *queue[I, T]) Next() I { return q.next } -func (q *queue[I, T]) Get(i I) T { return q.q[i] } -func (q *queue[I, T]) Len() I { return q.next - q.first } +func (q *queue[I, T]) Len() uint64 { + return uint64(q.next) - uint64(q.first) +} -func (q *queue[I, T]) PushBack(t T) { +func (q *queue[I, T]) pushBack(t T) { q.q[q.next] = t q.next += 1 } -func (q *queue[I, T]) Prune(newFirst I) { +func (q *queue[I, T]) prune(newFirst I) { if newFirst <= q.first { return } diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 1c50a7a943..8ce78ff311 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -268,7 +268,7 @@ func (s *State) PushCommitQC(ctx context.Context, qc *types.CommitQC) error { if idx != inner.commitQCs.next { return nil } - inner.commitQCs.PushBack(qc) + inner.commitQCs.pushBack(qc) // The persist goroutine publishes latestCommitQC after writing to disk // (or immediately for no-op persisters), so consensus won't advance // until the CommitQC is durable. @@ -302,7 +302,7 @@ func (s *State) PushAppVote(ctx context.Context, v *types.Signed[*types.AppVote] n := v.Msg().Proposal().GlobalNumber() q := inner.appVotes for q.next <= n { - q.PushBack(newAppVotes()) + q.pushBack(newAppVotes()) } appQC, ok := q.q[n].pushVote(s.data.Committee(), v) if !ok { @@ -432,7 +432,7 @@ func (s *State) PushBlock(ctx context.Context, p *types.Signed[*types.LanePropos return nil } } - q.PushBack(p) + q.pushBack(p) ctrl.Updated() } return nil @@ -463,7 +463,7 @@ func (s *State) PushVote(ctx context.Context, vote *types.Signed[*types.LaneVote return nil } for q.next <= h.BlockNumber() { - q.PushBack(newBlockVotes()) + q.pushBack(newBlockVotes()) } if _, ok := q.q[h.BlockNumber()].pushVote(s.data.Committee(), vote); ok { ctrl.Updated() @@ -592,7 +592,7 @@ func (s *State) produceBlock(n types.BlockNumber, key types.SecretKey, payload * parent = q.q[q.next-1].Msg().Block().Header().Hash() } result = types.Sign(key, types.NewLaneProposal(types.NewBlock(lane, q.next, parent, payload))) - q.PushBack(result) + q.pushBack(result) ctrl.Updated() } return result, nil diff --git a/sei-tendermint/internal/p2p/conn/secret_connection.go b/sei-tendermint/internal/p2p/conn/secret_connection.go index c5d970e03f..dff6272f0f 100644 --- a/sei-tendermint/internal/p2p/conn/secret_connection.go +++ b/sei-tendermint/internal/p2p/conn/secret_connection.go @@ -42,7 +42,22 @@ const ( labelSecretConnectionMac = "SECRET_CONNECTION_MAC" ) +type asyncMutex[T any] struct { + mu chan struct{} + v T +} + +func newAsyncMutex[T any](v T) asyncMutex[T] { + return asyncMutex[T]{make(chan struct{}, 1), v} +} +func (m *asyncMutex[T]) Lock(ctx context.Context, yield func(T) error) error { + if err := utils.Send(ctx, m.mu, struct{}{}); err != nil { + return err + } + defer func() { <-m.mu }() + return yield(m.v) +} var secretConnKeyAndChallengeGen = []byte("TENDERMINT_SECRET_CONNECTION_KEY_AND_CHALLENGE_GEN") @@ -92,8 +107,8 @@ type Challenge [32]byte type SecretConnection struct { conn Conn challenge Challenge - recvState utils.AsyncMutex[*recvState] - sendState utils.AsyncMutex[*sendState] + recvState asyncMutex[*recvState] + sendState asyncMutex[*sendState] } func (sc *SecretConnection) Challenge() Challenge { return sc.challenge } @@ -121,8 +136,8 @@ func newSecretConnection(conn Conn, loc ephSecret, rem ephPublic) (*SecretConnec return &SecretConnection{ conn: conn, challenge: challenge, - recvState: utils.NewAsyncMutex(newRecvState(aead.recv.Cipher())), - sendState: utils.NewAsyncMutex(newSendState(aead.send.Cipher())), + recvState: newAsyncMutex(newRecvState(aead.recv.Cipher())), + sendState: newAsyncMutex(newSendState(aead.send.Cipher())), }, nil } diff --git a/sei-tendermint/libs/utils/mutex.go b/sei-tendermint/libs/utils/mutex.go index f4416f8434..ffd5952e85 100644 --- a/sei-tendermint/libs/utils/mutex.go +++ b/sei-tendermint/libs/utils/mutex.go @@ -232,21 +232,3 @@ func MonitorWatchUpdates[T any](w *Watch[T], f func()) bool { return false } } - -type AsyncMutex[T any] struct { - _ NoCopy - mu chan struct{} - v T -} - -func NewAsyncMutex[T any](v T) AsyncMutex[T] { - return AsyncMutex[T]{mu:make(chan struct{}, 1), v:v} -} - -func (m *AsyncMutex[T]) Lock(ctx context.Context, yield func(T) error) error { - if err := Send(ctx, m.mu, struct{}{}); err != nil { - return err - } - defer func() { <-m.mu }() - return yield(m.v) -} From abaf008fa7b96042b25034021d988d57cae0ff8c Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 27 May 2026 14:21:25 +0200 Subject: [PATCH 067/100] WIP separating tm and autobahn mempools --- sei-tendermint/config/autobahn.go | 5 --- .../internal/autobahn/avail/state_test.go | 9 +++--- .../internal/autobahn/producer/mempool.go | 32 +++++++------------ .../internal/autobahn/producer/state.go | 2 +- .../internal/p2p/giga/avail_test.go | 7 ++-- .../internal/p2p/giga/consensus_test.go | 4 ++- sei-tendermint/internal/p2p/giga_router.go | 19 ++++++----- .../internal/p2p/giga_router_test.go | 21 ++++-------- sei-tendermint/internal/p2p/router_test.go | 8 +---- sei-tendermint/internal/rpc/core/blocks.go | 4 ++- sei-tendermint/node/node.go | 8 ++--- sei-tendermint/node/setup.go | 15 ++++----- 12 files changed, 53 insertions(+), 81 deletions(-) diff --git a/sei-tendermint/config/autobahn.go b/sei-tendermint/config/autobahn.go index ea92919f86..483af4dbfd 100644 --- a/sei-tendermint/config/autobahn.go +++ b/sei-tendermint/config/autobahn.go @@ -45,9 +45,7 @@ type AutobahnFileConfig struct { Validators []AutobahnValidator `json:"validators"` MaxTxsPerBlock uint64 `json:"max_txs_per_block"` MaxTxsPerSecond utils.Option[uint64] `json:"max_txs_per_second"` - MempoolSize uint64 `json:"mempool_size"` BlockInterval utils.Duration `json:"block_interval"` - AllowEmptyBlocks bool `json:"allow_empty_blocks"` ViewTimeout utils.Duration `json:"view_timeout"` PersistentStateDir utils.Option[string] `json:"persistent_state_dir"` DialInterval utils.Duration `json:"dial_interval"` @@ -61,9 +59,6 @@ func (fc *AutobahnFileConfig) Validate() error { if fc.MaxTxsPerBlock == 0 { return errors.New("max_txs_per_block must be > 0") } - if fc.MempoolSize == 0 { - return errors.New("mempool_size must be > 0") - } if fc.BlockInterval <= 0 { return errors.New("block_interval must be > 0") } diff --git a/sei-tendermint/internal/autobahn/avail/state_test.go b/sei-tendermint/internal/autobahn/avail/state_test.go index 0529d14605..324f86f074 100644 --- a/sei-tendermint/internal/autobahn/avail/state_test.go +++ b/sei-tendermint/internal/autobahn/avail/state_test.go @@ -128,7 +128,7 @@ func testState(t *testing.T, stateDir utils.Option[string]) { lane := key.Public() p := types.GenPayload(rng) want[lane] = append(want[lane], p.Hash()) - b, err := state.produceBlock(ctx, key, p) + b, err := state.produceBlock(state.NextBlock(lane), key, p) if err != nil { return fmt.Errorf("state.ProduceBlock(): %w", err) } @@ -254,7 +254,7 @@ func TestStateRestartFromPersisted(t *testing.T) { for range 5 { key := keys[rng.Intn(len(keys))] - if _, err := state.produceBlock(ctx, key, types.GenPayload(rng)); err != nil { + if _, err := state.produceBlock(state.NextBlock(key.Public()), key, types.GenPayload(rng)); err != nil { return fmt.Errorf("produceBlock: %w", err) } } @@ -340,7 +340,6 @@ func TestStateMismatchedQCs(t *testing.T) { }, utils.OrPanic1(data.NewDataWAL(utils.None[string](), committee)))) state, err := NewState(keys[0], ds, utils.None[string]()) require.NoError(t, err) - ctx := t.Context() // Helper to create a CommitQC for a specific index makeQC := func(prev utils.Option[*types.CommitQC], laneQCs map[types.LaneID]*types.LaneQC) *types.CommitQC { @@ -364,7 +363,7 @@ func TestStateMismatchedQCs(t *testing.T) { // 1. Produce a block so we have a non-empty range lane := keys[0].Public() p := types.GenPayload(rng) - b, err := state.ProduceBlock(ctx, p) + b, err := state.ProduceBlock(state.NextBlock(lane), p) require.NoError(t, err) // 2. Form a LaneQC for it @@ -398,7 +397,7 @@ func TestPushBlockRejectsBadParentHash(t *testing.T) { state := utils.OrPanic1(NewState(keys[0], ds, utils.None[string]())) // Produce a valid first block on our lane. - _, err := state.ProduceBlock(ctx, types.GenPayload(rng)) + _, err := state.ProduceBlock(state.NextBlock(keys[0].Public()), types.GenPayload(rng)) require.NoError(t, err) // Create a second block with a fake parentHash. diff --git a/sei-tendermint/internal/autobahn/producer/mempool.go b/sei-tendermint/internal/autobahn/producer/mempool.go index 266f299a99..010669dc6e 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool.go +++ b/sei-tendermint/internal/autobahn/producer/mempool.go @@ -11,25 +11,13 @@ import ( abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" ) -type evmAccount struct { - // nonce that the account will have after the readyTxs are executed. - nextNonce uint64 - // Nonces that this account is expected to be at after executing the given block. - // Since autobahn has asynchronous execution, there is no guarantee that the account nonce will - // be incremented at the time of constructing the lane block. - // On the other hand we need to be able to sequence many account txs before the first one is executed. - // To achieve that we track the expected per-account nonces after each block of the local lane. - // If after execution the nonce is below the expectation, it means that execution failed and the - // same will happen with all the subsequent txs (because of the nonce gap). In such a case we - // drop whole account from the mempool, because user needs to submit the txs again. - nonceByBlock map[types.BlockNumber]uint64 -} - type blockSpec struct { gasEstimated uint64 gasWanted uint64 sizeBytes uint64 txs [][]byte + // nonces of accounts which are expected to be bumped by this block. + // They are checked against the app state after the block is executed. evmNonces map[common.Address]uint64 } @@ -78,11 +66,10 @@ func (s *State) EvmNextPendingNonce(addr common.Address) uint64 { return nonce } } - return s.app.EvmNonce(addr) + return s.cfg.App.EvmNonce(addr) } var errTooLarge = errors.New("transaction too large") -var errFull = errors.New("mempool is full") var errBadNonce = errors.New("bad nonce") func (s *State) mempoolFirst() types.BlockNumber { @@ -104,7 +91,7 @@ func (s *State) pruneMempool(n types.BlockNumber) { if wantNonce == m.evmNonces[addr] { // Happy path: all account's txs got executed. delete(m.evmNonces,addr) - } else if gotNonce := s.app.EvmNonce(addr); gotNonce < wantNonce { + } else if gotNonce := s.cfg.App.EvmNonce(addr); gotNonce < wantNonce { // Some txs have not been executed - reset account tracking. // NOTE: app execution is not synchronized with mempool, so nonce could have already // proceeded past wantNonce and that is expected. @@ -122,18 +109,21 @@ func (s *State) pruneMempool(n types.BlockNumber) { } } -// Blocking insert. -func (s *State) Insert(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseCheckTx, error) { +// Inserts transaction. Blocks until there is capacity in the mempool. +func (s *State) InsertTx(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseCheckTx, error) { if uint64(len(tx)) > types.MaxTxsBytesPerBlock { return nil, errTooLarge } - resp, err := s.app.CheckTxSafe(ctx, &abci.RequestCheckTxV2{Tx: tx}) + resp, err := s.cfg.App.CheckTxSafe(ctx, &abci.RequestCheckTxV2{Tx: tx}) if err!=nil { return nil, err } if !resp.IsOK() { return resp.ResponseCheckTx, nil } gasWanted := utils.Clamp[uint64](resp.GasWanted) if gasWanted > s.cfg.MaxGasPerBlock { return nil, errTooLarge } for m,ctrl := range s.mempool.Lock() { + // mempool is constructed as a FIFO - we do not delay insertions of large txs (going over cap) + // in favor of waiting for smaller txs. This simple algorithm allows us to cap + // pending txs to size of a single block. We can refine this rule later if needed. if err:=ctrl.WaitUntil(ctx, func() bool { return !m.IsFull() }); err!=nil { return nil, err } @@ -141,7 +131,7 @@ func (s *State) Insert(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseCheckT addr := resp.EVMSenderAddress nonce,ok := m.evmNonces[addr] if !ok { - nonce = s.app.EvmNonce(addr) + nonce = s.cfg.App.EvmNonce(addr) } if nonce != resp.EVMNonce { return nil, fmt.Errorf("%w: got %v, want %v", errBadNonce, resp.EVMNonce, nonce) diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index 813d64aee6..4cafc23108 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -25,6 +25,7 @@ type Config struct { // benchmarks with stable throughput, in case execution performance degrades // when overloaded. MaxTxsPerSecond utils.Option[uint64] + App *proxy.Proxy } const minTxGas = 21000 @@ -35,7 +36,6 @@ func (c *Config) maxTxsPerBlock() uint64 { // State is the block producer state. type State struct { - app *proxy.Proxy cfg *Config mempool utils.Watch[*mempool] // consensus state to which published blocks will be reported. diff --git a/sei-tendermint/internal/p2p/giga/avail_test.go b/sei-tendermint/internal/p2p/giga/avail_test.go index 2141bf949f..14410556f8 100644 --- a/sei-tendermint/internal/p2p/giga/avail_test.go +++ b/sei-tendermint/internal/p2p/giga/avail_test.go @@ -38,15 +38,18 @@ func TestAvailClientServer(t *testing.T) { } s.SpawnBgNamed("fakeNode0", func() error { return utils.IgnoreCancel(fakeNode0.Run(ctx)) }) for range min(avail.BlocksPerLane, 4) { - b := utils.OrPanic1(fakeNode0.ProduceBlock(ctx, types.GenPayload(rng))) + a := fakeNode0.Avail() + b := utils.OrPanic1(a.ProduceBlock(a.NextBlock(a.PublicKey()), types.GenPayload(rng))) utils.OrPanic(nodes[2].consensus.Avail().PushBlock(ctx, b)) } t.Logf("Run block production") for _, node := range nodes { rng := rng.Split() s.Spawn(func() error { + a := node.consensus.Avail() + lane := a.PublicKey() for range totalBlocks { - if _, err := node.consensus.ProduceBlock(ctx, types.GenPayload(rng)); err != nil { + if _, err := a.ProduceBlock(a.NextBlock(lane), types.GenPayload(rng)); err != nil { return fmt.Errorf("produceBlock(): %w", err) } } diff --git a/sei-tendermint/internal/p2p/giga/consensus_test.go b/sei-tendermint/internal/p2p/giga/consensus_test.go index 1f431504a6..3a8ce052f3 100644 --- a/sei-tendermint/internal/p2p/giga/consensus_test.go +++ b/sei-tendermint/internal/p2p/giga/consensus_test.go @@ -27,7 +27,9 @@ func TestConsensusClientServer(t *testing.T) { for offset := range types.GlobalBlockNumber(20) { idx := firstBlock + offset t.Logf("[%v] Push a block.", idx) - b, err := nodes[rng.Intn(len(env.nodes))].consensus.ProduceBlock(ctx, types.GenPayload(rng)) + node := nodes[rng.Intn(len(env.nodes))] + a := node.consensus.Avail() + b, err := a.ProduceBlock(a.NextBlock(a.PublicKey()), types.GenPayload(rng)) if err != nil { return fmt.Errorf("ds.ProduceBlock(): %w", err) } diff --git a/sei-tendermint/internal/p2p/giga_router.go b/sei-tendermint/internal/p2p/giga_router.go index 6ec36b0027..53406a6040 100644 --- a/sei-tendermint/internal/p2p/giga_router.go +++ b/sei-tendermint/internal/p2p/giga_router.go @@ -16,7 +16,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/producer" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/giga" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/rpc" tmbytes "github.com/sei-protocol/sei-chain/sei-tendermint/libs/bytes" @@ -43,7 +42,6 @@ type GigaRouterConfig struct { ValidatorAddrs map[atypes.PublicKey]GigaNodeAddr Consensus *consensus.Config Producer *producer.Config - TxMempool *mempool.TxMempool GenDoc *types.GenesisDoc } @@ -95,7 +93,7 @@ func NewGigaRouter(cfg *GigaRouterConfig, key NodeSecretKey) (*GigaRouter, error if err != nil { return nil, fmt.Errorf("consensus.NewState(): %w", err) } - producerState := producer.NewState(cfg.Producer, cfg.TxMempool, consensusState) + producerState := producer.NewState(cfg.Producer, consensusState) logger.Info("GigaRouter initialized", "validators", len(cfg.ValidatorAddrs), "dial_interval", cfg.DialInterval) return &GigaRouter{ cfg: cfg, @@ -113,6 +111,10 @@ func NewGigaRouter(cfg *GigaRouterConfig, key NodeSecretKey) (*GigaRouter, error }, nil } +func (r *GigaRouter) InsertTx(ctx context.Context, tx types.Tx) (*abci.ResponseCheckTx, error) { + return r.producer.InsertTx(ctx,tx) +} + // LastCommittedBlockNumber returns the highest global block number finalized // by consensus (derived from the latest CommitQC). When no CommitQC has been // recorded yet, atypes.GlobalRangeOpt returns the committee's empty default @@ -132,8 +134,8 @@ func (r *GigaRouter) LastCommittedBlockNumber() int64 { // ResultBlockResults.ConsensusParamUpdates under Autobahn (where // FinalizeBlock responses are not stored on disk) without reaching into // the unexported router.cfg. -func (r *GigaRouter) MaxGasPerBlock() int64 { - return r.cfg.Producer.MaxGasPerBlockI64() +func (r *GigaRouter) MaxGasPerBlock() uint64 { + return r.cfg.Producer.MaxGasPerBlock } // BlockByNumber returns the finalized global block at height n translated @@ -245,7 +247,7 @@ func (r *GigaRouter) translateGlobalBlock(gb *atypes.GlobalBlock) *coretypes.Res } func (r *GigaRouter) executeBlock(ctx context.Context, b *atypes.GlobalBlock) (*abci.ResponseCommit, error) { - app := r.cfg.TxMempool.App() + app := r.cfg.Producer.App hash := b.Header.Hash() var proposerAddress types.Address if vals := app.GetValidators(); len(vals) > 0 { @@ -260,9 +262,6 @@ func (r *GigaRouter) executeBlock(ctx context.Context, b *atypes.GlobalBlock) (* } // TODO: add metrics to understand execution latency. - r.cfg.TxMempool.Lock() - defer r.cfg.TxMempool.Unlock() - resp, err := app.FinalizeBlock(ctx, &abci.RequestFinalizeBlock{ Txs: b.Payload.Txs(), // Empty DecidedLastCommit does not indicate missing votes. @@ -294,7 +293,7 @@ func (r *GigaRouter) executeBlock(ctx context.Context, b *atypes.GlobalBlock) (* } func (r *GigaRouter) runExecute(ctx context.Context) error { - app := r.cfg.TxMempool.App() + app := r.cfg.Producer.App info, err := app.Info(ctx, &version.RequestInfo) if err != nil { diff --git a/sei-tendermint/internal/p2p/giga_router_test.go b/sei-tendermint/internal/p2p/giga_router_test.go index 2c68c7f5b3..919855629a 100644 --- a/sei-tendermint/internal/p2p/giga_router_test.go +++ b/sei-tendermint/internal/p2p/giga_router_test.go @@ -21,7 +21,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/producer" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/conn" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" @@ -287,9 +286,6 @@ func TestGigaRouter_FinalizeBlocks(t *testing.T) { e := Endpoint{AddrPort: cfg.addr} app := newTestApp() proxyApp := proxy.New(app, proxy.NopMetrics()) - // In giga mode the CometBFT handshaker is skipped; the router's - // runExecute calls InitChain itself on fresh start. - txMempool := mempool.NewTxMempool(mempool.TestConfig(), proxyApp, mempool.NopMetrics(), mempool.NopTxConstraintsFetcher) router, err := NewRouter( NopMetrics(), cfg.nodeKey, @@ -312,14 +308,12 @@ func TestGigaRouter_FinalizeBlocks(t *testing.T) { PersistentStateDir: utils.None[string](), }, Producer: &producer.Config{ + App: proxyApp, MaxGasPerBlock: txGasUsed * maxTxsPerBlock, MaxTxsPerBlock: maxTxsPerBlock, MaxTxsPerSecond: utils.None[uint64](), - MempoolSize: 100, BlockInterval: 100 * time.Millisecond, - AllowEmptyBlocks: false, }, - TxMempool: txMempool, GenDoc: genDoc, }), }, @@ -335,8 +329,9 @@ func TestGigaRouter_FinalizeBlocks(t *testing.T) { allTxs = append(allTxs, tx) } s.SpawnNamed(fmt.Sprintf("producer[%v]", i), func() error { - for _, payload := range txs { - if _, err := txMempool.CheckTx(ctx, payload); err != nil { + giga := router.Giga().OrPanic("non-giga router") + for _, tx := range txs { + if _, err := giga.InsertTx(ctx, tx); err != nil { return fmt.Errorf("txMempool.CheckTx(): %w", err) } } @@ -359,8 +354,7 @@ func TestGigaRouter_FinalizeBlocks(t *testing.T) { // blocks have been finalized every node should report a non-zero // consensus-committed height through the new accessors used by /status. for i, r := range routers { - giga, ok := r.Giga().Get() - require.True(t, ok, "router[%v].Giga()", i) + giga := r.Giga().OrPanic("non-giga router") committed := giga.LastCommittedBlockNumber() require.Positive(t, committed, "router[%v].LastCommittedBlockNumber()", i) // Covers GigaRouter.BlockByNumber — the accessor used by the @@ -446,7 +440,6 @@ func TestGigaRouter_EvmProxy(t *testing.T) { } require.NoError(t, genDoc.ValidateAndComplete()) - txMempool := mempool.NewTxMempool(mempool.TestConfig(), proxy.New(newTestApp(), proxy.NopMetrics()), mempool.NopMetrics(), mempool.NopTxConstraintsFetcher) router, err := NewGigaRouter(&GigaRouterConfig{ DialInterval: time.Second, ValidatorAddrs: addrs, @@ -456,14 +449,12 @@ func TestGigaRouter_EvmProxy(t *testing.T) { PersistentStateDir: utils.None[string](), }, Producer: &producer.Config{ + App: proxy.New(newTestApp(), proxy.NopMetrics()), MaxGasPerBlock: 1, MaxTxsPerBlock: 1, MaxTxsPerSecond: utils.None[uint64](), - MempoolSize: 1, BlockInterval: time.Second, - AllowEmptyBlocks: false, }, - TxMempool: txMempool, GenDoc: genDoc, }, nodeKeys[0]) require.NoError(t, err) diff --git a/sei-tendermint/internal/p2p/router_test.go b/sei-tendermint/internal/p2p/router_test.go index f833fe976b..a2f1520391 100644 --- a/sei-tendermint/internal/p2p/router_test.go +++ b/sei-tendermint/internal/p2p/router_test.go @@ -19,7 +19,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/producer" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/conn" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" @@ -326,7 +325,6 @@ func TestRouter_GigaSetWhenConfigured(t *testing.T) { // Use intentionally non-default values to ensure config actually propagates. opts := makeRouterOptions() proxyApp := proxy.New(abci.BaseApplication{}, proxy.NopMetrics()) - txMempool := mempool.NewTxMempool(mempool.TestConfig(), proxyApp, mempool.NopMetrics(), mempool.NopTxConstraintsFetcher) opts.Giga = utils.Some(&GigaRouterConfig{ DialInterval: 7 * time.Second, ValidatorAddrs: validatorAddrs, @@ -336,14 +334,12 @@ func TestRouter_GigaSetWhenConfigured(t *testing.T) { PersistentStateDir: utils.None[string](), }, Producer: &producer.Config{ + App: proxyApp, MaxGasPerBlock: 77_000_000, MaxTxsPerBlock: 7_777, MaxTxsPerSecond: utils.Some(uint64(999)), - MempoolSize: 3_333, BlockInterval: 777 * time.Millisecond, - AllowEmptyBlocks: true, }, - TxMempool: txMempool, GenDoc: &types.GenesisDoc{ ChainID: "giga-e2e-test", InitialHeight: 42, @@ -375,9 +371,7 @@ func TestRouter_GigaSetWhenConfigured(t *testing.T) { maxTps, tpsOk := giga.cfg.Producer.MaxTxsPerSecond.Get() require.True(t, tpsOk) require.Equal(t, uint64(999), maxTps) - require.Equal(t, uint64(3_333), giga.cfg.Producer.MempoolSize) require.Equal(t, 777*time.Millisecond, giga.cfg.Producer.BlockInterval) - require.True(t, giga.cfg.Producer.AllowEmptyBlocks) // Verify genesis doc. require.Equal(t, "giga-e2e-test", giga.cfg.GenDoc.ChainID) diff --git a/sei-tendermint/internal/rpc/core/blocks.go b/sei-tendermint/internal/rpc/core/blocks.go index ebf24a99c2..7322493782 100644 --- a/sei-tendermint/internal/rpc/core/blocks.go +++ b/sei-tendermint/internal/rpc/core/blocks.go @@ -269,7 +269,9 @@ func (env *Environment) BlockResults(ctx context.Context, req *coretypes.Request return &coretypes.ResultBlockResults{ Height: height, ConsensusParamUpdates: &tmproto.ConsensusParams{ - Block: &tmproto.BlockParams{MaxGas: giga.MaxGasPerBlock()}, + Block: &tmproto.BlockParams { + MaxGas: utils.Clamp[int64](giga.MaxGasPerBlock()), + }, }, }, nil } diff --git a/sei-tendermint/node/node.go b/sei-tendermint/node/node.go index 573a9fda45..cf24af64b6 100644 --- a/sei-tendermint/node/node.go +++ b/sei-tendermint/node/node.go @@ -67,7 +67,7 @@ type nodeImpl struct { initialState sm.State stateStore sm.Store blockStore *store.BlockStore // store the blockchain to disk - mempool *mempool.TxMempool + mempool utils.Option[*mempool.TxMempool] evPool *evidence.Pool indexerService *indexer.Service services []service.Service @@ -193,14 +193,13 @@ func makeNode( return nil, fmt.Errorf("autobahn does not support remote validator signers (priv-validator.laddr is set)") } gigaEnabled := cfg.AutobahnConfigFile != "" - mp := mempool.NewTxMempool(cfg.Mempool.ToMempoolConfig(), proxyApp, nodeMetrics.mempool, sm.TxConstraintsFetcherFromStore(stateStore)) router, peerCloser, err := createRouter( nodeMetrics.p2p, node.NodeInfo, nodeKey, utils.Some(atypes.SecretKeyFromED25519(filePrivval.Key.PrivKey)), cfg, - utils.Some(mp), + utils.Some(proxyApp), genDoc, dbProvider, ) @@ -209,12 +208,13 @@ func makeNode( return nil, fmt.Errorf("failed to create router: %w", err) } node.router = router - node.mempool = mp node.rpcEnv.Router = router // Mempool gossiping is not compatible with Giga, // so we disable the mempool reactor. if !gigaEnabled { + mp := mempool.NewTxMempool(cfg.Mempool.ToMempoolConfig(), proxyApp, nodeMetrics.mempool, sm.TxConstraintsFetcherFromStore(stateStore)) + node.mempool = utils.Some(mp) mpReactor, err := mempoolreactor.NewReactor(cfg.Mempool, mp, router) if err != nil { return nil, fmt.Errorf("mempoolreactor.NewReactor(): %w", err) diff --git a/sei-tendermint/node/setup.go b/sei-tendermint/node/setup.go index cfd55f7f19..206c7d2518 100644 --- a/sei-tendermint/node/setup.go +++ b/sei-tendermint/node/setup.go @@ -20,7 +20,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/consensus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/eventbus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/evidence" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" mempoolreactor "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool/reactor" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/conn" @@ -194,7 +194,7 @@ func buildGigaConfig( autobahnConfigFile string, nodeKey types.NodeKey, validatorKey atypes.SecretKey, - txMempool *mempool.TxMempool, + app *proxy.Proxy, genDoc *types.GenesisDoc, ) (*p2p.GigaRouterConfig, error) { if autobahnConfigFile == "" { @@ -255,11 +255,8 @@ func buildGigaConfig( MaxGasPerBlock: maxGasPerBlock, MaxTxsPerBlock: fc.MaxTxsPerBlock, MaxTxsPerSecond: fc.MaxTxsPerSecond, - MempoolSize: fc.MempoolSize, BlockInterval: time.Duration(fc.BlockInterval), - AllowEmptyBlocks: fc.AllowEmptyBlocks, }, - TxMempool: txMempool, GenDoc: genDoc, }, nil } @@ -270,7 +267,7 @@ func createRouter( nodeKey types.NodeKey, validatorKey utils.Option[atypes.SecretKey], cfg *config.Config, - txMempool utils.Option[*mempool.TxMempool], + app utils.Option[*proxy.Proxy], genDoc *types.GenesisDoc, dbProvider config.DBProvider, ) (*p2p.Router, closer, error) { @@ -362,11 +359,11 @@ func createRouter( if !ok { return nil, closer, fmt.Errorf("autobahn non-validator nodes are not supported yet; a local validator key is required") } - mp, ok := txMempool.Get() + app,ok := app.Get() if !ok { - return nil, closer, errors.New("autobahn requires a tx mempool") + return nil, closer, fmt.Errorf("autobahn requires app") } - gigaCfg, err := buildGigaConfig(cfg.AutobahnConfigFile, nodeKey, valKey, mp, genDoc) + gigaCfg, err := buildGigaConfig(cfg.AutobahnConfigFile, nodeKey, valKey, app, genDoc) if err != nil { return nil, closer, fmt.Errorf("buildGigaConfig: %w", err) } From 090509c0e4eb7482fcbe5b2d703b185560c5c3d9 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 27 May 2026 17:10:52 +0200 Subject: [PATCH 068/100] WIP --- sei-tendermint/internal/blocksync/doc.go | 27 +- sei-tendermint/internal/blocksync/pool.go | 35 +- .../internal/blocksync/pool_test.go | 49 +- sei-tendermint/internal/blocksync/reactor.go | 663 ++++++++++-------- .../internal/blocksync/reactor_test.go | 132 +++- sei-tendermint/internal/rpc/core/env.go | 2 +- sei-tendermint/node/node.go | 18 +- 7 files changed, 544 insertions(+), 382 deletions(-) diff --git a/sei-tendermint/internal/blocksync/doc.go b/sei-tendermint/internal/blocksync/doc.go index 5f84b1261c..2c32741034 100644 --- a/sei-tendermint/internal/blocksync/doc.go +++ b/sei-tendermint/internal/blocksync/doc.go @@ -1,7 +1,7 @@ /* -Package blocksync implements two versions of a reactor Service that are -responsible for block propagation and gossip between peers. This mechanism was -formerly known as fast-sync. +Package blocksync implements the blocksync protocol used for serving block +requests and catching up to the network head. This mechanism was formerly known +as fast-sync. In order for a full node to successfully participate in consensus, it must have the latest view of state. The blocksync protocol is a mechanism in which peers @@ -13,19 +13,16 @@ will no longer blocksync and thus no longer run the blocksync process. Note, the blocksync reactor Service gossips entire block and relevant data such that each receiving peer may construct the entire view of the blocksync state. -There is currently only one version of the blocksync reactor Service -that is battle-tested, but whose test coverage is lacking and is not -formally verified. +There is currently one blocksync protocol implementation. Internally the +top-level Reactor owns the single blocksync p2p channel and the always-on query +serving path: -The v0 blocksync reactor Service has one p2p channel, BlockchainChannel. This -channel is responsible for handling messages that both request blocks and respond -to block requests from peers. For every block request from a peer, the reactor -will execute respondToPeer which will fetch the block from the node's state store -and respond to the peer. For every block response, the node will add the block -to its pool via AddBlock. +- serve inbound BlockRequest and StatusRequest messages +- advertise local status to newly connected peers -Internally, v0 runs a poolRoutine that constantly checks for what blocks it needs -and requests them. The poolRoutine is also responsible for taking blocks from the -pool, saving and executing each block. +Active syncing itself is handled by a separate sync controller that manages the +block pool, requests blocks, applies them locally, and hands off to consensus +once caught up. Sync-specific responses received on the shared channel are +forwarded into that controller. */ package blocksync diff --git a/sei-tendermint/internal/blocksync/pool.go b/sei-tendermint/internal/blocksync/pool.go index 83b4c59004..cf44719f6d 100644 --- a/sei-tendermint/internal/blocksync/pool.go +++ b/sei-tendermint/internal/blocksync/pool.go @@ -12,6 +12,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/libs/flowrate" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/service" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" "github.com/sei-protocol/sei-chain/sei-tendermint/types" "github.com/sei-protocol/seilog" ) @@ -82,8 +83,6 @@ type BlockRequest struct { // BlockPool keeps track of the block sync peers, block requests and block responses. type BlockPool struct { - service.BaseService - lastAdvance time.Time mtx sync.RWMutex @@ -105,6 +104,7 @@ type BlockPool struct { lastHundredBlockTimeStamp time.Time lastSyncRate float64 cancels []context.CancelFunc + running *atomicBool } // NewBlockPool returns a new BlockPool with the height equal to start. Block @@ -125,27 +125,34 @@ func NewBlockPool( errorsCh: errorsCh, lastSyncRate: 0, router: router, + running: newAtomicBool(false), } - bp.BaseService = *service.NewBaseService("BlockPool", bp) return bp } -// OnStart implements service.Service by spawning requesters routine and recording -// pool's start time. -func (pool *BlockPool) OnStart(ctx context.Context) error { +// run owns the pool's requester-management loop and its cleanup. Starting the +// task marks the pool active; exiting the task stops all outstanding requester +// work and marks the pool inactive. +func (pool *BlockPool) run(ctx context.Context) error { + pool.running.Set() + defer pool.shutdown() + pool.lastAdvance = time.Now() pool.lastHundredBlockTimeStamp = pool.lastAdvance - pool.Spawn("makeRequestersRoutine", func(ctx context.Context) error { - pool.makeRequestersRoutine(ctx) + + return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { + s.SpawnNamed("makeRequestersRoutine", func() error { + pool.makeRequestersRoutine(ctx) + return nil + }) return nil }) - - return nil } -func (pool *BlockPool) OnStop() { +func (pool *BlockPool) shutdown() { + pool.running.UnSet() // Requester shutdown must not block behind a full requestsCh; Stop cancels ctx - // and waits for the Spawn-managed requester goroutine to exit. + // and waits for the requester-management loop to observe pool.running=false. pool.mtx.Lock() cancels := pool.cancels pool.cancels = nil @@ -164,6 +171,10 @@ func (pool *BlockPool) OnStop() { } } +func (pool *BlockPool) IsRunning() bool { + return pool.running.IsSet() +} + // spawns requesters as needed func (pool *BlockPool) makeRequestersRoutine(ctx context.Context) { for pool.IsRunning() { diff --git a/sei-tendermint/internal/blocksync/pool_test.go b/sei-tendermint/internal/blocksync/pool_test.go index ce0502dcff..3223fa4d01 100644 --- a/sei-tendermint/internal/blocksync/pool_test.go +++ b/sei-tendermint/internal/blocksync/pool_test.go @@ -1,8 +1,10 @@ package blocksync import ( + "context" "crypto/rand" "encoding/hex" + "errors" "fmt" "math" "runtime" @@ -102,20 +104,26 @@ func makeRouter(peers map[types.NodeID]testPeer) *fakeRouter { } } -func TestBlockPoolBasic(t *testing.T) { - ctx := t.Context() +func runPoolForTest(t *testing.T, pool *BlockPool) { + done := make(chan error, 1) + go func() { + done <- pool.run(t.Context()) + }() + t.Cleanup(func() { + if err := <-done; err != nil && !errors.Is(err, context.Canceled) { + t.Fatalf("pool.run(): %v", err) + } + }) +} +func TestBlockPoolBasic(t *testing.T) { start := int64(42) peers := makePeers(10, start, 1000) errorsCh := make(chan peerError, 1000) requestsCh := make(chan BlockRequest, 1000) pool := NewBlockPool(start, requestsCh, errorsCh, makeRouter(peers)) - if err := pool.Start(ctx); err != nil { - t.Error(err) - } - - t.Cleanup(func() { pool.Wait() }) + runPoolForTest(t, pool) peers.start() defer peers.stop() @@ -130,7 +138,7 @@ func TestBlockPoolBasic(t *testing.T) { // Start a goroutine to pull blocks go func() { for { - if !pool.IsRunning() { + if t.Context().Err() != nil { return } first, second := pool.PeekTwoBlocks() @@ -158,17 +166,12 @@ func TestBlockPoolBasic(t *testing.T) { } func TestBlockPoolTimeout(t *testing.T) { - ctx := t.Context() - start := int64(42) peers := makePeers(10, start, 1000) errorsCh := make(chan peerError, 1000) requestsCh := make(chan BlockRequest, 1000) pool := NewBlockPool(start, requestsCh, errorsCh, makeRouter(peers)) - err := pool.Start(ctx) - if err != nil { - t.Error(err) - } + runPoolForTest(t, pool) // Introduce each peer. go func() { @@ -180,7 +183,7 @@ func TestBlockPoolTimeout(t *testing.T) { // Start a goroutine to pull blocks go func() { for { - if !pool.IsRunning() { + if t.Context().Err() != nil { return } first, second := pool.PeekTwoBlocks() @@ -197,8 +200,6 @@ func TestBlockPoolTimeout(t *testing.T) { } func TestBlockPoolRemovePeer(t *testing.T) { - ctx := t.Context() - peers := make(testPeers, 10) for i := range 10 { var peerID types.NodeID @@ -214,9 +215,7 @@ func TestBlockPoolRemovePeer(t *testing.T) { errorsCh := make(chan peerError) pool := NewBlockPool(1, requestsCh, errorsCh, makeRouter(peers)) - err := pool.Start(ctx) - require.NoError(t, err) - t.Cleanup(func() { pool.Wait() }) + runPoolForTest(t, pool) // add peers for peerID, peer := range peers { @@ -272,8 +271,6 @@ func TestBlockPoolRejectsWrongPeerWithoutDiscardingGoodBlock(t *testing.T) { } func testBlockPoolRejectsWrongPeerWithoutDiscardingGoodBlock(t *testing.T, goodFirst bool) { - ctx := t.Context() - goodPeerID := types.NodeID(strings.Repeat("a", 40)) badPeerID := types.NodeID(strings.Repeat("b", 40)) peers := testPeers{ @@ -283,8 +280,7 @@ func testBlockPoolRejectsWrongPeerWithoutDiscardingGoodBlock(t *testing.T, goodF errorsCh := make(chan peerError, 2) pool := NewBlockPool(1, requestsCh, errorsCh, makeRouter(peers)) - require.NoError(t, pool.Start(ctx)) - t.Cleanup(func() { pool.Wait() }) + runPoolForTest(t, pool) pool.SetPeerRange(goodPeerID, 1, 2) @@ -329,16 +325,13 @@ func testBlockPoolRejectsWrongPeerWithoutDiscardingGoodBlock(t *testing.T, goodF // parked there. Without the fix, AddBlock holds pool.mtx for the // duration of the parked send and TryLock never succeeds. func TestBlockPoolAddBlockReleasesLockBeforeSend(t *testing.T) { - ctx := t.Context() - peerID := types.NodeID(strings.Repeat("a", 40)) peers := testPeers{peerID: {peerID, 1, 100, make(chan inputData)}} errorsCh := make(chan peerError) requestsCh := make(chan BlockRequest, 1000) pool := NewBlockPool(1, requestsCh, errorsCh, makeRouter(peers)) - require.NoError(t, pool.Start(ctx)) - t.Cleanup(func() { pool.Wait() }) + runPoolForTest(t, pool) pool.SetPeerRange(peerID, 1, 100) diff --git a/sei-tendermint/internal/blocksync/reactor.go b/sei-tendermint/internal/blocksync/reactor.go index 0956b5b6bc..38f090e346 100644 --- a/sei-tendermint/internal/blocksync/reactor.go +++ b/sei-tendermint/internal/blocksync/reactor.go @@ -16,6 +16,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/store" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/service" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" pb "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/blocksync" "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) @@ -23,7 +24,7 @@ import ( var _ service.Service = (*Reactor)(nil) const ( - // BlockSyncChannel is a channel for blocks and status updates + // BlockSyncChannel is a channel for blocks and status updates. BlockSyncChannel = p2p.ChannelID(0x40) trySyncIntervalMS = 10 @@ -38,6 +39,14 @@ const ( syncTimeout = 180 * time.Second ) +// Metricer is the RPC-facing blocksync surface. The facade and any future +// replacement can expose sync progress without leaking the concrete type. +type Metricer interface { + GetMaxPeerBlockHeight() int64 + GetTotalSyncedTime() time.Duration + GetRemainingSyncTime() time.Duration +} + // TODO(gprusak): that's not sufficient - parsing proto requires checking nils everywhere. func wrap[T *pb.BlockRequest | *pb.NoBlockResponse | *pb.BlockResponse | *pb.StatusRequest | *pb.StatusResponse](msg T) *pb.Message { switch msg := any(msg).(type) { @@ -60,7 +69,7 @@ func GetChannelDescriptor() p2p.ChannelDescriptor[*pb.Message] { return p2p.ChannelDescriptor[*pb.Message]{ ID: BlockSyncChannel, MessageType: new(pb.Message), - PreDecode: utils.Some[func([]byte) error](pb.SchemaForMessage.Scan), + PreDecode: utils.Some(pb.SchemaForMessage.Scan), Priority: 5, SendQueueCapacity: 1000, RecvBufferCapacity: 1024, @@ -86,104 +95,132 @@ func (e peerError) Error() string { type blocksyncResult struct{ stateSynced bool } -// Reactor handles long-term catchup syncing. +// SyncerConfig groups dependencies and startup knobs used only by the active +// blocksync controller. Reactor itself does not need these when running as an +// always-on query responder only. +type SyncerConfig struct { + BlockExec *sm.BlockExecutor + ConsReactor consensusReactor + BlockSync bool + Metrics *consensus.Metrics + EventBus *eventbus.EventBus + RestartEvent func() + SelfRemediationConfig *config.SelfRemediationConfig +} + +// Reactor owns the blocksync channel and always-on query serving path, while +// delegating active sync responsibilities to a separate sync controller. type Reactor struct { service.BaseService - // immutable - initialState sm.State - // store + // stateStore and store back both the query-serving path and the sync + // controller. They stay on the facade because inbound requests are served + // directly from Reactor even when local blocksync is inactive. stateStore sm.Store + store sm.BlockStore - blockExec *sm.BlockExecutor - store sm.BlockStore - pool *BlockPool - consReactor consensusReactor - blockSync *atomicBool - previousMaxPeerHeight int64 - - // blocksyncReady fires when blocksync should start processing blocks — - // either at OnStart (if blockSync was initially set) or via - // SwitchToBlockSync. Pre-spawned requestRoutine and poolRoutine wait on - // it before doing any work. - blocksyncReady utils.AtomicSend[utils.Option[blocksyncResult]] - // consensusReady fires once the blocksync->consensus handoff has - // happened. The pre-spawned autoRestartIfBehind monitor gates on this - // signal. - consensusReady utils.AtomicSend[bool] - + // Reactor is the sole owner of the blocksync channel because the router + // allows a channel ID to be opened only once. router *p2p.Router channel *p2p.Channel[*pb.Message] - requestsCh <-chan BlockRequest - errorsCh <-chan peerError + // syncer owns all active catch-up responsibilities: pool management, + // outgoing requests, block execution, consensus handoff, and lag metrics. + syncer utils.Option[*syncController] +} - metrics *consensus.Metrics - eventBus *eventbus.EventBus +type syncController struct { + // Immutable dependencies for the active sync path. + stateStore sm.Store + blockExec *sm.BlockExecutor + store sm.BlockStore + router *p2p.Router + channel *p2p.Channel[*pb.Message] + consReactor consensusReactor + metrics *consensus.Metrics + eventBus *eventbus.EventBus + + // Mutable sync state initialized on start and updated as blocksync runs. + initialState sm.State + pool *BlockPool + requestsCh <-chan BlockRequest + errorsCh <-chan peerError - syncStartTime time.Time + // blockSync tracks whether the node is actively in blocksync mode. The + // channel responder stays up regardless of this flag. + blockSync *atomicBool + previousMaxPeerHeight int64 + syncStartTime time.Time + // Auto-remediation configuration and restart bookkeeping. restartEvent func() lastRestartTime time.Time blocksBehindThreshold uint64 blocksBehindCheckInterval time.Duration restartCooldownSeconds uint64 + + // blocksyncReady fires when the active sync routines should begin processing + // work, either during OnStart or later via SwitchToBlockSync. + blocksyncReady utils.AtomicSend[utils.Option[blocksyncResult]] + // consensusReady fires after blocksync hands off to consensus so the + // auto-restart monitor can start observing lag from that point forward. + consensusReady utils.AtomicSend[bool] } // NewReactor returns new reactor instance. func NewReactor( stateStore sm.Store, - blockExec *sm.BlockExecutor, store *store.BlockStore, - consReactor consensusReactor, router *p2p.Router, - blockSync bool, - metrics *consensus.Metrics, - eventBus *eventbus.EventBus, - restartEvent func(), // should be idempotent and non-blocking - selfRemediationConfig *config.SelfRemediationConfig, + syncerConfig utils.Option[SyncerConfig], ) (*Reactor, error) { channel, err := p2p.OpenChannel(router, GetChannelDescriptor()) if err != nil { return nil, fmt.Errorf("router.AddChannel(): %w", err) } + + syncer := utils.None[*syncController]() + if cfg, ok := syncerConfig.Get(); ok { + syncer = utils.Some(&syncController{ + stateStore: stateStore, + blockExec: cfg.BlockExec, + store: store, + router: router, + channel: channel, + consReactor: cfg.ConsReactor, + metrics: cfg.Metrics, + eventBus: cfg.EventBus, + blockSync: newAtomicBool(cfg.BlockSync), + restartEvent: cfg.RestartEvent, + lastRestartTime: time.Now(), + blocksBehindThreshold: cfg.SelfRemediationConfig.BlocksBehindThreshold, + blocksBehindCheckInterval: time.Duration(cfg.SelfRemediationConfig.BlocksBehindCheckIntervalSeconds) * time.Second, //nolint:gosec // validated in config.ValidateBasic against MaxInt64 + restartCooldownSeconds: cfg.SelfRemediationConfig.RestartCooldownSeconds, + blocksyncReady: utils.NewAtomicSend(utils.None[blocksyncResult]()), + consensusReady: utils.NewAtomicSend(false), + }) + } + r := &Reactor{ - stateStore: stateStore, - blockExec: blockExec, - store: store, - consReactor: consReactor, - blockSync: newAtomicBool(blockSync), - router: router, - channel: channel, - metrics: metrics, - eventBus: eventBus, - restartEvent: restartEvent, - lastRestartTime: time.Now(), - blocksBehindThreshold: selfRemediationConfig.BlocksBehindThreshold, - blocksBehindCheckInterval: time.Duration(selfRemediationConfig.BlocksBehindCheckIntervalSeconds) * time.Second, //nolint:gosec // validated in config.ValidateBasic against MaxInt64 - restartCooldownSeconds: selfRemediationConfig.RestartCooldownSeconds, - blocksyncReady: utils.NewAtomicSend(utils.None[blocksyncResult]()), - consensusReady: utils.NewAtomicSend(false), + stateStore: stateStore, + store: store, + router: router, + channel: channel, + syncer: syncer, } r.BaseService = *service.NewBaseService("BlockSync", r) return r, nil } -// OnStart starts separate go routines for each p2p Channel and listens for -// envelopes on each. In addition, it also listens for peer updates and handles -// messages on that p2p channel accordingly. The caller must be sure to execute -// OnStop to ensure the outbound p2p Channels are closed. -// -// If blockSync is enabled, we also start the pool and the pool processing -// goroutine. If the pool fails to start, an error is returned. +// OnStart starts the always-on query handling loops and one sync controller +// supervisor task. The active sync routines inside that controller remain +// gated until blocksync is enabled, either on startup or via +// SwitchToBlockSync after state sync. func (r *Reactor) OnStart(ctx context.Context) error { state, err := r.stateStore.Load() if err != nil { return err } - r.initialState = state - r.lastRestartTime = time.Now() - if state.LastBlockHeight != r.store.Height() { return fmt.Errorf("state (%v) and store (%v) height mismatch", state.LastBlockHeight, r.store.Height()) } @@ -193,38 +230,9 @@ func (r *Reactor) OnStart(ctx context.Context) error { startHeight = state.InitialHeight } - requestsCh := make(chan BlockRequest, maxTotalRequesters) - errorsCh := make(chan peerError, maxPeerErrBuffer) // NOTE: The capacity should be larger than the peer count. - r.pool = NewBlockPool(startHeight, requestsCh, errorsCh, r.router) - r.requestsCh = requestsCh - r.errorsCh = errorsCh - - // Pre-spawn all long-running routines so their lifetime is bound to the - // BaseService WaitGroup. Conditional routines gate on AtomicSend[bool] - // signals so SwitchToBlockSync (and the in-poolRoutine consensus handoff - // for autoRestartIfBehind) can wake them later without spawning fresh - // goroutines from outside OnStart. - r.Spawn("requestRoutine", func(ctx context.Context) error { - _, err := r.blocksyncReady.Wait(ctx, func(o utils.Option[blocksyncResult]) bool { - return o.IsPresent() - }) - if err != nil { - return err - } - r.requestRoutine(ctx) - return nil - }) - r.Spawn("poolRoutine", func(ctx context.Context) error { - result, err := r.blocksyncReady.Wait(ctx, func(o utils.Option[blocksyncResult]) bool { - return o.IsPresent() - }) - if err != nil { - return err - } - res := result.OrPanic("no blocksync result") - r.poolRoutine(ctx, res.stateSynced) - return nil - }) + if syncer, ok := r.syncer.Get(); ok { + syncer.initialize(state, startHeight) + } r.SpawnCritical("processBlockSyncCh", func(ctx context.Context) error { r.processBlockSyncCh(ctx) return nil @@ -233,34 +241,63 @@ func (r *Reactor) OnStart(ctx context.Context) error { r.processPeerUpdates(ctx) return nil }) - r.SpawnCritical("autoRestartIfBehind", func(ctx context.Context) error { - if _, err := r.consensusReady.Wait(ctx, func(ready bool) bool { return ready }); err != nil { - return err - } - r.autoRestartIfBehind(ctx) - return nil - }) - - if r.blockSync.IsSet() { - if err := r.pool.Start(ctx); err != nil { - return err + if syncer, ok := r.syncer.Get(); ok { + r.SpawnCritical("syncController.run", func(ctx context.Context) error { + return syncer.run(ctx) + }) + if syncer.blockSync.IsSet() { + syncer.blocksyncReady.Store(utils.Some(blocksyncResult{false})) } - r.blocksyncReady.Store(utils.Some(blocksyncResult{false})) } return nil } -// OnStop stops the BlockPool. The reactor's own long-running goroutines were -// registered with the BaseService WaitGroup via Spawn in OnStart, so the -// BaseService blocks Stop() on their exit before this method returns. -func (r *Reactor) OnStop() { - if r.blockSync.IsSet() { - r.pool.Stop() +// OnStop relies on the query loops and sync controller supervisor being +// registered with BaseService via Spawn. Their internal cleanup runs as those +// tasks exit. +func (r *Reactor) OnStop() {} + +// SwitchToBlockSync is called by the state sync reactor when switching to fast +// sync. +func (r *Reactor) SwitchToBlockSync(state sm.State) error { + syncer, ok := r.syncer.Get() + if !ok { + return errors.New("blocksync syncer is not configured") + } + return syncer.switchToBlockSync(state) +} + +func (r *Reactor) GetMaxPeerBlockHeight() int64 { + if syncer, ok := r.syncer.Get(); ok { + return syncer.GetMaxPeerBlockHeight() + } + return 0 +} + +func (r *Reactor) GetTotalSyncedTime() time.Duration { + if syncer, ok := r.syncer.Get(); ok { + return syncer.GetTotalSyncedTime() + } + return 0 +} + +func (r *Reactor) GetRemainingSyncTime() time.Duration { + if syncer, ok := r.syncer.Get(); ok { + return syncer.GetRemainingSyncTime() } + return 0 +} + +func (r *Reactor) PublishStatus(event types.EventDataBlockSyncStatus) error { + syncer, ok := r.syncer.Get() + if !ok { + return errors.New("blocksync syncer is not configured") + } + return syncer.PublishStatus(event) } // respondToPeer loads a block and sends it to the requesting peer, if we have it. -// Otherwise, we'll respond saying we do not have it. +// Otherwise, it responds saying we do not have it. func (r *Reactor) respondToPeer(msg *pb.BlockRequest, peerID types.NodeID) error { block := r.store.LoadBlock(msg.GetHeight()) if block == nil { @@ -278,9 +315,9 @@ func (r *Reactor) respondToPeer(msg *pb.BlockRequest, peerID types.NodeID) error return nil } -// handleMessage handles an Envelope sent from a peer on a specific p2p Channel. -// It will handle errors and any possible panics gracefully. A caller can handle -// any error returned by sending a PeerError on the respective channel. +// handleMessage handles an inbound blocksync message. Reactor only owns block +// request serving; every other blocksync message is forwarded to the sync +// controller. func (r *Reactor) handleMessage(m p2p.RecvMsg[*pb.Message]) (err error) { defer func() { if e := recover(); e != nil { @@ -295,43 +332,17 @@ func (r *Reactor) handleMessage(m p2p.RecvMsg[*pb.Message]) (err error) { logger.Debug("received message", "message", m.Message, "peer", m.From) - switch msg := m.Message.Sum.(type) { - case *pb.Message_BlockRequest: + if msg, ok := m.Message.Sum.(*pb.Message_BlockRequest); ok { return r.respondToPeer(msg.BlockRequest, m.From) - case *pb.Message_BlockResponse: - block, err := types.BlockFromProto(msg.BlockResponse.GetBlock()) - if err != nil { - return fmt.Errorf("types.BlockFromProto(): %w", err) - } - logger.Info("received block response from peer", "peer", m.From, "height", block.Height) - if err := r.pool.AddBlock(m.From, block, block.Size()); err != nil { - logger.Error("failed to add block", "err", err) - } - return nil - case *pb.Message_StatusRequest: - r.channel.Send(wrap(&pb.StatusResponse{ - Height: r.store.Height(), - Base: r.store.Base(), - }), m.From) - return nil - case *pb.Message_StatusResponse: - r.pool.SetPeerRange(m.From, msg.StatusResponse.GetBase(), msg.StatusResponse.GetHeight()) - return nil - case *pb.Message_NoBlockResponse: - logger.Debug("peer does not have the requested block", - "peer", m.From, - "height", msg.NoBlockResponse.GetHeight()) + } + syncer, ok := r.syncer.Get() + if !ok { return nil - default: - return fmt.Errorf("received unknown message: %T", msg) } + return syncer.handleMessage(m) } -// processBlockSyncCh initiates a blocking process where we listen for and handle -// envelopes on the BlockSyncChannel and blockSyncOutBridgeCh. Any error encountered during -// message execution will result in a PeerError being sent on the BlockSyncChannel. -// When the reactor is stopped, we will catch the signal and close the p2p Channel -// gracefully. +// processBlockSyncCh listens for messages on the shared blocksync channel. func (r *Reactor) processBlockSyncCh(ctx context.Context) { for ctx.Err() == nil { m, err := r.channel.Recv(ctx) @@ -344,69 +355,28 @@ func (r *Reactor) processBlockSyncCh(ctx context.Context) { } } -// autoRestartIfBehind will check if the node is behind the max peer height by -// a certain threshold. If it is, the node will attempt to restart itself -// TODO(gprusak): this should be a sub task of the consensus reactor instead. -func (r *Reactor) autoRestartIfBehind(ctx context.Context) { - if r.blocksBehindThreshold == 0 || r.blocksBehindCheckInterval <= 0 { - logger.Info("Auto remediation is disabled") - return - } - - logger.Info("checking if node is behind threshold, auto restarting if its behind", "threshold", r.blocksBehindThreshold, "interval", r.blocksBehindCheckInterval) - for { - select { - case <-time.After(r.blocksBehindCheckInterval): - selfHeight := r.store.Height() - maxPeerHeight := r.pool.MaxPeerHeight() - threshold := int64(r.blocksBehindThreshold) //nolint:gosec // validated in config.ValidateBasic against MaxInt64 - behindHeight := maxPeerHeight - selfHeight - blockSyncIsSet := r.blockSync.IsSet() - if maxPeerHeight > r.previousMaxPeerHeight { - r.previousMaxPeerHeight = maxPeerHeight - } - - // We do not restart if we are not lagging behind, or we are already in block sync mode - if maxPeerHeight == 0 || behindHeight < threshold || blockSyncIsSet { - logger.Debug("does not exceed threshold or is already in block sync mode", "threshold", threshold, "behindHeight", behindHeight, "maxPeerHeight", maxPeerHeight, "selfHeight", selfHeight, "blockSyncIsSet", blockSyncIsSet) - continue - } - // Check if we have met cooldown time - if time.Since(r.lastRestartTime).Seconds() < float64(r.restartCooldownSeconds) { - logger.Debug("we are lagging behind, going to trigger a restart after cooldown time passes") - continue - } - logger.Info("Blocks behind threshold, restarting node", "threshold", threshold, "behindHeight", behindHeight, "maxPeerHeight", maxPeerHeight, "selfHeight", selfHeight) - - // Send signal to restart the node - r.blockSync.Set() - r.restartEvent() - return - case <-ctx.Done(): - return - } - } -} - -// processPeerUpdate processes a PeerUpdate. +// processPeerUpdate handles the subset of peer lifecycle needed by blocksync: +// advertise our local range on connect and let the sync controller clean up any +// in-flight state on disconnect. func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) { logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status) switch peerUpdate.Status { case p2p.PeerStatusUp: - // send a status update the newly added peer r.channel.Send(wrap(&pb.StatusResponse{ Base: r.store.Base(), Height: r.store.Height(), }), peerUpdate.NodeID) case p2p.PeerStatusDown: - r.pool.RemovePeer(peerUpdate.NodeID) + if syncer, ok := r.syncer.Get(); ok { + syncer.handlePeerDown(peerUpdate.NodeID) + } } } -// processPeerUpdates initiates a blocking process where we listen for and handle -// PeerUpdate messages. When the reactor is stopped, we will catch the signal and -// close the p2p PeerUpdatesCh gracefully. +// processPeerUpdates listens for peer updates. The reactor owns peer-up +// status announcements; the sync controller only receives peer-down callbacks +// for pool cleanup. func (r *Reactor) processPeerUpdates(ctx context.Context) { recv := r.router.Subscribe() for { @@ -418,21 +388,113 @@ func (r *Reactor) processPeerUpdates(ctx context.Context) { } } -// SwitchToBlockSync is called by the state sync reactor when switching to fast -// sync. -func (r *Reactor) SwitchToBlockSync(ctx context.Context, state sm.State) error { - r.blockSync.Set() - r.initialState = state - r.pool.height = state.LastBlockHeight + 1 +// initialize prepares the active sync controller state for a given starting +// height before any gated sync goroutines begin work. +func (s *syncController) initialize(initialState sm.State, startHeight int64) { + requestsCh := make(chan BlockRequest, maxTotalRequesters) + errorsCh := make(chan peerError, maxPeerErrBuffer) // NOTE: capacity should exceed peer count. - if err := r.pool.Start(ctx); err != nil { - return err + s.initialState = initialState + s.lastRestartTime = time.Now() + s.pool = NewBlockPool(startHeight, requestsCh, errorsCh, s.router) + s.requestsCh = requestsCh + s.errorsCh = errorsCh +} + +// run owns the active sync controller's internal concurrency. A single +// coordinator task waits for blocksyncReady, then starts the pool and spawns +// the active sync subtasks for that session. +func (s *syncController) run(ctx context.Context) error { + return scope.Run(ctx, func(ctx context.Context, sc scope.Scope) error { + sc.SpawnNamed("blocksyncSession", func() error { + result, err := s.blocksyncReady.Wait(ctx, func(o utils.Option[blocksyncResult]) bool { + return o.IsPresent() + }) + if err != nil { + return err + } + + return scope.Run(ctx, func(ctx context.Context, session scope.Scope) error { + session.SpawnNamed("pool.run", func() error { + return s.pool.run(ctx) + }) + session.SpawnNamed("requestRoutine", func() error { + s.requestRoutine(ctx) + return nil + }) + session.SpawnNamed("poolRoutine", func() error { + s.poolRoutine(ctx, result.OrPanic("no blocksync result").stateSynced) + return nil + }) + return nil + }) + }) + sc.SpawnNamed("autoRestartIfBehind", func() error { + if _, err := s.consensusReady.Wait(ctx, func(ready bool) bool { return ready }); err != nil { + return err + } + s.autoRestartIfBehind(ctx) + return nil + }) + return nil + }) +} + +// handleMessage processes all non-BlockRequest blocksync protocol messages. +func (s *syncController) handleMessage(m p2p.RecvMsg[*pb.Message]) error { + switch msg := m.Message.Sum.(type) { + case *pb.Message_BlockResponse: + block, err := types.BlockFromProto(msg.BlockResponse.GetBlock()) + if err != nil { + return fmt.Errorf("types.BlockFromProto(): %w", err) + } + return s.handleBlockResponse(m.From, block) + case *pb.Message_StatusRequest: + s.channel.Send(wrap(&pb.StatusResponse{ + Height: s.store.Height(), + Base: s.store.Base(), + }), m.From) + return nil + case *pb.Message_StatusResponse: + s.handleStatusResponse(m.From, msg.StatusResponse.GetBase(), msg.StatusResponse.GetHeight()) + return nil + case *pb.Message_NoBlockResponse: + s.handleNoBlockResponse(m.From, msg.NoBlockResponse.GetHeight()) + return nil + default: + return fmt.Errorf("received unknown message: %T", msg) + } +} + +func (s *syncController) handleBlockResponse(peerID types.NodeID, block *types.Block) error { + logger.Info("received block response from peer", "peer", peerID, "height", block.Height) + if err := s.pool.AddBlock(peerID, block, block.Size()); err != nil { + logger.Error("failed to add block", "err", err) } + return nil +} - r.syncStartTime = time.Now() - r.blocksyncReady.Store(utils.Some(blocksyncResult{true})) +func (s *syncController) handleStatusResponse(peerID types.NodeID, base, height int64) { + s.pool.SetPeerRange(peerID, base, height) +} + +func (s *syncController) handleNoBlockResponse(peerID types.NodeID, height int64) { + logger.Debug("peer does not have the requested block", "peer", peerID, "height", height) +} - if err := r.PublishStatus(types.EventDataBlockSyncStatus{ +func (s *syncController) handlePeerDown(peerID types.NodeID) { + s.pool.RemovePeer(peerID) +} + +func (s *syncController) switchToBlockSync(state sm.State) error { + s.blockSync.Set() + s.initialState = state + s.pool.height = state.LastBlockHeight + 1 + + s.syncStartTime = time.Now() + s.blocksyncReady.Store(utils.Some(blocksyncResult{true})) + + if err := s.PublishStatus(types.EventDataBlockSyncStatus{ Complete: false, Height: state.LastBlockHeight, }); err != nil { @@ -442,27 +504,29 @@ func (r *Reactor) SwitchToBlockSync(ctx context.Context, state sm.State) error { return nil } -func (r *Reactor) requestRoutine(ctx context.Context) { +func (s *syncController) requestRoutine(ctx context.Context) { statusUpdateTicker := time.NewTicker(statusUpdateInterval) + defer statusUpdateTicker.Stop() + for { select { case <-ctx.Done(): return - case request := <-r.requestsCh: - r.channel.Send(wrap(&pb.BlockRequest{Height: request.Height}), request.PeerID) - case pErr := <-r.errorsCh: - r.router.Evict(pErr.peerID, fmt.Errorf("blocksync.request: %w", pErr.err)) + case request := <-s.requestsCh: + s.channel.Send(wrap(&pb.BlockRequest{Height: request.Height}), request.PeerID) + case pErr := <-s.errorsCh: + s.router.Evict(pErr.peerID, fmt.Errorf("blocksync.request: %w", pErr.err)) case <-statusUpdateTicker.C: - r.channel.Broadcast(wrap(&pb.StatusRequest{})) + s.channel.Broadcast(wrap(&pb.StatusRequest{})) } } } -// poolRoutine handles messages from the poolReactor telling the reactor what to -// do. +// poolRoutine handles messages from the poolReactor telling the controller what +// to do. // // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down! -func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool) { +func (s *syncController) poolRoutine(ctx context.Context, stateSynced bool) { var ( trySyncTicker = time.NewTicker(trySyncIntervalMS * time.Millisecond) switchToConsensusTicker = time.NewTicker(switchToConsensusIntervalSeconds * time.Second) @@ -470,8 +534,8 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool) { blocksSynced = uint64(0) - chainID = r.initialState.ChainID - state = r.initialState + chainID = s.initialState.ChainID + state = s.initialState lastHundred = time.Now() lastRate = 0.0 @@ -488,8 +552,8 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool) { return case <-switchToConsensusTicker.C: var ( - height, numPending, lenRequesters = r.pool.GetStatus() - lastAdvance = r.pool.LastAdvance() + height, numPending, lenRequesters = s.pool.GetStatus() + lastAdvance = s.pool.LastAdvance() ) logger.Debug( @@ -500,34 +564,27 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool) { ) switch { - case r.pool.IsCaughtUp() && r.previousMaxPeerHeight <= r.pool.MaxPeerHeight(): + case s.pool.IsCaughtUp() && s.previousMaxPeerHeight <= s.pool.MaxPeerHeight(): logger.Info("switching to consensus reactor after caught up", "height", height) - case time.Since(lastAdvance) > syncTimeout: logger.Error("no progress since last advance", "last_advance", lastAdvance) continue - default: logger.Info( "not caught up yet", "height", height, - "max_peer_height", r.pool.MaxPeerHeight(), + "max_peer_height", s.pool.MaxPeerHeight(), "timeout_in", syncTimeout-time.Since(lastAdvance), ) continue } - r.pool.Stop() - - r.blockSync.UnSet() + s.blockSync.UnSet() - if r.consReactor != nil { - logger.Info("switching to consensus reactor", "height", height, "blocks_synced", blocksSynced, "state_synced", stateSynced, "max_peer_height", r.pool.MaxPeerHeight()) - // Use the node-scoped context: SwitchToConsensus is a handoff - // to a peer reactor whose lifecycle is not tied to blocksync. - r.consReactor.SwitchToConsensus(state, blocksSynced > 0 || stateSynced) - // Wake the pre-spawned auto-restart monitor. - r.consensusReady.Store(true) + if s.consReactor != nil { + logger.Info("switching to consensus reactor", "height", height, "blocks_synced", blocksSynced, "state_synced", stateSynced, "max_peer_height", s.pool.MaxPeerHeight()) + s.consReactor.SwitchToConsensus(state, blocksSynced > 0 || stateSynced) + s.consensusReady.Store(true) } return @@ -540,50 +597,26 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool) { case <-didProcessCh: // NOTE: It is a subtle mistake to process more than a single block at a // time (e.g. 10) here, because we only send one BlockRequest per loop - // iteration. The ratio mismatch can result in starving of blocks, i.e. a - // sudden burst of requests and responses, and repeat. Consequently, it is - // better to split these routines rather than coupling them as it is - // written here. - // - // TODO: Uncouple from request routine. - - // see if there are any blocks to sync - first, second := r.pool.PeekTwoBlocks() + // iteration. The ratio mismatch can result in starving of blocks. + first, second := s.pool.PeekTwoBlocks() if first == nil || second == nil { - // we need to have fetched two consecutive blocks in order to perform blocksync verification continue } - // try again quickly next loop didProcessCh <- struct{}{} firstParts, err := first.MakePartSet(types.BlockPartSizeBytes) if err != nil { - logger.Error("failed to make ", - "height", first.Height, - "err", err) + logger.Error("failed to make ", "height", first.Height, "err", err) return } - var ( - firstPartSetHeader = firstParts.Header() - firstID = types.BlockID{Hash: first.Hash(), PartSetHeader: firstPartSetHeader} - ) + firstID := types.BlockID{Hash: first.Hash(), PartSetHeader: firstParts.Header()} - // Finally, verify the first block using the second's commit. - // - // NOTE: We can probably make this more efficient, but note that calling - // first.Hash() doesn't verify the tx contents, so MakePartSet() is - // currently necessary. - // TODO(sergio): Should we also validate against the extended commit? err = state.Validators.VerifyCommitLight(chainID, firstID, first.Height, second.LastCommit) - if err == nil { - // validate the block before we persist it - err = r.blockExec.ValidateBlock(ctx, state, first) + err = s.blockExec.ValidateBlock(ctx, state, first) } - // If either of the checks failed we log the error and request for a new block - // at that height if err != nil { logger.Error( "Failed to validate block or verify commit", @@ -593,89 +626,121 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool) { "err", err, ) - // NOTE: We've already removed the peer's request, but we still need - // to clean up the rest. - peerID := r.pool.RedoRequest(first.Height) - r.router.Evict(peerID, fmt.Errorf("blocksync: %w", err)) + peerID := s.pool.RedoRequest(first.Height) + s.router.Evict(peerID, fmt.Errorf("blocksync: %w", err)) - peerID2 := r.pool.RedoRequest(second.Height) + peerID2 := s.pool.RedoRequest(second.Height) if peerID2 != peerID { - r.router.Evict(peerID2, fmt.Errorf("blocksync: %w", err)) + s.router.Evict(peerID2, fmt.Errorf("blocksync: %w", err)) } return } - r.pool.PopRequest() + s.pool.PopRequest() + s.store.SaveBlock(first, firstParts, second.LastCommit) - // We use LastCommit here instead of extCommit. extCommit is not - // guaranteed to be populated by the peer if extensions are not enabled. - // Currently, the peer should provide an extCommit even if the vote extension data are absent - // but this may change so using second.LastCommit is safer. - r.store.SaveBlock(first, firstParts, second.LastCommit) - - // TODO: Same thing for app - but we would need a way to get the hash - // without persisting the state. logger.Info("Requesting block from peer", "block", first.Height, "took", time.Since(lastApplyBlockTime)) startTime := time.Now() - state, err = r.blockExec.ApplyBlock(ctx, state, firstID, first, nil) + state, err = s.blockExec.ApplyBlock(ctx, state, firstID, first, nil) logger.Info("ApplyBlock", "block", first.Height, "took", time.Since(startTime)) lastApplyBlockTime = time.Now() if err != nil { panic(fmt.Sprintf("failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) } - r.metrics.RecordConsMetrics(first) - + s.metrics.RecordConsMetrics(first) blocksSynced++ if blocksSynced%100 == 0 { lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds()) logger.Info( "block sync rate", - "height", r.pool.height, - "max_peer_height", r.pool.MaxPeerHeight(), + "height", s.pool.height, + "max_peer_height", s.pool.MaxPeerHeight(), "blocks/s", lastRate, ) - lastHundred = time.Now() } } } } -func (r *Reactor) GetMaxPeerBlockHeight() int64 { - return r.pool.MaxPeerHeight() +// autoRestartIfBehind will check if the node is behind the max peer height by +// a certain threshold. If it is, the node will attempt to restart itself. +// TODO(gprusak): this should be a sub task of the consensus reactor instead. +func (s *syncController) autoRestartIfBehind(ctx context.Context) { + if s.blocksBehindThreshold == 0 || s.blocksBehindCheckInterval <= 0 { + logger.Info("Auto remediation is disabled") + return + } + + logger.Info("checking if node is behind threshold, auto restarting if its behind", "threshold", s.blocksBehindThreshold, "interval", s.blocksBehindCheckInterval) + for { + select { + case <-time.After(s.blocksBehindCheckInterval): + selfHeight := s.store.Height() + maxPeerHeight := s.pool.MaxPeerHeight() + threshold := int64(s.blocksBehindThreshold) //nolint:gosec // validated in config.ValidateBasic against MaxInt64 + behindHeight := maxPeerHeight - selfHeight + blockSyncIsSet := s.blockSync.IsSet() + if maxPeerHeight > s.previousMaxPeerHeight { + s.previousMaxPeerHeight = maxPeerHeight + } + + if maxPeerHeight == 0 || behindHeight < threshold || blockSyncIsSet { + logger.Debug("does not exceed threshold or is already in block sync mode", "threshold", threshold, "behindHeight", behindHeight, "maxPeerHeight", maxPeerHeight, "selfHeight", selfHeight, "blockSyncIsSet", blockSyncIsSet) + continue + } + if time.Since(s.lastRestartTime).Seconds() < float64(s.restartCooldownSeconds) { + logger.Debug("we are lagging behind, going to trigger a restart after cooldown time passes") + continue + } + logger.Info("Blocks behind threshold, restarting node", "threshold", threshold, "behindHeight", behindHeight, "maxPeerHeight", maxPeerHeight, "selfHeight", selfHeight) + + s.blockSync.Set() + s.restartEvent() + return + case <-ctx.Done(): + return + } + } } -func (r *Reactor) GetTotalSyncedTime() time.Duration { - if !r.blockSync.IsSet() || r.syncStartTime.IsZero() { +func (s *syncController) GetMaxPeerBlockHeight() int64 { + if s.pool == nil { + return 0 + } + return s.pool.MaxPeerHeight() +} + +func (s *syncController) GetTotalSyncedTime() time.Duration { + if !s.blockSync.IsSet() || s.syncStartTime.IsZero() { return time.Duration(0) } - return time.Since(r.syncStartTime) + return time.Since(s.syncStartTime) } -func (r *Reactor) GetRemainingSyncTime() time.Duration { - if !r.blockSync.IsSet() { +func (s *syncController) GetRemainingSyncTime() time.Duration { + if !s.blockSync.IsSet() || s.pool == nil { return time.Duration(0) } - targetSyncs := r.pool.targetSyncBlocks() - currentSyncs := r.store.Height() - r.pool.startHeight + 1 - lastSyncRate := r.pool.getLastSyncRate() + targetSyncs := s.pool.targetSyncBlocks() + currentSyncs := s.store.Height() - s.pool.startHeight + 1 + lastSyncRate := s.pool.getLastSyncRate() if currentSyncs < 0 || lastSyncRate < 0.001 { return time.Duration(0) } remain := float64(targetSyncs-currentSyncs) / lastSyncRate - return time.Duration(int64(remain * float64(time.Second))) } -func (r *Reactor) PublishStatus(event types.EventDataBlockSyncStatus) error { - if r.eventBus == nil { +func (s *syncController) PublishStatus(event types.EventDataBlockSyncStatus) error { + if s.eventBus == nil { return errors.New("event bus is not configured") } - return r.eventBus.PublishEventBlockSyncStatus(event) + return s.eventBus.PublishEventBlockSyncStatus(event) } // atomicBool is an atomic Boolean, safe for concurrent use by multiple diff --git a/sei-tendermint/internal/blocksync/reactor_test.go b/sei-tendermint/internal/blocksync/reactor_test.go index 2fd2aee03e..deed986820 100644 --- a/sei-tendermint/internal/blocksync/reactor_test.go +++ b/sei-tendermint/internal/blocksync/reactor_test.go @@ -26,6 +26,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/store" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/test/factory" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + pb "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/blocksync" "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) @@ -82,6 +83,7 @@ func makeReactor( t *testing.T, genDoc *types.GenesisDoc, router *p2p.Router, + blockSync bool, restartEvent func(), selfRemediationConfig *config.SelfRemediationConfig, ) *Reactor { @@ -114,15 +116,17 @@ func makeReactor( r, err := NewReactor( stateStore, - blockExec, blockStore, - nil, router, - true, - consensus.NopMetrics(), - nil, // eventbus, can be nil - restartEvent, - selfRemediationConfig, + utils.Some(SyncerConfig{ + BlockExec: blockExec, + ConsReactor: nil, + BlockSync: blockSync, + Metrics: consensus.NopMetrics(), + EventBus: nil, // eventbus can be nil + RestartEvent: restartEvent, + SelfRemediationConfig: selfRemediationConfig, + }), ) if err != nil { t.Fatalf("NewReactor(): %v", err) @@ -150,8 +154,9 @@ func (rts *reactorTestSuite) addNode( t, genDoc, rts.network.Node(nodeID).Router, + true, func() {}, - config.DefaultSelfRemediationConfig(), + remediationConfig, ) lastCommit := &types.Commit{} @@ -160,7 +165,8 @@ func (rts *reactorTestSuite) addNode( for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ { block, blockID, partSet, seenCommit := makeNextBlock(ctx, t, state, privVal, blockHeight, lastCommit) - state, err = reactor.blockExec.ApplyBlock(ctx, state, blockID, block, nil) + syncer := reactor.syncer.OrPanic("syncer should be configured in tests") + state, err = syncer.blockExec.ApplyBlock(ctx, state, blockID, block, nil) require.NoError(t, err) reactor.store.SaveBlock(block, partSet, seenCommit) @@ -226,7 +232,7 @@ func TestReactor_AbruptDisconnect(t *testing.T) { rts.start(t) - secondaryPool := rts.reactors[rts.nodes[1]].pool + secondaryPool := rts.reactors[rts.nodes[1]].syncer.OrPanic("syncer should be configured in tests").pool require.Eventually( t, @@ -341,7 +347,7 @@ func TestReactor_SyncTime(t *testing.T) { t, func() bool { return rts.reactors[rts.nodes[1]].GetRemainingSyncTime() > time.Nanosecond && - rts.reactors[rts.nodes[1]].pool.getLastSyncRate() > 0.001 + rts.reactors[rts.nodes[1]].syncer.OrPanic("syncer should be configured in tests").pool.getLastSyncRate() > 0.001 }, 10*time.Second, 10*time.Millisecond, @@ -421,24 +427,112 @@ func TestAutoRestartIfBehind(t *testing.T) { restart := utils.NewAtomicSend(false) r := &Reactor{ - store: mockBlockStore, - pool: blockPool, - blocksBehindThreshold: tt.blocksBehindThreshold, - blocksBehindCheckInterval: tt.blocksBehindCheckInterval, - restartEvent: func() { restart.Store(true) }, - blockSync: newAtomicBool(tt.isBlockSync), + syncer: utils.Some(&syncController{ + store: mockBlockStore, + pool: blockPool, + blocksBehindThreshold: tt.blocksBehindThreshold, + blocksBehindCheckInterval: tt.blocksBehindCheckInterval, + restartEvent: func() { restart.Store(true) }, + blockSync: newAtomicBool(tt.isBlockSync), + }), } ctx := t.Context() if tt.restartExpected { - r.autoRestartIfBehind(ctx) + r.syncer.OrPanic("syncer").autoRestartIfBehind(ctx) assert.True(t, restart.Load(), "Expected restart but did not occur") } else { ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) defer cancel() - r.autoRestartIfBehind(ctx) + r.syncer.OrPanic("syncer").autoRestartIfBehind(ctx) assert.False(t, restart.Load(), "Unexpected restart") } }) } } + +func TestQueryResponder_ServesBlockRequestsWhenBlockSyncDisabled(t *testing.T) { + ctx := t.Context() + + cfg, err := config.ResetTestRoot(t.TempDir(), "block_sync_query_responder_test") + require.NoError(t, err) + + valSet, privVals := factory.ValidatorSet(ctx, 1, 30) + genDoc := factory.GenesisDoc(cfg, time.Now(), valSet.Validators, factory.ConsensusParams()) + network := p2p.MakeTestNetwork(t, p2p.TestNetworkOptions{NumNodes: 2}) + nodeIDs := network.NodeIDs() + + server := makeReactor( + ctx, + t, + genDoc, + network.Node(nodeIDs[0]).Router, + false, + func() {}, + config.DefaultSelfRemediationConfig(), + ) + lastCommit := &types.Commit{} + state, err := server.stateStore.Load() + require.NoError(t, err) + for height := int64(1); height <= 3; height++ { + block, blockID, partSet, seenCommit := makeNextBlock(ctx, t, state, privVals[0], height, lastCommit) + state, err = server.syncer.OrPanic("syncer should be configured in tests").blockExec.ApplyBlock(ctx, state, blockID, block, nil) + require.NoError(t, err) + server.store.SaveBlock(block, partSet, seenCommit) + lastCommit = seenCommit + } + require.NoError(t, server.Start(ctx)) + t.Cleanup(server.Wait) + + client := p2p.TestMakeChannelNoCleanup(t, network.Node(nodeIDs[1]), GetChannelDescriptor()) + network.Start(t) + + client.Send(wrap(&pb.BlockRequest{Height: 2}), nodeIDs[0]) + for range 2 { + msg, err := client.Recv(ctx) + require.NoError(t, err) + if blockResponse, ok := msg.Message.Sum.(*pb.Message_BlockResponse); ok { + require.Equal(t, int64(2), blockResponse.BlockResponse.GetBlock().Header.Height) + require.Equal(t, nodeIDs[0], msg.From) + return + } + } + t.Fatal("did not receive block response") +} + +func TestQueryResponder_ServesStatusRequestsWhenBlockSyncDisabled(t *testing.T) { + ctx := t.Context() + + cfg, err := config.ResetTestRoot(t.TempDir(), "block_sync_query_status_test") + require.NoError(t, err) + + valSet, _ := factory.ValidatorSet(ctx, 1, 30) + genDoc := factory.GenesisDoc(cfg, time.Now(), valSet.Validators, factory.ConsensusParams()) + network := p2p.MakeTestNetwork(t, p2p.TestNetworkOptions{NumNodes: 2}) + nodeIDs := network.NodeIDs() + + server := makeReactor( + ctx, + t, + genDoc, + network.Node(nodeIDs[0]).Router, + false, + func() {}, + config.DefaultSelfRemediationConfig(), + ) + require.NoError(t, server.Start(ctx)) + t.Cleanup(server.Wait) + + client := p2p.TestMakeChannelNoCleanup(t, network.Node(nodeIDs[1]), GetChannelDescriptor()) + network.Start(t) + + client.Send(wrap(&pb.StatusRequest{}), nodeIDs[0]) + msg, err := client.Recv(ctx) + require.NoError(t, err) + + statusResponse, ok := msg.Message.Sum.(*pb.Message_StatusResponse) + require.True(t, ok) + require.Equal(t, server.store.Base(), statusResponse.StatusResponse.GetBase()) + require.Equal(t, server.store.Height(), statusResponse.StatusResponse.GetHeight()) + require.Equal(t, nodeIDs[0], msg.From) +} diff --git a/sei-tendermint/internal/rpc/core/env.go b/sei-tendermint/internal/rpc/core/env.go index ed41cc6a97..aafba973e3 100644 --- a/sei-tendermint/internal/rpc/core/env.go +++ b/sei-tendermint/internal/rpc/core/env.go @@ -73,7 +73,7 @@ type Environment struct { EvidencePool sm.EvidencePool ConsensusState consensusState ConsensusReactor *consensus.Reactor - BlockSyncReactor *blocksync.Reactor + BlockSyncReactor blocksync.Metricer IsListening bool Listeners []string diff --git a/sei-tendermint/node/node.go b/sei-tendermint/node/node.go index 573a9fda45..761cccd68d 100644 --- a/sei-tendermint/node/node.go +++ b/sei-tendermint/node/node.go @@ -311,15 +311,17 @@ func makeNode( // doing a state sync first. bcReactor, err := blocksync.NewReactor( stateStore, - blockExec, blockStore, - csReactor, node.router, - blockSync && !stateSync, - nodeMetrics.consensus, - eventBus, - restartEvent, - cfg.SelfRemediation, + utils.Some(blocksync.SyncerConfig{ + BlockExec: blockExec, + ConsReactor: csReactor, + BlockSync: blockSync && !stateSync, + Metrics: nodeMetrics.consensus, + EventBus: eventBus, + RestartEvent: restartEvent, + SelfRemediationConfig: cfg.SelfRemediation, + }), ) if err != nil { return nil, fmt.Errorf("blocksync.NewReactor(): %w", err) @@ -354,7 +356,7 @@ func makeNode( // is running // FIXME Very ugly to have these metrics bleed through here. csReactor.SetBlockSyncingMetrics(1) - if err := bcReactor.SwitchToBlockSync(ctx, state); err != nil { + if err := bcReactor.SwitchToBlockSync(state); err != nil { logger.Error("failed to switch to block sync", "err", err) return err } From aac57aa9db30f4d05c1c1e3e352cf734dd40dab8 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 27 May 2026 17:44:40 +0200 Subject: [PATCH 069/100] WIP --- sei-tendermint/internal/blocksync/pool.go | 48 ++++--- .../internal/blocksync/pool_test.go | 85 ++++------- sei-tendermint/internal/blocksync/reactor.go | 134 ++++++++---------- .../internal/blocksync/reactor_test.go | 19 +-- 4 files changed, 126 insertions(+), 160 deletions(-) diff --git a/sei-tendermint/internal/blocksync/pool.go b/sei-tendermint/internal/blocksync/pool.go index cf44719f6d..ee3dbd481d 100644 --- a/sei-tendermint/internal/blocksync/pool.go +++ b/sei-tendermint/internal/blocksync/pool.go @@ -97,35 +97,41 @@ type BlockPool struct { // atomic numPending int32 // number of requests pending assignment or block response - requestsCh chan<- BlockRequest - errorsCh chan<- peerError + requestsCh chan BlockRequest + errorsCh chan peerError + reportErr func(peerError) startHeight int64 lastHundredBlockTimeStamp time.Time lastSyncRate float64 cancels []context.CancelFunc - running *atomicBool + running atomic.Bool } // NewBlockPool returns a new BlockPool with the height equal to start. Block -// requests and errors will be sent to requestsCh and errorsCh accordingly. -func NewBlockPool( - start int64, - requestsCh chan<- BlockRequest, - errorsCh chan<- peerError, - router router, -) *BlockPool { +// requests and peer errors are published on the pool-owned request and error +// channels exposed via Requests and Errors. +func NewBlockPool(start int64, router router) *BlockPool { + return newBlockPool(start, router, nil) +} + +func newBlockPool(start int64, router router, reportErr func(peerError)) *BlockPool { bp := &BlockPool{ peers: make(map[types.NodeID]*bpPeer), requesters: make(map[int64]*bpRequester), height: start, startHeight: start, numPending: 0, - requestsCh: requestsCh, - errorsCh: errorsCh, + requestsCh: make(chan BlockRequest, maxTotalRequesters), + errorsCh: make(chan peerError, maxPeerErrBuffer), // NOTE: capacity should exceed peer count. lastSyncRate: 0, router: router, - running: newAtomicBool(false), + } + bp.reportErr = reportErr + if bp.reportErr == nil { + bp.reportErr = func(pe peerError) { + bp.errorsCh <- pe + } } return bp } @@ -134,7 +140,7 @@ func NewBlockPool( // task marks the pool active; exiting the task stops all outstanding requester // work and marks the pool inactive. func (pool *BlockPool) run(ctx context.Context) error { - pool.running.Set() + pool.running.Store(true) defer pool.shutdown() pool.lastAdvance = time.Now() @@ -150,7 +156,7 @@ func (pool *BlockPool) run(ctx context.Context) error { } func (pool *BlockPool) shutdown() { - pool.running.UnSet() + pool.running.Store(false) // Requester shutdown must not block behind a full requestsCh; Stop cancels ctx // and waits for the requester-management loop to observe pool.running=false. pool.mtx.Lock() @@ -172,7 +178,15 @@ func (pool *BlockPool) shutdown() { } func (pool *BlockPool) IsRunning() bool { - return pool.running.IsSet() + return pool.running.Load() +} + +func (pool *BlockPool) Requests() <-chan BlockRequest { + return pool.requestsCh +} + +func (pool *BlockPool) Errors() <-chan peerError { + return pool.errorsCh } // spawns requesters as needed @@ -555,7 +569,7 @@ func (pool *BlockPool) sendError(err error, peerID types.NodeID) { if !pool.IsRunning() { return } - pool.errorsCh <- peerError{err, peerID} + pool.reportErr(peerError{err, peerID}) } func (pool *BlockPool) targetSyncBlocks() int64 { diff --git a/sei-tendermint/internal/blocksync/pool_test.go b/sei-tendermint/internal/blocksync/pool_test.go index 3223fa4d01..5bdde87eee 100644 --- a/sei-tendermint/internal/blocksync/pool_test.go +++ b/sei-tendermint/internal/blocksync/pool_test.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "math" - "runtime" "strings" "testing" "time" @@ -119,9 +118,7 @@ func runPoolForTest(t *testing.T, pool *BlockPool) { func TestBlockPoolBasic(t *testing.T) { start := int64(42) peers := makePeers(10, start, 1000) - errorsCh := make(chan peerError, 1000) - requestsCh := make(chan BlockRequest, 1000) - pool := NewBlockPool(start, requestsCh, errorsCh, makeRouter(peers)) + pool := NewBlockPool(start, makeRouter(peers)) runPoolForTest(t, pool) @@ -153,9 +150,9 @@ func TestBlockPoolBasic(t *testing.T) { // Pull from channels for { select { - case err := <-errorsCh: + case err := <-pool.Errors(): t.Error(err) - case request := <-requestsCh: + case request := <-pool.Requests(): if request.Height == 300 { return // Done! } @@ -168,9 +165,7 @@ func TestBlockPoolBasic(t *testing.T) { func TestBlockPoolTimeout(t *testing.T) { start := int64(42) peers := makePeers(10, start, 1000) - errorsCh := make(chan peerError, 1000) - requestsCh := make(chan BlockRequest, 1000) - pool := NewBlockPool(start, requestsCh, errorsCh, makeRouter(peers)) + pool := NewBlockPool(start, makeRouter(peers)) runPoolForTest(t, pool) // Introduce each peer. @@ -196,7 +191,7 @@ func TestBlockPoolTimeout(t *testing.T) { }() // Pull from channels - <-errorsCh + <-pool.Errors() } func TestBlockPoolRemovePeer(t *testing.T) { @@ -211,10 +206,7 @@ func TestBlockPoolRemovePeer(t *testing.T) { height := int64(i + 1) peers[peerID] = testPeer{peerID, 0, height, make(chan inputData)} } - requestsCh := make(chan BlockRequest) - errorsCh := make(chan peerError) - - pool := NewBlockPool(1, requestsCh, errorsCh, makeRouter(peers)) + pool := NewBlockPool(1, makeRouter(peers)) runPoolForTest(t, pool) // add peers @@ -246,10 +238,7 @@ func TestBlockPoolMaliciousNodeMaxInt64(t *testing.T) { goodNodeId: testPeer{goodNodeId, 1, initialHeight, make(chan inputData)}, badNodeId: testPeer{badNodeId, 1, math.MaxInt64, make(chan inputData)}, } - errorsCh := make(chan peerError, 3) - requestsCh := make(chan BlockRequest) - - pool := NewBlockPool(1, requestsCh, errorsCh, makeRouter(peers)) + pool := NewBlockPool(1, makeRouter(peers)) // add peers for peerID, peer := range peers { pool.SetPeerRange(peerID, peer.base, peer.height) @@ -276,9 +265,7 @@ func testBlockPoolRejectsWrongPeerWithoutDiscardingGoodBlock(t *testing.T, goodF peers := testPeers{ goodPeerID: {goodPeerID, 1, 2, make(chan inputData)}, } - requestsCh := make(chan BlockRequest, 2) - errorsCh := make(chan peerError, 2) - pool := NewBlockPool(1, requestsCh, errorsCh, makeRouter(peers)) + pool := NewBlockPool(1, makeRouter(peers)) runPoolForTest(t, pool) @@ -286,7 +273,7 @@ func testBlockPoolRejectsWrongPeerWithoutDiscardingGoodBlock(t *testing.T, goodF requests := map[int64]BlockRequest{} for range 2 { - request := <-requestsCh + request := <-pool.Requests() requests[request.Height] = request } @@ -317,23 +304,22 @@ func testBlockPoolRejectsWrongPeerWithoutDiscardingGoodBlock(t *testing.T, goodF } // TestBlockPoolAddBlockReleasesLockBeforeSend asserts that AddBlock does -// not hold pool.mtx across a send on errorsCh. -// -// The test parks AddBlock on its sendError call (by leaving the -// unbuffered errorsCh unread) and then asserts via runtime.Stack + -// pool.mtx.TryLock that the mutex is acquirable while the goroutine is -// parked there. Without the fix, AddBlock holds pool.mtx for the -// duration of the parked send and TryLock never succeeds. +// not hold pool.mtx while reporting an error. func TestBlockPoolAddBlockReleasesLockBeforeSend(t *testing.T) { peerID := types.NodeID(strings.Repeat("a", 40)) peers := testPeers{peerID: {peerID, 1, 100, make(chan inputData)}} - errorsCh := make(chan peerError) - requestsCh := make(chan BlockRequest, 1000) - pool := NewBlockPool(1, requestsCh, errorsCh, makeRouter(peers)) + mtxUnlocked := make(chan bool, 1) + var pool *BlockPool + pool = newBlockPool(1, makeRouter(peers), func(peerError) { + unlocked := pool.mtx.TryLock() + if unlocked { + pool.mtx.Unlock() + } + mtxUnlocked <- unlocked + }) runPoolForTest(t, pool) - - pool.SetPeerRange(peerID, 1, 100) + require.Eventually(t, pool.IsRunning, time.Second, time.Millisecond) // pool.height starts at 1 and the peer reports height 100, so no // requester is created for far-ahead heights. A block more than @@ -349,33 +335,14 @@ func TestBlockPoolAddBlockReleasesLockBeforeSend(t *testing.T) { close(addBlockDone) }() t.Cleanup(func() { - // Drain so the AddBlock goroutine can exit even if the assertion - // below fails. - select { - case <-errorsCh: - case <-addBlockDone: - } <-addBlockDone }) - require.Eventually(t, func() bool { - if !anyGoroutineParkedIn("blocksync.(*BlockPool).sendError") { - return false - } - if !pool.mtx.TryLock() { - return false - } - pool.mtx.Unlock() - return true - }, 5*time.Second, 10*time.Millisecond, - "pool.mtx not released while a goroutine is parked in sendError") - - <-errorsCh + select { + case unlocked := <-mtxUnlocked: + require.True(t, unlocked, "pool.mtx held while reporting an error") + case <-t.Context().Done(): + t.Fatal(t.Context().Err()) + } <-addBlockDone } - -func anyGoroutineParkedIn(frame string) bool { - buf := make([]byte, 64<<10) - n := runtime.Stack(buf, true) - return strings.Contains(string(buf[:n]), frame) -} diff --git a/sei-tendermint/internal/blocksync/reactor.go b/sei-tendermint/internal/blocksync/reactor.go index 38f090e346..2746fe1325 100644 --- a/sei-tendermint/internal/blocksync/reactor.go +++ b/sei-tendermint/internal/blocksync/reactor.go @@ -93,7 +93,15 @@ func (e peerError) Error() string { return fmt.Sprintf("error with peer %v: %s", e.peerID, e.err.Error()) } -type blocksyncResult struct{ stateSynced bool } +type blocksyncResult struct { + stateSynced bool + state sm.State + syncStartAt time.Time +} + +type syncState struct { + initialState sm.State +} // SyncerConfig groups dependencies and startup knobs used only by the active // blocksync controller. Reactor itself does not need these when running as an @@ -141,16 +149,12 @@ type syncController struct { eventBus *eventbus.EventBus // Mutable sync state initialized on start and updated as blocksync runs. - initialState sm.State - pool *BlockPool - requestsCh <-chan BlockRequest - errorsCh <-chan peerError + pool *BlockPool // blockSync tracks whether the node is actively in blocksync mode. The // channel responder stays up regardless of this flag. - blockSync *atomicBool + blockSync atomic.Bool previousMaxPeerHeight int64 - syncStartTime time.Time // Auto-remediation configuration and restart bookkeeping. restartEvent func() @@ -181,7 +185,7 @@ func NewReactor( syncer := utils.None[*syncController]() if cfg, ok := syncerConfig.Get(); ok { - syncer = utils.Some(&syncController{ + s := &syncController{ stateStore: stateStore, blockExec: cfg.BlockExec, store: store, @@ -190,7 +194,6 @@ func NewReactor( consReactor: cfg.ConsReactor, metrics: cfg.Metrics, eventBus: cfg.EventBus, - blockSync: newAtomicBool(cfg.BlockSync), restartEvent: cfg.RestartEvent, lastRestartTime: time.Now(), blocksBehindThreshold: cfg.SelfRemediationConfig.BlocksBehindThreshold, @@ -198,7 +201,11 @@ func NewReactor( restartCooldownSeconds: cfg.SelfRemediationConfig.RestartCooldownSeconds, blocksyncReady: utils.NewAtomicSend(utils.None[blocksyncResult]()), consensusReady: utils.NewAtomicSend(false), - }) + } + if cfg.BlockSync { + s.blockSync.Store(true) + } + syncer = utils.Some(s) } r := &Reactor{ @@ -230,9 +237,6 @@ func (r *Reactor) OnStart(ctx context.Context) error { startHeight = state.InitialHeight } - if syncer, ok := r.syncer.Get(); ok { - syncer.initialize(state, startHeight) - } r.SpawnCritical("processBlockSyncCh", func(ctx context.Context) error { r.processBlockSyncCh(ctx) return nil @@ -245,8 +249,12 @@ func (r *Reactor) OnStart(ctx context.Context) error { r.SpawnCritical("syncController.run", func(ctx context.Context) error { return syncer.run(ctx) }) - if syncer.blockSync.IsSet() { - syncer.blocksyncReady.Store(utils.Some(blocksyncResult{false})) + if syncer.blockSync.Load() { + syncer.blocksyncReady.Store(utils.Some(blocksyncResult{ + stateSynced: false, + state: state, + syncStartAt: time.Now(), + })) } } return nil @@ -288,14 +296,6 @@ func (r *Reactor) GetRemainingSyncTime() time.Duration { return 0 } -func (r *Reactor) PublishStatus(event types.EventDataBlockSyncStatus) error { - syncer, ok := r.syncer.Get() - if !ok { - return errors.New("blocksync syncer is not configured") - } - return syncer.PublishStatus(event) -} - // respondToPeer loads a block and sends it to the requesting peer, if we have it. // Otherwise, it responds saying we do not have it. func (r *Reactor) respondToPeer(msg *pb.BlockRequest, peerID types.NodeID) error { @@ -388,19 +388,6 @@ func (r *Reactor) processPeerUpdates(ctx context.Context) { } } -// initialize prepares the active sync controller state for a given starting -// height before any gated sync goroutines begin work. -func (s *syncController) initialize(initialState sm.State, startHeight int64) { - requestsCh := make(chan BlockRequest, maxTotalRequesters) - errorsCh := make(chan peerError, maxPeerErrBuffer) // NOTE: capacity should exceed peer count. - - s.initialState = initialState - s.lastRestartTime = time.Now() - s.pool = NewBlockPool(startHeight, requestsCh, errorsCh, s.router) - s.requestsCh = requestsCh - s.errorsCh = errorsCh -} - // run owns the active sync controller's internal concurrency. A single // coordinator task waits for blocksyncReady, then starts the pool and spawns // the active sync subtasks for that session. @@ -413,17 +400,24 @@ func (s *syncController) run(ctx context.Context) error { if err != nil { return err } + res := result.OrPanic("no blocksync result") + s.lastRestartTime = time.Now() + + state := syncState{ + initialState: res.state, + } + s.pool = NewBlockPool(startHeightForState(res.state), s.router) return scope.Run(ctx, func(ctx context.Context, session scope.Scope) error { session.SpawnNamed("pool.run", func() error { return s.pool.run(ctx) }) session.SpawnNamed("requestRoutine", func() error { - s.requestRoutine(ctx) + s.requestRoutine(ctx, state) return nil }) session.SpawnNamed("poolRoutine", func() error { - s.poolRoutine(ctx, result.OrPanic("no blocksync result").stateSynced) + s.poolRoutine(ctx, state, res.stateSynced) return nil }) return nil @@ -487,12 +481,12 @@ func (s *syncController) handlePeerDown(peerID types.NodeID) { } func (s *syncController) switchToBlockSync(state sm.State) error { - s.blockSync.Set() - s.initialState = state - s.pool.height = state.LastBlockHeight + 1 - - s.syncStartTime = time.Now() - s.blocksyncReady.Store(utils.Some(blocksyncResult{true})) + s.blockSync.Store(true) + s.blocksyncReady.Store(utils.Some(blocksyncResult{ + stateSynced: true, + state: state, + syncStartAt: time.Now(), + })) if err := s.PublishStatus(types.EventDataBlockSyncStatus{ Complete: false, @@ -504,7 +498,7 @@ func (s *syncController) switchToBlockSync(state sm.State) error { return nil } -func (s *syncController) requestRoutine(ctx context.Context) { +func (s *syncController) requestRoutine(ctx context.Context, state syncState) { statusUpdateTicker := time.NewTicker(statusUpdateInterval) defer statusUpdateTicker.Stop() @@ -512,9 +506,9 @@ func (s *syncController) requestRoutine(ctx context.Context) { select { case <-ctx.Done(): return - case request := <-s.requestsCh: + case request := <-s.pool.Requests(): s.channel.Send(wrap(&pb.BlockRequest{Height: request.Height}), request.PeerID) - case pErr := <-s.errorsCh: + case pErr := <-s.pool.Errors(): s.router.Evict(pErr.peerID, fmt.Errorf("blocksync.request: %w", pErr.err)) case <-statusUpdateTicker.C: s.channel.Broadcast(wrap(&pb.StatusRequest{})) @@ -526,7 +520,7 @@ func (s *syncController) requestRoutine(ctx context.Context) { // to do. // // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down! -func (s *syncController) poolRoutine(ctx context.Context, stateSynced bool) { +func (s *syncController) poolRoutine(ctx context.Context, syncState syncState, stateSynced bool) { var ( trySyncTicker = time.NewTicker(trySyncIntervalMS * time.Millisecond) switchToConsensusTicker = time.NewTicker(switchToConsensusIntervalSeconds * time.Second) @@ -534,8 +528,8 @@ func (s *syncController) poolRoutine(ctx context.Context, stateSynced bool) { blocksSynced = uint64(0) - chainID = s.initialState.ChainID - state = s.initialState + chainID = syncState.initialState.ChainID + state = syncState.initialState lastHundred = time.Now() lastRate = 0.0 @@ -579,7 +573,7 @@ func (s *syncController) poolRoutine(ctx context.Context, stateSynced bool) { continue } - s.blockSync.UnSet() + s.blockSync.Store(false) if s.consReactor != nil { logger.Info("switching to consensus reactor", "height", height, "blocks_synced", blocksSynced, "state_synced", stateSynced, "max_peer_height", s.pool.MaxPeerHeight()) @@ -682,7 +676,7 @@ func (s *syncController) autoRestartIfBehind(ctx context.Context) { maxPeerHeight := s.pool.MaxPeerHeight() threshold := int64(s.blocksBehindThreshold) //nolint:gosec // validated in config.ValidateBasic against MaxInt64 behindHeight := maxPeerHeight - selfHeight - blockSyncIsSet := s.blockSync.IsSet() + blockSyncIsSet := s.blockSync.Load() if maxPeerHeight > s.previousMaxPeerHeight { s.previousMaxPeerHeight = maxPeerHeight } @@ -697,7 +691,7 @@ func (s *syncController) autoRestartIfBehind(ctx context.Context) { } logger.Info("Blocks behind threshold, restarting node", "threshold", threshold, "behindHeight", behindHeight, "maxPeerHeight", maxPeerHeight, "selfHeight", selfHeight) - s.blockSync.Set() + s.blockSync.Store(true) s.restartEvent() return case <-ctx.Done(): @@ -714,14 +708,18 @@ func (s *syncController) GetMaxPeerBlockHeight() int64 { } func (s *syncController) GetTotalSyncedTime() time.Duration { - if !s.blockSync.IsSet() || s.syncStartTime.IsZero() { + if !s.blockSync.Load() { return time.Duration(0) } - return time.Since(s.syncStartTime) + result, ok := s.blocksyncReady.Load().Get() + if !ok || result.syncStartAt.IsZero() { + return time.Duration(0) + } + return time.Since(result.syncStartAt) } func (s *syncController) GetRemainingSyncTime() time.Duration { - if !s.blockSync.IsSet() || s.pool == nil { + if !s.blockSync.Load() || s.pool == nil { return time.Duration(0) } @@ -743,24 +741,10 @@ func (s *syncController) PublishStatus(event types.EventDataBlockSyncStatus) err return s.eventBus.PublishEventBlockSyncStatus(event) } -// atomicBool is an atomic Boolean, safe for concurrent use by multiple -// goroutines. -type atomicBool int32 - -// newAtomicBool creates an atomicBool with given initial value. -func newAtomicBool(ok bool) *atomicBool { - ab := new(atomicBool) - if ok { - ab.Set() +func startHeightForState(state sm.State) int64 { + startHeight := state.LastBlockHeight + 1 + if startHeight == 1 { + startHeight = state.InitialHeight } - return ab + return startHeight } - -// Set sets the Boolean to true. -func (ab *atomicBool) Set() { atomic.StoreInt32((*int32)(ab), 1) } - -// UnSet sets the Boolean to false. -func (ab *atomicBool) UnSet() { atomic.StoreInt32((*int32)(ab), 0) } - -// IsSet returns whether the Boolean is true. -func (ab *atomicBool) IsSet() bool { return atomic.LoadInt32((*int32)(ab))&1 == 1 } diff --git a/sei-tendermint/internal/blocksync/reactor_test.go b/sei-tendermint/internal/blocksync/reactor_test.go index deed986820..2f1fc477ce 100644 --- a/sei-tendermint/internal/blocksync/reactor_test.go +++ b/sei-tendermint/internal/blocksync/reactor_test.go @@ -426,16 +426,17 @@ func TestAutoRestartIfBehind(t *testing.T) { } restart := utils.NewAtomicSend(false) - r := &Reactor{ - syncer: utils.Some(&syncController{ - store: mockBlockStore, - pool: blockPool, - blocksBehindThreshold: tt.blocksBehindThreshold, - blocksBehindCheckInterval: tt.blocksBehindCheckInterval, - restartEvent: func() { restart.Store(true) }, - blockSync: newAtomicBool(tt.isBlockSync), - }), + syncer := &syncController{ + store: mockBlockStore, + pool: blockPool, + blocksBehindThreshold: tt.blocksBehindThreshold, + blocksBehindCheckInterval: tt.blocksBehindCheckInterval, + restartEvent: func() { restart.Store(true) }, } + if tt.isBlockSync { + syncer.blockSync.Store(true) + } + r := &Reactor{syncer: utils.Some(syncer)} ctx := t.Context() if tt.restartExpected { From f23fadf612af18b580ddf767b2e0e325d5fc06ee Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 27 May 2026 18:00:39 +0200 Subject: [PATCH 070/100] WIP --- sei-tendermint/internal/blocksync/reactor.go | 160 +++++++----------- .../internal/blocksync/reactor_test.go | 6 + 2 files changed, 65 insertions(+), 101 deletions(-) diff --git a/sei-tendermint/internal/blocksync/reactor.go b/sei-tendermint/internal/blocksync/reactor.go index 2746fe1325..d51d31797e 100644 --- a/sei-tendermint/internal/blocksync/reactor.go +++ b/sei-tendermint/internal/blocksync/reactor.go @@ -99,10 +99,6 @@ type blocksyncResult struct { syncStartAt time.Time } -type syncState struct { - initialState sm.State -} - // SyncerConfig groups dependencies and startup knobs used only by the active // blocksync controller. Reactor itself does not need these when running as an // always-on query responder only. @@ -154,6 +150,7 @@ type syncController struct { // blockSync tracks whether the node is actively in blocksync mode. The // channel responder stays up regardless of this flag. blockSync atomic.Bool + startInBlockSync bool previousMaxPeerHeight int64 // Auto-remediation configuration and restart bookkeeping. @@ -201,9 +198,7 @@ func NewReactor( restartCooldownSeconds: cfg.SelfRemediationConfig.RestartCooldownSeconds, blocksyncReady: utils.NewAtomicSend(utils.None[blocksyncResult]()), consensusReady: utils.NewAtomicSend(false), - } - if cfg.BlockSync { - s.blockSync.Store(true) + startInBlockSync: cfg.BlockSync, } syncer = utils.Some(s) } @@ -232,24 +227,15 @@ func (r *Reactor) OnStart(ctx context.Context) error { return fmt.Errorf("state (%v) and store (%v) height mismatch", state.LastBlockHeight, r.store.Height()) } - startHeight := r.store.Height() + 1 - if startHeight == 1 { - startHeight = state.InitialHeight - } - r.SpawnCritical("processBlockSyncCh", func(ctx context.Context) error { r.processBlockSyncCh(ctx) return nil }) - r.SpawnCritical("processPeerUpdates", func(ctx context.Context) error { - r.processPeerUpdates(ctx) - return nil - }) if syncer, ok := r.syncer.Get(); ok { r.SpawnCritical("syncController.run", func(ctx context.Context) error { return syncer.run(ctx) }) - if syncer.blockSync.Load() { + if syncer.startInBlockSync { syncer.blocksyncReady.Store(utils.Some(blocksyncResult{ stateSynced: false, state: state, @@ -316,7 +302,7 @@ func (r *Reactor) respondToPeer(msg *pb.BlockRequest, peerID types.NodeID) error } // handleMessage handles an inbound blocksync message. Reactor only owns block -// request serving; every other blocksync message is forwarded to the sync +// and status request serving; every other blocksync message is forwarded to the sync // controller. func (r *Reactor) handleMessage(m p2p.RecvMsg[*pb.Message]) (err error) { defer func() { @@ -335,6 +321,13 @@ func (r *Reactor) handleMessage(m p2p.RecvMsg[*pb.Message]) (err error) { if msg, ok := m.Message.Sum.(*pb.Message_BlockRequest); ok { return r.respondToPeer(msg.BlockRequest, m.From) } + if _, ok := m.Message.Sum.(*pb.Message_StatusRequest); ok { + r.channel.Send(wrap(&pb.StatusResponse{ + Base: r.store.Base(), + Height: r.store.Height(), + }), m.From) + return nil + } syncer, ok := r.syncer.Get() if !ok { return nil @@ -355,44 +348,15 @@ func (r *Reactor) processBlockSyncCh(ctx context.Context) { } } -// processPeerUpdate handles the subset of peer lifecycle needed by blocksync: -// advertise our local range on connect and let the sync controller clean up any -// in-flight state on disconnect. -func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) { - logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status) - - switch peerUpdate.Status { - case p2p.PeerStatusUp: - r.channel.Send(wrap(&pb.StatusResponse{ - Base: r.store.Base(), - Height: r.store.Height(), - }), peerUpdate.NodeID) - case p2p.PeerStatusDown: - if syncer, ok := r.syncer.Get(); ok { - syncer.handlePeerDown(peerUpdate.NodeID) - } - } -} - -// processPeerUpdates listens for peer updates. The reactor owns peer-up -// status announcements; the sync controller only receives peer-down callbacks -// for pool cleanup. -func (r *Reactor) processPeerUpdates(ctx context.Context) { - recv := r.router.Subscribe() - for { - update, err := recv.Recv(ctx) - if err != nil { - return - } - r.processPeerUpdate(update) - } -} - // run owns the active sync controller's internal concurrency. A single // coordinator task waits for blocksyncReady, then starts the pool and spawns // the active sync subtasks for that session. func (s *syncController) run(ctx context.Context) error { return scope.Run(ctx, func(ctx context.Context, sc scope.Scope) error { + sc.SpawnNamed("processPeerUpdates", func() error { + s.processPeerUpdates(ctx) + return nil + }) sc.SpawnNamed("blocksyncSession", func() error { result, err := s.blocksyncReady.Wait(ctx, func(o utils.Option[blocksyncResult]) bool { return o.IsPresent() @@ -401,11 +365,17 @@ func (s *syncController) run(ctx context.Context) error { return err } res := result.OrPanic("no blocksync result") + s.blockSync.Store(true) + if res.stateSynced { + if err := s.PublishStatus(types.EventDataBlockSyncStatus{ + Complete: false, + Height: res.state.LastBlockHeight, + }); err != nil { + logger.Info("failed to publish blocksync status", "height", res.state.LastBlockHeight, "err", err) + } + } s.lastRestartTime = time.Now() - state := syncState{ - initialState: res.state, - } s.pool = NewBlockPool(startHeightForState(res.state), s.router) return scope.Run(ctx, func(ctx context.Context, session scope.Scope) error { @@ -413,11 +383,11 @@ func (s *syncController) run(ctx context.Context) error { return s.pool.run(ctx) }) session.SpawnNamed("requestRoutine", func() error { - s.requestRoutine(ctx, state) + s.requestRoutine(ctx) return nil }) session.SpawnNamed("poolRoutine", func() error { - s.poolRoutine(ctx, state, res.stateSynced) + s.poolRoutine(ctx, res.state, res.stateSynced) return nil }) return nil @@ -434,6 +404,29 @@ func (s *syncController) run(ctx context.Context) error { }) } +// processPeerUpdates listens for peer updates. Active blocksync owns peer-up +// status announcements and peer-down cleanup for the shared pool. +func (s *syncController) processPeerUpdates(ctx context.Context) { + recv := s.router.Subscribe() + for { + update, err := recv.Recv(ctx) + if err != nil { + return + } + + logger.Debug("received peer update", "peer", update.NodeID, "status", update.Status) + + switch update.Status { + case p2p.PeerStatusUp: + s.channel.Send(wrap(&pb.StatusRequest{}), update.NodeID) + case p2p.PeerStatusDown: + if s.pool != nil { + s.pool.RemovePeer(update.NodeID) + } + } + } +} + // handleMessage processes all non-BlockRequest blocksync protocol messages. func (s *syncController) handleMessage(m p2p.RecvMsg[*pb.Message]) error { switch msg := m.Message.Sum.(type) { @@ -442,65 +435,33 @@ func (s *syncController) handleMessage(m p2p.RecvMsg[*pb.Message]) error { if err != nil { return fmt.Errorf("types.BlockFromProto(): %w", err) } - return s.handleBlockResponse(m.From, block) - case *pb.Message_StatusRequest: - s.channel.Send(wrap(&pb.StatusResponse{ - Height: s.store.Height(), - Base: s.store.Base(), - }), m.From) + logger.Info("received block response from peer", "peer", m.From, "height", block.Height) + if err := s.pool.AddBlock(m.From, block, block.Size()); err != nil { + logger.Error("failed to add block", "err", err) + } return nil case *pb.Message_StatusResponse: - s.handleStatusResponse(m.From, msg.StatusResponse.GetBase(), msg.StatusResponse.GetHeight()) + s.pool.SetPeerRange(m.From, msg.StatusResponse.GetBase(), msg.StatusResponse.GetHeight()) return nil case *pb.Message_NoBlockResponse: - s.handleNoBlockResponse(m.From, msg.NoBlockResponse.GetHeight()) + logger.Debug("peer does not have the requested block", "peer", m.From, "height", msg.NoBlockResponse.GetHeight()) return nil default: return fmt.Errorf("received unknown message: %T", msg) } } -func (s *syncController) handleBlockResponse(peerID types.NodeID, block *types.Block) error { - logger.Info("received block response from peer", "peer", peerID, "height", block.Height) - if err := s.pool.AddBlock(peerID, block, block.Size()); err != nil { - logger.Error("failed to add block", "err", err) - } - return nil -} - -func (s *syncController) handleStatusResponse(peerID types.NodeID, base, height int64) { - s.pool.SetPeerRange(peerID, base, height) -} - -func (s *syncController) handleNoBlockResponse(peerID types.NodeID, height int64) { - logger.Debug("peer does not have the requested block", "peer", peerID, "height", height) -} - -func (s *syncController) handlePeerDown(peerID types.NodeID) { - s.pool.RemovePeer(peerID) -} - func (s *syncController) switchToBlockSync(state sm.State) error { - s.blockSync.Store(true) s.blocksyncReady.Store(utils.Some(blocksyncResult{ stateSynced: true, state: state, syncStartAt: time.Now(), })) - - if err := s.PublishStatus(types.EventDataBlockSyncStatus{ - Complete: false, - Height: state.LastBlockHeight, - }); err != nil { - return err - } - return nil } -func (s *syncController) requestRoutine(ctx context.Context, state syncState) { +func (s *syncController) requestRoutine(ctx context.Context) { statusUpdateTicker := time.NewTicker(statusUpdateInterval) - defer statusUpdateTicker.Stop() for { select { @@ -520,7 +481,7 @@ func (s *syncController) requestRoutine(ctx context.Context, state syncState) { // to do. // // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down! -func (s *syncController) poolRoutine(ctx context.Context, syncState syncState, stateSynced bool) { +func (s *syncController) poolRoutine(ctx context.Context, initialState sm.State, stateSynced bool) { var ( trySyncTicker = time.NewTicker(trySyncIntervalMS * time.Millisecond) switchToConsensusTicker = time.NewTicker(switchToConsensusIntervalSeconds * time.Second) @@ -528,8 +489,8 @@ func (s *syncController) poolRoutine(ctx context.Context, syncState syncState, s blocksSynced = uint64(0) - chainID = syncState.initialState.ChainID - state = syncState.initialState + chainID = initialState.ChainID + state = initialState lastHundred = time.Now() lastRate = 0.0 @@ -537,9 +498,6 @@ func (s *syncController) poolRoutine(ctx context.Context, syncState syncState, s didProcessCh = make(chan struct{}, 1) ) - defer trySyncTicker.Stop() - defer switchToConsensusTicker.Stop() - for { select { case <-ctx.Done(): diff --git a/sei-tendermint/internal/blocksync/reactor_test.go b/sei-tendermint/internal/blocksync/reactor_test.go index 2f1fc477ce..c221472aab 100644 --- a/sei-tendermint/internal/blocksync/reactor_test.go +++ b/sei-tendermint/internal/blocksync/reactor_test.go @@ -527,6 +527,12 @@ func TestQueryResponder_ServesStatusRequestsWhenBlockSyncDisabled(t *testing.T) client := p2p.TestMakeChannelNoCleanup(t, network.Node(nodeIDs[1]), GetChannelDescriptor()) network.Start(t) + peerUpMsg, err := client.Recv(ctx) + require.NoError(t, err) + _, ok := peerUpMsg.Message.Sum.(*pb.Message_StatusRequest) + require.True(t, ok) + require.Equal(t, nodeIDs[0], peerUpMsg.From) + client.Send(wrap(&pb.StatusRequest{}), nodeIDs[0]) msg, err := client.Recv(ctx) require.NoError(t, err) From 9e0f1212a009a8b5dd838d2cf2aef13228b8d41b Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 27 May 2026 18:21:34 +0200 Subject: [PATCH 071/100] WIP --- .../internal/blocksync/pool_test.go | 6 +- sei-tendermint/internal/blocksync/reactor.go | 104 ++++++++++-------- .../internal/blocksync/reactor_test.go | 21 ++-- 3 files changed, 74 insertions(+), 57 deletions(-) diff --git a/sei-tendermint/internal/blocksync/pool_test.go b/sei-tendermint/internal/blocksync/pool_test.go index 5bdde87eee..4dc7927740 100644 --- a/sei-tendermint/internal/blocksync/pool_test.go +++ b/sei-tendermint/internal/blocksync/pool_test.go @@ -318,8 +318,10 @@ func TestBlockPoolAddBlockReleasesLockBeforeSend(t *testing.T) { } mtxUnlocked <- unlocked }) - runPoolForTest(t, pool) - require.Eventually(t, pool.IsRunning, time.Second, time.Millisecond) + pool.running.Store(true) + t.Cleanup(func() { + pool.running.Store(false) + }) // pool.height starts at 1 and the peer reports height 100, so no // requester is created for far-ahead heights. A block more than diff --git a/sei-tendermint/internal/blocksync/reactor.go b/sei-tendermint/internal/blocksync/reactor.go index d51d31797e..d792ecbf9e 100644 --- a/sei-tendermint/internal/blocksync/reactor.go +++ b/sei-tendermint/internal/blocksync/reactor.go @@ -145,17 +145,15 @@ type syncController struct { eventBus *eventbus.EventBus // Mutable sync state initialized on start and updated as blocksync runs. - pool *BlockPool + pool atomic.Pointer[BlockPool] // blockSync tracks whether the node is actively in blocksync mode. The // channel responder stays up regardless of this flag. - blockSync atomic.Bool - startInBlockSync bool - previousMaxPeerHeight int64 + blockSync atomic.Bool + startInBlockSync bool // Auto-remediation configuration and restart bookkeeping. restartEvent func() - lastRestartTime time.Time blocksBehindThreshold uint64 blocksBehindCheckInterval time.Duration restartCooldownSeconds uint64 @@ -192,7 +190,6 @@ func NewReactor( metrics: cfg.Metrics, eventBus: cfg.EventBus, restartEvent: cfg.RestartEvent, - lastRestartTime: time.Now(), blocksBehindThreshold: cfg.SelfRemediationConfig.BlocksBehindThreshold, blocksBehindCheckInterval: time.Duration(cfg.SelfRemediationConfig.BlocksBehindCheckIntervalSeconds) * time.Second, //nolint:gosec // validated in config.ValidateBasic against MaxInt64 restartCooldownSeconds: cfg.SelfRemediationConfig.RestartCooldownSeconds, @@ -374,32 +371,32 @@ func (s *syncController) run(ctx context.Context) error { logger.Info("failed to publish blocksync status", "height", res.state.LastBlockHeight, "err", err) } } - s.lastRestartTime = time.Now() - s.pool = NewBlockPool(startHeightForState(res.state), s.router) + pool := NewBlockPool(startHeightForState(res.state), s.router) + s.pool.Store(pool) return scope.Run(ctx, func(ctx context.Context, session scope.Scope) error { session.SpawnNamed("pool.run", func() error { - return s.pool.run(ctx) + return pool.run(ctx) }) session.SpawnNamed("requestRoutine", func() error { - s.requestRoutine(ctx) + s.requestRoutine(ctx, pool) return nil }) session.SpawnNamed("poolRoutine", func() error { - s.poolRoutine(ctx, res.state, res.stateSynced) + s.poolRoutine(ctx, pool, res.state, res.stateSynced) + return nil + }) + session.SpawnNamed("autoRestartIfBehind", func() error { + if _, err := s.consensusReady.Wait(ctx, func(ready bool) bool { return ready }); err != nil { + return err + } + s.autoRestartIfBehind(ctx, pool) return nil }) return nil }) }) - sc.SpawnNamed("autoRestartIfBehind", func() error { - if _, err := s.consensusReady.Wait(ctx, func(ready bool) bool { return ready }); err != nil { - return err - } - s.autoRestartIfBehind(ctx) - return nil - }) return nil }) } @@ -420,8 +417,8 @@ func (s *syncController) processPeerUpdates(ctx context.Context) { case p2p.PeerStatusUp: s.channel.Send(wrap(&pb.StatusRequest{}), update.NodeID) case p2p.PeerStatusDown: - if s.pool != nil { - s.pool.RemovePeer(update.NodeID) + if pool := s.pool.Load(); pool != nil { + pool.RemovePeer(update.NodeID) } } } @@ -431,17 +428,23 @@ func (s *syncController) processPeerUpdates(ctx context.Context) { func (s *syncController) handleMessage(m p2p.RecvMsg[*pb.Message]) error { switch msg := m.Message.Sum.(type) { case *pb.Message_BlockResponse: + pool := s.pool.Load() + if pool == nil { + return nil + } block, err := types.BlockFromProto(msg.BlockResponse.GetBlock()) if err != nil { return fmt.Errorf("types.BlockFromProto(): %w", err) } logger.Info("received block response from peer", "peer", m.From, "height", block.Height) - if err := s.pool.AddBlock(m.From, block, block.Size()); err != nil { + if err := pool.AddBlock(m.From, block, block.Size()); err != nil { logger.Error("failed to add block", "err", err) } return nil case *pb.Message_StatusResponse: - s.pool.SetPeerRange(m.From, msg.StatusResponse.GetBase(), msg.StatusResponse.GetHeight()) + if pool := s.pool.Load(); pool != nil { + pool.SetPeerRange(m.From, msg.StatusResponse.GetBase(), msg.StatusResponse.GetHeight()) + } return nil case *pb.Message_NoBlockResponse: logger.Debug("peer does not have the requested block", "peer", m.From, "height", msg.NoBlockResponse.GetHeight()) @@ -460,16 +463,16 @@ func (s *syncController) switchToBlockSync(state sm.State) error { return nil } -func (s *syncController) requestRoutine(ctx context.Context) { +func (s *syncController) requestRoutine(ctx context.Context, pool *BlockPool) { statusUpdateTicker := time.NewTicker(statusUpdateInterval) for { select { case <-ctx.Done(): return - case request := <-s.pool.Requests(): + case request := <-pool.Requests(): s.channel.Send(wrap(&pb.BlockRequest{Height: request.Height}), request.PeerID) - case pErr := <-s.pool.Errors(): + case pErr := <-pool.Errors(): s.router.Evict(pErr.peerID, fmt.Errorf("blocksync.request: %w", pErr.err)) case <-statusUpdateTicker.C: s.channel.Broadcast(wrap(&pb.StatusRequest{})) @@ -481,7 +484,7 @@ func (s *syncController) requestRoutine(ctx context.Context) { // to do. // // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down! -func (s *syncController) poolRoutine(ctx context.Context, initialState sm.State, stateSynced bool) { +func (s *syncController) poolRoutine(ctx context.Context, pool *BlockPool, initialState sm.State, stateSynced bool) { var ( trySyncTicker = time.NewTicker(trySyncIntervalMS * time.Millisecond) switchToConsensusTicker = time.NewTicker(switchToConsensusIntervalSeconds * time.Second) @@ -504,8 +507,8 @@ func (s *syncController) poolRoutine(ctx context.Context, initialState sm.State, return case <-switchToConsensusTicker.C: var ( - height, numPending, lenRequesters = s.pool.GetStatus() - lastAdvance = s.pool.LastAdvance() + height, numPending, lenRequesters = pool.GetStatus() + lastAdvance = pool.LastAdvance() ) logger.Debug( @@ -516,7 +519,7 @@ func (s *syncController) poolRoutine(ctx context.Context, initialState sm.State, ) switch { - case s.pool.IsCaughtUp() && s.previousMaxPeerHeight <= s.pool.MaxPeerHeight(): + case pool.IsCaughtUp(): logger.Info("switching to consensus reactor after caught up", "height", height) case time.Since(lastAdvance) > syncTimeout: logger.Error("no progress since last advance", "last_advance", lastAdvance) @@ -525,7 +528,7 @@ func (s *syncController) poolRoutine(ctx context.Context, initialState sm.State, logger.Info( "not caught up yet", "height", height, - "max_peer_height", s.pool.MaxPeerHeight(), + "max_peer_height", pool.MaxPeerHeight(), "timeout_in", syncTimeout-time.Since(lastAdvance), ) continue @@ -534,7 +537,7 @@ func (s *syncController) poolRoutine(ctx context.Context, initialState sm.State, s.blockSync.Store(false) if s.consReactor != nil { - logger.Info("switching to consensus reactor", "height", height, "blocks_synced", blocksSynced, "state_synced", stateSynced, "max_peer_height", s.pool.MaxPeerHeight()) + logger.Info("switching to consensus reactor", "height", height, "blocks_synced", blocksSynced, "state_synced", stateSynced, "max_peer_height", pool.MaxPeerHeight()) s.consReactor.SwitchToConsensus(state, blocksSynced > 0 || stateSynced) s.consensusReady.Store(true) } @@ -550,7 +553,7 @@ func (s *syncController) poolRoutine(ctx context.Context, initialState sm.State, // NOTE: It is a subtle mistake to process more than a single block at a // time (e.g. 10) here, because we only send one BlockRequest per loop // iteration. The ratio mismatch can result in starving of blocks. - first, second := s.pool.PeekTwoBlocks() + first, second := pool.PeekTwoBlocks() if first == nil || second == nil { continue } @@ -578,17 +581,17 @@ func (s *syncController) poolRoutine(ctx context.Context, initialState sm.State, "err", err, ) - peerID := s.pool.RedoRequest(first.Height) + peerID := pool.RedoRequest(first.Height) s.router.Evict(peerID, fmt.Errorf("blocksync: %w", err)) - peerID2 := s.pool.RedoRequest(second.Height) + peerID2 := pool.RedoRequest(second.Height) if peerID2 != peerID { s.router.Evict(peerID2, fmt.Errorf("blocksync: %w", err)) } return } - s.pool.PopRequest() + pool.PopRequest() s.store.SaveBlock(first, firstParts, second.LastCommit) logger.Info("Requesting block from peer", "block", first.Height, "took", time.Since(lastApplyBlockTime)) @@ -607,8 +610,8 @@ func (s *syncController) poolRoutine(ctx context.Context, initialState sm.State, lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds()) logger.Info( "block sync rate", - "height", s.pool.height, - "max_peer_height", s.pool.MaxPeerHeight(), + "height", pool.height, + "max_peer_height", pool.MaxPeerHeight(), "blocks/s", lastRate, ) lastHundred = time.Now() @@ -620,30 +623,33 @@ func (s *syncController) poolRoutine(ctx context.Context, initialState sm.State, // autoRestartIfBehind will check if the node is behind the max peer height by // a certain threshold. If it is, the node will attempt to restart itself. // TODO(gprusak): this should be a sub task of the consensus reactor instead. -func (s *syncController) autoRestartIfBehind(ctx context.Context) { +func (s *syncController) autoRestartIfBehind(ctx context.Context, pool *BlockPool) { if s.blocksBehindThreshold == 0 || s.blocksBehindCheckInterval <= 0 { logger.Info("Auto remediation is disabled") return } + lastRestartTime := time.Now() + var previousMaxPeerHeight int64 + logger.Info("checking if node is behind threshold, auto restarting if its behind", "threshold", s.blocksBehindThreshold, "interval", s.blocksBehindCheckInterval) for { select { case <-time.After(s.blocksBehindCheckInterval): selfHeight := s.store.Height() - maxPeerHeight := s.pool.MaxPeerHeight() + maxPeerHeight := pool.MaxPeerHeight() threshold := int64(s.blocksBehindThreshold) //nolint:gosec // validated in config.ValidateBasic against MaxInt64 behindHeight := maxPeerHeight - selfHeight blockSyncIsSet := s.blockSync.Load() - if maxPeerHeight > s.previousMaxPeerHeight { - s.previousMaxPeerHeight = maxPeerHeight + if maxPeerHeight > previousMaxPeerHeight { + previousMaxPeerHeight = maxPeerHeight } if maxPeerHeight == 0 || behindHeight < threshold || blockSyncIsSet { logger.Debug("does not exceed threshold or is already in block sync mode", "threshold", threshold, "behindHeight", behindHeight, "maxPeerHeight", maxPeerHeight, "selfHeight", selfHeight, "blockSyncIsSet", blockSyncIsSet) continue } - if time.Since(s.lastRestartTime).Seconds() < float64(s.restartCooldownSeconds) { + if time.Since(lastRestartTime).Seconds() < float64(s.restartCooldownSeconds) { logger.Debug("we are lagging behind, going to trigger a restart after cooldown time passes") continue } @@ -659,10 +665,11 @@ func (s *syncController) autoRestartIfBehind(ctx context.Context) { } func (s *syncController) GetMaxPeerBlockHeight() int64 { - if s.pool == nil { + pool := s.pool.Load() + if pool == nil { return 0 } - return s.pool.MaxPeerHeight() + return pool.MaxPeerHeight() } func (s *syncController) GetTotalSyncedTime() time.Duration { @@ -677,13 +684,14 @@ func (s *syncController) GetTotalSyncedTime() time.Duration { } func (s *syncController) GetRemainingSyncTime() time.Duration { - if !s.blockSync.Load() || s.pool == nil { + pool := s.pool.Load() + if !s.blockSync.Load() || pool == nil { return time.Duration(0) } - targetSyncs := s.pool.targetSyncBlocks() - currentSyncs := s.store.Height() - s.pool.startHeight + 1 - lastSyncRate := s.pool.getLastSyncRate() + targetSyncs := pool.targetSyncBlocks() + currentSyncs := s.store.Height() - pool.startHeight + 1 + lastSyncRate := pool.getLastSyncRate() if currentSyncs < 0 || lastSyncRate < 0.001 { return time.Duration(0) } diff --git a/sei-tendermint/internal/blocksync/reactor_test.go b/sei-tendermint/internal/blocksync/reactor_test.go index c221472aab..5f25747a86 100644 --- a/sei-tendermint/internal/blocksync/reactor_test.go +++ b/sei-tendermint/internal/blocksync/reactor_test.go @@ -232,13 +232,17 @@ func TestReactor_AbruptDisconnect(t *testing.T) { rts.start(t) - secondaryPool := rts.reactors[rts.nodes[1]].syncer.OrPanic("syncer should be configured in tests").pool + secondarySyncer := rts.reactors[rts.nodes[1]].syncer.OrPanic("syncer should be configured in tests") require.Eventually( t, func() bool { - height, _, _ := secondaryPool.GetStatus() - return secondaryPool.MaxPeerHeight() == maxBlockHeight && height > 0 && height <= maxBlockHeight + pool := secondarySyncer.pool.Load() + if pool == nil { + return false + } + height, _, _ := pool.GetStatus() + return pool.MaxPeerHeight() == maxBlockHeight && height > 0 && height <= maxBlockHeight }, 10*time.Second, 10*time.Millisecond, @@ -346,8 +350,12 @@ func TestReactor_SyncTime(t *testing.T) { require.Eventually( t, func() bool { + pool := rts.reactors[rts.nodes[1]].syncer.OrPanic("syncer should be configured in tests").pool.Load() + if pool == nil { + return false + } return rts.reactors[rts.nodes[1]].GetRemainingSyncTime() > time.Nanosecond && - rts.reactors[rts.nodes[1]].syncer.OrPanic("syncer should be configured in tests").pool.getLastSyncRate() > 0.001 + pool.getLastSyncRate() > 0.001 }, 10*time.Second, 10*time.Millisecond, @@ -428,7 +436,6 @@ func TestAutoRestartIfBehind(t *testing.T) { restart := utils.NewAtomicSend(false) syncer := &syncController{ store: mockBlockStore, - pool: blockPool, blocksBehindThreshold: tt.blocksBehindThreshold, blocksBehindCheckInterval: tt.blocksBehindCheckInterval, restartEvent: func() { restart.Store(true) }, @@ -440,12 +447,12 @@ func TestAutoRestartIfBehind(t *testing.T) { ctx := t.Context() if tt.restartExpected { - r.syncer.OrPanic("syncer").autoRestartIfBehind(ctx) + r.syncer.OrPanic("syncer").autoRestartIfBehind(ctx, blockPool) assert.True(t, restart.Load(), "Expected restart but did not occur") } else { ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) defer cancel() - r.syncer.OrPanic("syncer").autoRestartIfBehind(ctx) + r.syncer.OrPanic("syncer").autoRestartIfBehind(ctx, blockPool) assert.False(t, restart.Load(), "Unexpected restart") } }) From 27b8ee09b77d48cbc39802b8eb8089a0024b5378 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 27 May 2026 19:20:29 +0200 Subject: [PATCH 072/100] terminating pool early --- sei-tendermint/internal/blocksync/reactor.go | 80 ++++++++++---------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/sei-tendermint/internal/blocksync/reactor.go b/sei-tendermint/internal/blocksync/reactor.go index d792ecbf9e..dbe2897a1d 100644 --- a/sei-tendermint/internal/blocksync/reactor.go +++ b/sei-tendermint/internal/blocksync/reactor.go @@ -99,6 +99,14 @@ type blocksyncResult struct { syncStartAt time.Time } +type consensusHandoff struct { + state sm.State + blocksSynced uint64 + stateSynced bool + height int64 + maxPeerHeight int64 +} + // SyncerConfig groups dependencies and startup knobs used only by the active // blocksync controller. Reactor itself does not need these when running as an // always-on query responder only. @@ -161,9 +169,6 @@ type syncController struct { // blocksyncReady fires when the active sync routines should begin processing // work, either during OnStart or later via SwitchToBlockSync. blocksyncReady utils.AtomicSend[utils.Option[blocksyncResult]] - // consensusReady fires after blocksync hands off to consensus so the - // auto-restart monitor can start observing lag from that point forward. - consensusReady utils.AtomicSend[bool] } // NewReactor returns new reactor instance. @@ -194,7 +199,6 @@ func NewReactor( blocksBehindCheckInterval: time.Duration(cfg.SelfRemediationConfig.BlocksBehindCheckIntervalSeconds) * time.Second, //nolint:gosec // validated in config.ValidateBasic against MaxInt64 restartCooldownSeconds: cfg.SelfRemediationConfig.RestartCooldownSeconds, blocksyncReady: utils.NewAtomicSend(utils.None[blocksyncResult]()), - consensusReady: utils.NewAtomicSend(false), startInBlockSync: cfg.BlockSync, } syncer = utils.Some(s) @@ -346,8 +350,9 @@ func (r *Reactor) processBlockSyncCh(ctx context.Context) { } // run owns the active sync controller's internal concurrency. A single -// coordinator task waits for blocksyncReady, then starts the pool and spawns -// the active sync subtasks for that session. +// coordinator task waits for blocksyncReady, then starts the pool and runs +// the active sync session. The consensus handoff happens only after the +// blocksync-only session tasks have exited. func (s *syncController) run(ctx context.Context) error { return scope.Run(ctx, func(ctx context.Context, sc scope.Scope) error { sc.SpawnNamed("processPeerUpdates", func() error { @@ -374,28 +379,28 @@ func (s *syncController) run(ctx context.Context) error { pool := NewBlockPool(startHeightForState(res.state), s.router) s.pool.Store(pool) + sc.SpawnBgNamed("requestRoutine", func() error { + s.requestRoutine(ctx, pool) + return nil + }) - return scope.Run(ctx, func(ctx context.Context, session scope.Scope) error { - session.SpawnNamed("pool.run", func() error { + handoff, err := scope.Run1(ctx, func(ctx context.Context, session scope.Scope) (consensusHandoff, error) { + session.SpawnBgNamed("pool.run", func() error { return pool.run(ctx) }) - session.SpawnNamed("requestRoutine", func() error { - s.requestRoutine(ctx, pool) - return nil - }) - session.SpawnNamed("poolRoutine", func() error { - s.poolRoutine(ctx, pool, res.state, res.stateSynced) - return nil - }) - session.SpawnNamed("autoRestartIfBehind", func() error { - if _, err := s.consensusReady.Wait(ctx, func(ready bool) bool { return ready }); err != nil { - return err - } - s.autoRestartIfBehind(ctx, pool) - return nil - }) - return nil + return s.poolRoutine(ctx, pool, res.state, res.stateSynced) }) + if err != nil { + return err + } + + s.blockSync.Store(false) + if s.consReactor != nil { + logger.Info("switching to consensus reactor", "height", handoff.height, "blocks_synced", handoff.blocksSynced, "state_synced", handoff.stateSynced, "max_peer_height", handoff.maxPeerHeight) + s.consReactor.SwitchToConsensus(handoff.state, handoff.blocksSynced > 0 || handoff.stateSynced) + s.autoRestartIfBehind(ctx, pool) + } + return nil }) return nil }) @@ -481,10 +486,10 @@ func (s *syncController) requestRoutine(ctx context.Context, pool *BlockPool) { } // poolRoutine handles messages from the poolReactor telling the controller what -// to do. +// to do and returns a handoff result once blocksync has fully caught up. // // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down! -func (s *syncController) poolRoutine(ctx context.Context, pool *BlockPool, initialState sm.State, stateSynced bool) { +func (s *syncController) poolRoutine(ctx context.Context, pool *BlockPool, initialState sm.State, stateSynced bool) (consensusHandoff, error) { var ( trySyncTicker = time.NewTicker(trySyncIntervalMS * time.Millisecond) switchToConsensusTicker = time.NewTicker(switchToConsensusIntervalSeconds * time.Second) @@ -504,7 +509,7 @@ func (s *syncController) poolRoutine(ctx context.Context, pool *BlockPool, initi for { select { case <-ctx.Done(): - return + return consensusHandoff{}, ctx.Err() case <-switchToConsensusTicker.C: var ( height, numPending, lenRequesters = pool.GetStatus() @@ -534,15 +539,13 @@ func (s *syncController) poolRoutine(ctx context.Context, pool *BlockPool, initi continue } - s.blockSync.Store(false) - - if s.consReactor != nil { - logger.Info("switching to consensus reactor", "height", height, "blocks_synced", blocksSynced, "state_synced", stateSynced, "max_peer_height", pool.MaxPeerHeight()) - s.consReactor.SwitchToConsensus(state, blocksSynced > 0 || stateSynced) - s.consensusReady.Store(true) - } - - return + return consensusHandoff{ + state: state, + blocksSynced: blocksSynced, + stateSynced: stateSynced, + height: height, + maxPeerHeight: pool.MaxPeerHeight(), + }, nil case <-trySyncTicker.C: select { @@ -562,8 +565,7 @@ func (s *syncController) poolRoutine(ctx context.Context, pool *BlockPool, initi firstParts, err := first.MakePartSet(types.BlockPartSizeBytes) if err != nil { - logger.Error("failed to make ", "height", first.Height, "err", err) - return + return consensusHandoff{}, fmt.Errorf("first.MakePartSet(%d): %w", first.Height, err) } firstID := types.BlockID{Hash: first.Hash(), PartSetHeader: firstParts.Header()} @@ -588,7 +590,7 @@ func (s *syncController) poolRoutine(ctx context.Context, pool *BlockPool, initi if peerID2 != peerID { s.router.Evict(peerID2, fmt.Errorf("blocksync: %w", err)) } - return + continue } pool.PopRequest() From d130dfdae263f3b0987c70440a1571159984338c Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 27 May 2026 19:55:31 +0200 Subject: [PATCH 073/100] tests --- sei-tendermint/internal/blocksync/pool.go | 11 +- .../internal/blocksync/reactor_test.go | 165 +++++++++++++++++- 2 files changed, 171 insertions(+), 5 deletions(-) diff --git a/sei-tendermint/internal/blocksync/pool.go b/sei-tendermint/internal/blocksync/pool.go index ee3dbd481d..ac99c5def0 100644 --- a/sei-tendermint/internal/blocksync/pool.go +++ b/sei-tendermint/internal/blocksync/pool.go @@ -744,6 +744,11 @@ func (bpr *bpRequester) reset(force bool) bool { // NOTE: Nonblocking, and does nothing if another redo // was already requested. func (bpr *bpRequester) redo(peerID types.NodeID, retryReason RetryReason) { + if retryReason == BadBlock { + // Clear the bad block immediately so callers do not keep re-reading the + // same invalid pair before the requester goroutine processes redoCh. + bpr.reset(true) + } select { case bpr.redoCh <- RedoOp{ PeerId: peerID, @@ -809,8 +814,10 @@ OUTER_LOOP: continue WAIT_LOOP } case <-bpr.gotBlockCh: - // We got a block! - return + // Keep the requester alive until its block is either popped or + // invalidated. Validation happens outside the requester, so a bad + // block must still be able to trigger redo(). + continue WAIT_LOOP } } } diff --git a/sei-tendermint/internal/blocksync/reactor_test.go b/sei-tendermint/internal/blocksync/reactor_test.go index 5f25747a86..54d634c384 100644 --- a/sei-tendermint/internal/blocksync/reactor_test.go +++ b/sei-tendermint/internal/blocksync/reactor_test.go @@ -2,6 +2,7 @@ package blocksync import ( "context" + "errors" "runtime" "strings" "testing" @@ -100,8 +101,8 @@ func makeReactor( require.NoError(t, err) require.NoError(t, stateStore.Save(state)) mp := mempool.NewTxMempool(mempool.TestConfig(), proxyApp, mempool.NopMetrics(), mempool.NopTxConstraintsFetcher) - eventbus := eventbus.NewDefault() - require.NoError(t, eventbus.Start(ctx)) + bus := eventbus.NewDefault() + require.NoError(t, bus.Start(ctx)) blockExec := sm.NewBlockExecutor( stateStore, @@ -109,7 +110,7 @@ func makeReactor( mp, sm.EmptyEvidencePool{}, blockStore, - eventbus, + bus, sm.NopMetrics(), types.DefaultConsensusPolicy(), ) @@ -459,6 +460,164 @@ func TestAutoRestartIfBehind(t *testing.T) { } } +func makeValidationFailurePair( + ctx context.Context, + t *testing.T, + testRootName string, +) (sm.State, *types.Block, *types.Block) { + t.Helper() + + cfg, err := config.ResetTestRoot(t.TempDir(), testRootName) + require.NoError(t, err) + + valSet, privVals := factory.ValidatorSet(ctx, 1, 30) + genDoc := factory.GenesisDoc(cfg, time.Now(), valSet.Validators, factory.ConsensusParams()) + initialState, err := sm.MakeGenesisState(genDoc) + require.NoError(t, err) + + lastCommit := &types.Commit{} + block1, _, _, seenCommit1 := makeNextBlock(ctx, t, initialState, privVals[0], 1, lastCommit) + block2, _, _, _ := makeNextBlock(ctx, t, initialState, privVals[0], 2, seenCommit1) + + badBlock2Proto, err := block2.ToProto() + require.NoError(t, err) + badBlock2Proto.LastCommit.Signatures[0].Signature[0] ^= 0xFF + badCommit, err := types.CommitFromProto(badBlock2Proto.LastCommit) + require.NoError(t, err) + badBlock2Proto.Header.LastCommitHash = badCommit.Hash() + badBlock2, err := types.BlockFromProto(badBlock2Proto) + require.NoError(t, err) + + return initialState, block1, badBlock2 +} + +func TestPoolRoutine_DoesNotReturnOnValidationFailure(t *testing.T) { + ctx := t.Context() + + initialState, block1, badBlock2 := makeValidationFailurePair(ctx, t, "block_sync_validation_failure_does_not_return") + + badPeer := types.NodeID(strings.Repeat("a", 40)) + goodPeer := types.NodeID(strings.Repeat("b", 40)) + router := makeRouter(testPeers{ + badPeer: {id: badPeer, base: 1, height: 2, inputChan: make(chan inputData, 1)}, + goodPeer: {id: goodPeer, base: 1, height: 2, inputChan: make(chan inputData, 1)}, + }) + pool := NewBlockPool(1, router) + done := make(chan error, 1) + go func() { done <- pool.run(ctx) }() + t.Cleanup(func() { + if err := <-done; err != nil && !errors.Is(err, context.Canceled) { + t.Fatalf("pool.run(): %v", err) + } + }) + pool.SetPeerRange(badPeer, 1, 2) + + evictNetwork := p2p.MakeTestNetwork(t, p2p.TestNetworkOptions{NumNodes: 1}) + syncer := &syncController{ + router: evictNetwork.Node(evictNetwork.NodeIDs()[0]).Router, + metrics: consensus.NopMetrics(), + } + + results := make(chan error, 1) + go func() { + _, err := syncer.poolRoutine(ctx, pool, initialState, false) + results <- err + }() + t.Cleanup(func() { + err := <-results + require.ErrorIs(t, err, context.Canceled) + }) + + introducedGoodPeer := false + for { + select { + case err := <-results: + t.Fatalf("poolRoutine returned early after validation failure: %v", err) + case request := <-pool.Requests(): + if request.PeerID == goodPeer { + return + } + + switch request.Height { + case 1: + _ = pool.AddBlock(request.PeerID, block1, block1.Size()) + case 2: + _ = pool.AddBlock(request.PeerID, badBlock2, badBlock2.Size()) + if !introducedGoodPeer { + introducedGoodPeer = true + pool.SetPeerRange(goodPeer, 1, 2) + } + } + } + } +} + +func TestPoolRoutine_RetriesAfterValidationFailure(t *testing.T) { + ctx := t.Context() + + initialState, block1, badBlock2 := makeValidationFailurePair(ctx, t, "block_sync_retry_after_validation_failure") + network := p2p.MakeTestNetwork(t, p2p.TestNetworkOptions{NumNodes: 1}) + + badPeer := types.NodeID(strings.Repeat("a", 40)) + goodPeer1 := types.NodeID(strings.Repeat("b", 40)) + goodPeer2 := types.NodeID(strings.Repeat("c", 40)) + peers := testPeers{ + badPeer: {id: badPeer, base: 1, height: 2, inputChan: make(chan inputData, 1)}, + goodPeer1: {id: goodPeer1, base: 1, height: 2, inputChan: make(chan inputData, 1)}, + goodPeer2: {id: goodPeer2, base: 1, height: 2, inputChan: make(chan inputData, 1)}, + } + pool := NewBlockPool(1, makeRouter(peers)) + runPoolForTest(t, pool) + pool.SetPeerRange(badPeer, 1, 2) + + syncer := &syncController{ + router: network.Node(network.NodeIDs()[0]).Router, + metrics: consensus.NopMetrics(), + } + + results := make(chan error, 1) + go func() { + _, err := syncer.poolRoutine(ctx, pool, initialState, false) + results <- err + }() + t.Cleanup(func() { + err := <-results + require.ErrorIs(t, err, context.Canceled) + }) + + introducedGoodPeers := false + height1Requests := map[types.NodeID]int{} + + for { + select { + case err := <-results: + t.Fatalf("poolRoutine returned before retry was observed: %v", err) + case request := <-pool.Requests(): + if request.Height == 1 { + height1Requests[request.PeerID]++ + if request.PeerID != badPeer && height1Requests[request.PeerID] == 1 { + return + } + } + + if request.PeerID == badPeer && request.Height == 2 && !introducedGoodPeers { + introducedGoodPeers = true + pool.SetPeerRange(goodPeer1, 1, 2) + pool.SetPeerRange(goodPeer2, 1, 2) + } + + if request.PeerID == badPeer { + switch request.Height { + case 1: + _ = pool.AddBlock(request.PeerID, block1, block1.Size()) + case 2: + _ = pool.AddBlock(request.PeerID, badBlock2, badBlock2.Size()) + } + } + } + } +} + func TestQueryResponder_ServesBlockRequestsWhenBlockSyncDisabled(t *testing.T) { ctx := t.Context() From f93ee9fa86df4390362965391a8e3508d27f495d Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 28 May 2026 09:54:11 +0200 Subject: [PATCH 074/100] attempt to remove more --- sei-tendermint/internal/blocksync/pool.go | 203 +++++++------------ sei-tendermint/internal/statesync/reactor.go | 2 +- 2 files changed, 76 insertions(+), 129 deletions(-) diff --git a/sei-tendermint/internal/blocksync/pool.go b/sei-tendermint/internal/blocksync/pool.go index ac99c5def0..b182af5973 100644 --- a/sei-tendermint/internal/blocksync/pool.go +++ b/sei-tendermint/internal/blocksync/pool.go @@ -11,7 +11,6 @@ import ( "time" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/libs/flowrate" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/service" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" "github.com/sei-protocol/sei-chain/sei-tendermint/types" "github.com/sei-protocol/seilog" @@ -141,40 +140,27 @@ func newBlockPool(start int64, router router, reportErr func(peerError)) *BlockP // work and marks the pool inactive. func (pool *BlockPool) run(ctx context.Context) error { pool.running.Store(true) - defer pool.shutdown() pool.lastAdvance = time.Now() pool.lastHundredBlockTimeStamp = pool.lastAdvance return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { - s.SpawnNamed("makeRequestersRoutine", func() error { - pool.makeRequestersRoutine(ctx) - return nil - }) - return nil - }) -} - -func (pool *BlockPool) shutdown() { - pool.running.Store(false) - // Requester shutdown must not block behind a full requestsCh; Stop cancels ctx - // and waits for the requester-management loop to observe pool.running=false. - pool.mtx.Lock() - cancels := pool.cancels - pool.cancels = nil - requesters := make([]*bpRequester, 0, len(pool.requesters)) - for _, requester := range pool.requesters { - requesters = append(requesters, requester) - } - pool.mtx.Unlock() + for ctx.Err() == nil { + _, numPending, lenRequesters := pool.GetStatus() + if numPending >= maxPendingRequests || lenRequesters >= maxTotalRequesters { + // This is preferable to using a timer because the request interval + // is so small. Larger request intervals may necessitate using a + // timer/ticker. + time.Sleep(requestInterval) + pool.removeTimedoutPeers() + continue + } - // Stop requesters outside pool.mtx; their shutdown path may observe pool state. - for _, cancel := range cancels { - cancel() - } - for _, requester := range requesters { - requester.Stop() - } + // request for more blocks. + pool.makeNextRequester(ctx) + } + return ctx.Err() + }) } func (pool *BlockPool) IsRunning() bool { @@ -189,27 +175,6 @@ func (pool *BlockPool) Errors() <-chan peerError { return pool.errorsCh } -// spawns requesters as needed -func (pool *BlockPool) makeRequestersRoutine(ctx context.Context) { - for pool.IsRunning() { - if ctx.Err() != nil { - return - } - - _, numPending, lenRequesters := pool.GetStatus() - if numPending >= maxPendingRequests || lenRequesters >= maxTotalRequesters { - // This is preferable to using a timer because the request interval - // is so small. Larger request intervals may necessitate using a - // timer/ticker. - time.Sleep(requestInterval) - pool.removeTimedoutPeers() - continue - } - - // request for more blocks. - pool.makeNextRequester(ctx) - } -} func (pool *BlockPool) removeTimedoutPeers() { var errsToSend []peerError @@ -541,8 +506,6 @@ func (pool *BlockPool) makeNextRequester(ctx context.Context) { pool.requesters[nextHeight] = request atomic.AddInt32(&pool.numPending, 1) - ctx, cancel := context.WithCancel(ctx) - pool.cancels = append(pool.cancels, cancel) err := request.Start(ctx) if err != nil { logger.Error("error starting request", "err", err) @@ -647,7 +610,6 @@ func (peer *bpPeer) onTimeout() { //------------------------------------- type bpRequester struct { - service.BaseService pool *BlockPool height int64 gotBlockCh chan struct{} @@ -666,7 +628,7 @@ type RedoOp struct { } func newBPRequester(pool *BlockPool, height int64) *bpRequester { - bpr := &bpRequester{ + return &bpRequester{ pool: pool, height: height, gotBlockCh: make(chan struct{}, 1), @@ -675,19 +637,68 @@ func newBPRequester(pool *BlockPool, height int64) *bpRequester { peerID: "", block: nil, } - bpr.BaseService = *service.NewBaseService("bpRequester", bpr) - return bpr } -func (bpr *bpRequester) OnStart(ctx context.Context) error { - bpr.Spawn("requestRoutine", func(ctx context.Context) error { - bpr.requestRoutine(ctx) - return nil - }) - return nil -} +// Responsible for making more requests as necessary +// Returns only when a block is found (e.g. AddBlock() is called) +func (bpr *bpRequester) run(ctx context.Context) error { + defer bpr.timeoutTicker.Stop() -func (*bpRequester) OnStop() {} +OUTER_LOOP: + for { + // Pick a peer to send request to. + var peer *bpPeer + PICK_PEER_LOOP: + for { + if ctx.Err() != nil { + return nil + } + peer = bpr.pool.pickIncrAvailablePeer(bpr.height) + if peer == nil { + // This is preferable to using a timer because the request + // interval is so small. Larger request intervals may + // necessitate using a timer/ticker. + time.Sleep(requestInterval) + continue PICK_PEER_LOOP + } + break PICK_PEER_LOOP + } + bpr.mtx.Lock() + bpr.peerID = peer.id + bpr.mtx.Unlock() + + // Send request and wait. + if !bpr.pool.sendRequest(ctx, bpr.height, peer.id) { + return nil + } + bpr.timeoutTicker.Reset(peerTimeout) + WAIT_LOOP: + for { + select { + case <-ctx.Done(): + return nil + case redoOp := <-bpr.redoCh: + // if we don't have an existing block or this is a bad block + // we should reset the previous block + if bpr.reset(redoOp.Reason == BadBlock) { + continue OUTER_LOOP + } + continue WAIT_LOOP + case <-bpr.timeoutTicker.C: + if bpr.reset(false) { + continue OUTER_LOOP + } else { + continue WAIT_LOOP + } + case <-bpr.gotBlockCh: + // Keep the requester alive until its block is either popped or + // invalidated. Validation happens outside the requester, so a bad + // block must still be able to trigger redo(). + continue WAIT_LOOP + } + } + } +} // Returns 0 if block doesn't already exist. // Returns -1 if peer doesn't match. @@ -758,67 +769,3 @@ func (bpr *bpRequester) redo(peerID types.NodeID, retryReason RetryReason) { } } -// Responsible for making more requests as necessary -// Returns only when a block is found (e.g. AddBlock() is called) -func (bpr *bpRequester) requestRoutine(ctx context.Context) { - defer bpr.timeoutTicker.Stop() - -OUTER_LOOP: - for { - // Pick a peer to send request to. - var peer *bpPeer - PICK_PEER_LOOP: - for { - if !bpr.IsRunning() || !bpr.pool.IsRunning() || ctx.Err() != nil { - return - } - if ctx.Err() != nil { - return - } - - peer = bpr.pool.pickIncrAvailablePeer(bpr.height) - if peer == nil { - // This is preferable to using a timer because the request - // interval is so small. Larger request intervals may - // necessitate using a timer/ticker. - time.Sleep(requestInterval) - continue PICK_PEER_LOOP - } - break PICK_PEER_LOOP - } - bpr.mtx.Lock() - bpr.peerID = peer.id - bpr.mtx.Unlock() - - // Send request and wait. - if !bpr.pool.sendRequest(ctx, bpr.height, peer.id) { - return - } - bpr.timeoutTicker.Reset(peerTimeout) - WAIT_LOOP: - for { - select { - case <-ctx.Done(): - return - case redoOp := <-bpr.redoCh: - // if we don't have an existing block or this is a bad block - // we should reset the previous block - if bpr.reset(redoOp.Reason == BadBlock) { - continue OUTER_LOOP - } - continue WAIT_LOOP - case <-bpr.timeoutTicker.C: - if bpr.reset(false) { - continue OUTER_LOOP - } else { - continue WAIT_LOOP - } - case <-bpr.gotBlockCh: - // Keep the requester alive until its block is either popped or - // invalidated. Validation happens outside the requester, so a bad - // block must still be able to trigger redo(). - continue WAIT_LOOP - } - } - } -} diff --git a/sei-tendermint/internal/statesync/reactor.go b/sei-tendermint/internal/statesync/reactor.go index d53c8c0e93..18ccc6d83f 100644 --- a/sei-tendermint/internal/statesync/reactor.go +++ b/sei-tendermint/internal/statesync/reactor.go @@ -125,7 +125,7 @@ func GetLightBlockChannelDescriptor() p2p.ChannelDescriptor[*pb.Message] { return p2p.ChannelDescriptor[*pb.Message]{ ID: LightBlockChannel, MessageType: new(pb.Message), - PreDecode: utils.Some[func([]byte) error](pb.SchemaForMessage.Scan), + PreDecode: utils.Some(pb.SchemaForMessage.Scan), Priority: 5, SendQueueCapacity: 10, RecvMessageCapacity: lightBlockMsgSize, From 1dc6d72cb63a8a04cb82f8c7c8c7442f91501476 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 28 May 2026 11:46:33 +0200 Subject: [PATCH 075/100] blocksync pool refactor --- sei-tendermint/internal/blocksync/pool.go | 285 +++++++----------- .../internal/blocksync/pool_test.go | 4 - sei-tendermint/internal/blocksync/reactor.go | 11 +- .../internal/blocksync/reactor_test.go | 2 +- sei-tendermint/libs/utils/option.go | 2 +- 5 files changed, 117 insertions(+), 187 deletions(-) diff --git a/sei-tendermint/internal/blocksync/pool.go b/sei-tendermint/internal/blocksync/pool.go index b182af5973..7a021bb3af 100644 --- a/sei-tendermint/internal/blocksync/pool.go +++ b/sei-tendermint/internal/blocksync/pool.go @@ -11,6 +11,7 @@ import ( "time" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/libs/flowrate" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" "github.com/sei-protocol/sei-chain/sei-tendermint/types" "github.com/sei-protocol/seilog" @@ -94,7 +95,7 @@ type BlockPool struct { maxPeerHeight int64 // the biggest reported height // atomic - numPending int32 // number of requests pending assignment or block response + numPending atomic.Int32 // number of requests pending assignment or block response requestsCh chan BlockRequest errorsCh chan peerError @@ -103,8 +104,6 @@ type BlockPool struct { startHeight int64 lastHundredBlockTimeStamp time.Time lastSyncRate float64 - cancels []context.CancelFunc - running atomic.Bool } // NewBlockPool returns a new BlockPool with the height equal to start. Block @@ -120,16 +119,18 @@ func newBlockPool(start int64, router router, reportErr func(peerError)) *BlockP requesters: make(map[int64]*bpRequester), height: start, startHeight: start, - numPending: 0, requestsCh: make(chan BlockRequest, maxTotalRequesters), errorsCh: make(chan peerError, maxPeerErrBuffer), // NOTE: capacity should exceed peer count. lastSyncRate: 0, router: router, + reportErr: reportErr, } - bp.reportErr = reportErr if bp.reportErr == nil { bp.reportErr = func(pe peerError) { - bp.errorsCh <- pe + select { + case bp.errorsCh <- pe: + default: + } } } return bp @@ -139,8 +140,6 @@ func newBlockPool(start int64, router router, reportErr func(peerError)) *BlockP // task marks the pool active; exiting the task stops all outstanding requester // work and marks the pool inactive. func (pool *BlockPool) run(ctx context.Context) error { - pool.running.Store(true) - pool.lastAdvance = time.Now() pool.lastHundredBlockTimeStamp = pool.lastAdvance @@ -151,22 +150,22 @@ func (pool *BlockPool) run(ctx context.Context) error { // This is preferable to using a timer because the request interval // is so small. Larger request intervals may necessitate using a // timer/ticker. - time.Sleep(requestInterval) + if err := utils.Sleep(ctx, requestInterval); err != nil { + return err + } pool.removeTimedoutPeers() continue } // request for more blocks. - pool.makeNextRequester(ctx) + if r, ok := pool.makeNextRequester(); ok { + s.Spawn(func() error { return r.run(ctx) }) + } } return ctx.Err() }) } -func (pool *BlockPool) IsRunning() bool { - return pool.running.Load() -} - func (pool *BlockPool) Requests() <-chan BlockRequest { return pool.requestsCh } @@ -175,7 +174,6 @@ func (pool *BlockPool) Errors() <-chan peerError { return pool.errorsCh } - func (pool *BlockPool) removeTimedoutPeers() { var errsToSend []peerError defer func() { @@ -214,7 +212,7 @@ func (pool *BlockPool) GetStatus() (height int64, numPending int32, lenRequester pool.mtx.RLock() defer pool.mtx.RUnlock() - return pool.height, atomic.LoadInt32(&pool.numPending), len(pool.requesters) + return pool.height, pool.numPending.Load(), len(pool.requesters) } // IsCaughtUp returns true if this node is caught up, false - otherwise. @@ -243,11 +241,11 @@ func (pool *BlockPool) PeekTwoBlocks() (first, second *types.Block) { pool.mtx.RLock() defer pool.mtx.RUnlock() - if r := pool.requesters[pool.height]; r != nil { - first = r.getBlock() + if r, ok := pool.requesters[pool.height]; ok { + first = r.getBlock().Or(nil) } - if r := pool.requesters[pool.height+1]; r != nil { - second = r.getBlock() + if r, ok := pool.requesters[pool.height+1]; ok { + second = r.getBlock().Or(nil) } return } @@ -258,8 +256,11 @@ func (pool *BlockPool) PopRequest() { pool.mtx.Lock() defer pool.mtx.Unlock() - if r := pool.requesters[pool.height]; r != nil { - r.Stop() + if r, ok := pool.requesters[pool.height]; ok { + for inner, ctrl := range r.inner.Lock() { + inner.done = true + ctrl.Updated() + } delete(pool.requesters, pool.height) pool.height++ pool.lastAdvance = time.Now() @@ -284,19 +285,17 @@ func (pool *BlockPool) PopRequest() { // RedoRequest invalidates the block at pool.height, // Remove the peer and redo request from others. // Returns the ID of the removed peer. -func (pool *BlockPool) RedoRequest(height int64) types.NodeID { +func (pool *BlockPool) RedoRequest(height int64) utils.Option[types.NodeID] { pool.mtx.Lock() defer pool.mtx.Unlock() request := pool.requesters[height] peerID := request.getPeerID() - if peerID != types.NodeID("") { - pool.removePeer(peerID, false) - } - // Redo all requesters associated with this peer. - for _, requester := range pool.requesters { - if requester.getPeerID() == peerID { - requester.redo(peerID, BadBlock) + if id, ok := peerID.Get(); ok { + pool.removePeer(id, false) + // Redo all requesters associated with this peer. + for _, r := range pool.requesters { + r.reset(id, true) } } return peerID @@ -339,7 +338,7 @@ func (pool *BlockPool) AddBlock(peerID types.NodeID, block *types.Block, blockSi setBlockResult := requester.setBlock(block, peerID) if setBlockResult == 0 { - atomic.AddInt32(&pool.numPending, -1) + pool.numPending.Add(-1) peer := pool.peers[peerID] if peer != nil { peer.decrPending(blockSize) @@ -349,7 +348,7 @@ func (pool *BlockPool) AddBlock(peerID types.NodeID, block *types.Block, blockSi pendingErr = errors.New("requester is different or block already exists") pendingPeerID = peerID - return fmt.Errorf("%w (peer: %s, requester: %s, block height: %d)", pendingErr, peerID, requester.getPeerID(), block.Height) + return fmt.Errorf("%w (peer: %s, requester: %v, block height: %d)", pendingErr, peerID, requester.getPeerID(), block.Height) } // MaxPeerHeight returns the highest reported height. @@ -412,16 +411,13 @@ func (pool *BlockPool) SetPeerRange(peerID types.NodeID, base int64, height int6 func (pool *BlockPool) RemovePeer(peerID types.NodeID) { pool.mtx.Lock() defer pool.mtx.Unlock() - pool.removePeer(peerID, true) } func (pool *BlockPool) removePeer(peerID types.NodeID, redo bool) { if redo { for _, requester := range pool.requesters { - if requester.getPeerID() == peerID { - requester.redo(peerID, PeerRemoved) - } + requester.reset(peerID, false) } } @@ -492,46 +488,27 @@ func (pool *BlockPool) pickIncrAvailablePeer(height int64) *bpPeer { return nil } -func (pool *BlockPool) makeNextRequester(ctx context.Context) { +func (pool *BlockPool) makeNextRequester() (*bpRequester, bool) { pool.mtx.Lock() defer pool.mtx.Unlock() nextHeight := pool.height + pool.requestersLen() if nextHeight > pool.maxPeerHeight { - return + return nil, false } - request := newBPRequester(pool, nextHeight) - - pool.requesters[nextHeight] = request - atomic.AddInt32(&pool.numPending, 1) + r := newBPRequester(pool, nextHeight) - err := request.Start(ctx) - if err != nil { - logger.Error("error starting request", "err", err) - } + pool.requesters[nextHeight] = r + pool.numPending.Add(1) + return r, true } func (pool *BlockPool) requestersLen() int64 { return int64(len(pool.requesters)) } -func (pool *BlockPool) sendRequest(ctx context.Context, height int64, peerID types.NodeID) bool { - if !pool.IsRunning() { - return false - } - select { - case pool.requestsCh <- BlockRequest{height, peerID}: - return true - case <-ctx.Done(): - return false - } -} - func (pool *BlockPool) sendError(err error, peerID types.NodeID) { - if !pool.IsRunning() { - return - } pool.reportErr(peerError{err, peerID}) } @@ -609,15 +586,16 @@ func (peer *bpPeer) onTimeout() { //------------------------------------- +type bpRequesterInner struct { + peerID utils.Option[types.NodeID] + block utils.Option[*types.Block] + done bool +} + type bpRequester struct { - pool *BlockPool - height int64 - gotBlockCh chan struct{} - redoCh chan RedoOp // redo may send multitime, add peerId to identify repeat - timeoutTicker *time.Ticker - mtx sync.Mutex - peerID types.NodeID - block *types.Block + pool *BlockPool + height int64 + inner utils.Watch[*bpRequesterInner] } type RetryReason string @@ -629,72 +607,56 @@ type RedoOp struct { func newBPRequester(pool *BlockPool, height int64) *bpRequester { return &bpRequester{ - pool: pool, - height: height, - gotBlockCh: make(chan struct{}, 1), - redoCh: make(chan RedoOp, 1), - timeoutTicker: time.NewTicker(peerTimeout), - peerID: "", - block: nil, + pool: pool, + height: height, + inner: utils.NewWatch(&bpRequesterInner{}), } } // Responsible for making more requests as necessary // Returns only when a block is found (e.g. AddBlock() is called) func (bpr *bpRequester) run(ctx context.Context) error { - defer bpr.timeoutTicker.Stop() - -OUTER_LOOP: for { + if err := ctx.Err(); err != nil { + return err + } + // Wait until reset. + for inner, ctrl := range bpr.inner.Lock() { + for { + if inner.done { + return nil + } + if !inner.peerID.IsPresent() { + break + } + if err := ctrl.Wait(ctx); err != nil { + return err + } + } + } // Pick a peer to send request to. var peer *bpPeer - PICK_PEER_LOOP: for { - if ctx.Err() != nil { - return nil + if peer = bpr.pool.pickIncrAvailablePeer(bpr.height); peer != nil { + break } - peer = bpr.pool.pickIncrAvailablePeer(bpr.height) - if peer == nil { - // This is preferable to using a timer because the request - // interval is so small. Larger request intervals may - // necessitate using a timer/ticker. - time.Sleep(requestInterval) - continue PICK_PEER_LOOP + if err := utils.Sleep(ctx, requestInterval); err != nil { + return err } - break PICK_PEER_LOOP } - bpr.mtx.Lock() - bpr.peerID = peer.id - bpr.mtx.Unlock() - - // Send request and wait. - if !bpr.pool.sendRequest(ctx, bpr.height, peer.id) { - return nil + for inner := range bpr.inner.Lock() { + inner.peerID = utils.Some(peer.id) } - bpr.timeoutTicker.Reset(peerTimeout) - WAIT_LOOP: - for { - select { - case <-ctx.Done(): - return nil - case redoOp := <-bpr.redoCh: - // if we don't have an existing block or this is a bad block - // we should reset the previous block - if bpr.reset(redoOp.Reason == BadBlock) { - continue OUTER_LOOP - } - continue WAIT_LOOP - case <-bpr.timeoutTicker.C: - if bpr.reset(false) { - continue OUTER_LOOP - } else { - continue WAIT_LOOP - } - case <-bpr.gotBlockCh: - // Keep the requester alive until its block is either popped or - // invalidated. Validation happens outside the requester, so a bad - // block must still be able to trigger redo(). - continue WAIT_LOOP + // Send request. + if err := utils.Send(ctx, bpr.pool.requestsCh, BlockRequest{bpr.height, peer.id}); err != nil { + return err + } + // Wait for response with timeout + for inner, ctrl := range bpr.inner.Lock() { + if err := utils.WithTimeout(ctx, peerTimeout, func(ctx context.Context) error { + return ctrl.WaitUntil(ctx, func() bool { return inner.block.IsPresent() }) + }); err != nil { + inner.peerID = utils.None[types.NodeID]() } } } @@ -704,68 +666,41 @@ OUTER_LOOP: // Returns -1 if peer doesn't match. // Return 1 if block exist and peer matches. func (bpr *bpRequester) setBlock(block *types.Block, peerID types.NodeID) int { - bpr.mtx.Lock() - defer bpr.mtx.Unlock() - if bpr.peerID != peerID { - return -1 - } - if bpr.block != nil { - return 1 - } - bpr.block = block - select { - case bpr.gotBlockCh <- struct{}{}: - default: + for inner, ctrl := range bpr.inner.Lock() { + if inner.peerID != utils.Some(peerID) { + return -1 + } + if inner.block.IsPresent() { + return 1 + } + inner.block = utils.Some(block) + ctrl.Updated() } return 0 } -func (bpr *bpRequester) getBlock() *types.Block { - bpr.mtx.Lock() - defer bpr.mtx.Unlock() - return bpr.block -} - -func (bpr *bpRequester) getPeerID() types.NodeID { - bpr.mtx.Lock() - defer bpr.mtx.Unlock() - return bpr.peerID -} - -// This is called from the requestRoutine, upon redo(). -func (bpr *bpRequester) reset(force bool) bool { - bpr.mtx.Lock() - defer bpr.mtx.Unlock() - - if bpr.block != nil && !force { - // Do not reset if we already have a block - return false +func (bpr *bpRequester) getBlock() utils.Option[*types.Block] { + for inner := range bpr.inner.Lock() { + return inner.block } + panic("unreachable") +} - if bpr.block != nil { - atomic.AddInt32(&bpr.pool.numPending, 1) +func (bpr *bpRequester) getPeerID() utils.Option[types.NodeID] { + for inner := range bpr.inner.Lock() { + return inner.peerID } - - bpr.peerID = "" - bpr.block = nil - return true + panic("unreachable") } -// Tells bpRequester to pick another peer and try again. -// NOTE: Nonblocking, and does nothing if another redo -// was already requested. -func (bpr *bpRequester) redo(peerID types.NodeID, retryReason RetryReason) { - if retryReason == BadBlock { - // Clear the bad block immediately so callers do not keep re-reading the - // same invalid pair before the requester goroutine processes redoCh. - bpr.reset(true) - } - select { - case bpr.redoCh <- RedoOp{ - PeerId: peerID, - Reason: retryReason, - }: - default: +func (bpr *bpRequester) reset(peerID types.NodeID, force bool) bool { + for inner, ctrl := range bpr.inner.Lock() { + if inner.peerID != utils.Some(peerID) || (inner.block.IsPresent() && !force) { + return false + } + inner.peerID = utils.None[types.NodeID]() + inner.block = utils.None[*types.Block]() + ctrl.Updated() } + return true } - diff --git a/sei-tendermint/internal/blocksync/pool_test.go b/sei-tendermint/internal/blocksync/pool_test.go index 4dc7927740..f2c9044d5b 100644 --- a/sei-tendermint/internal/blocksync/pool_test.go +++ b/sei-tendermint/internal/blocksync/pool_test.go @@ -318,10 +318,6 @@ func TestBlockPoolAddBlockReleasesLockBeforeSend(t *testing.T) { } mtxUnlocked <- unlocked }) - pool.running.Store(true) - t.Cleanup(func() { - pool.running.Store(false) - }) // pool.height starts at 1 and the peer reports height 100, so no // requester is created for far-ahead heights. A block more than diff --git a/sei-tendermint/internal/blocksync/reactor.go b/sei-tendermint/internal/blocksync/reactor.go index dbe2897a1d..62d6b4f87d 100644 --- a/sei-tendermint/internal/blocksync/reactor.go +++ b/sei-tendermint/internal/blocksync/reactor.go @@ -583,12 +583,11 @@ func (s *syncController) poolRoutine(ctx context.Context, pool *BlockPool, initi "err", err, ) - peerID := pool.RedoRequest(first.Height) - s.router.Evict(peerID, fmt.Errorf("blocksync: %w", err)) - - peerID2 := pool.RedoRequest(second.Height) - if peerID2 != peerID { - s.router.Evict(peerID2, fmt.Errorf("blocksync: %w", err)) + if peerID, ok := pool.RedoRequest(first.Height).Get(); ok { + s.router.Evict(peerID, fmt.Errorf("blocksync: %w", err)) + } + if peerID, ok := pool.RedoRequest(second.Height).Get(); ok { + s.router.Evict(peerID, fmt.Errorf("blocksync: %w", err)) } continue } diff --git a/sei-tendermint/internal/blocksync/reactor_test.go b/sei-tendermint/internal/blocksync/reactor_test.go index 54d634c384..dd3d4963b2 100644 --- a/sei-tendermint/internal/blocksync/reactor_test.go +++ b/sei-tendermint/internal/blocksync/reactor_test.go @@ -499,7 +499,7 @@ func TestPoolRoutine_DoesNotReturnOnValidationFailure(t *testing.T) { badPeer := types.NodeID(strings.Repeat("a", 40)) goodPeer := types.NodeID(strings.Repeat("b", 40)) router := makeRouter(testPeers{ - badPeer: {id: badPeer, base: 1, height: 2, inputChan: make(chan inputData, 1)}, + badPeer: {id: badPeer, base: 1, height: 2, inputChan: make(chan inputData, 1)}, goodPeer: {id: goodPeer, base: 1, height: 2, inputChan: make(chan inputData, 1)}, }) pool := NewBlockPool(1, router) diff --git a/sei-tendermint/libs/utils/option.go b/sei-tendermint/libs/utils/option.go index 58cf58b66c..2dd26f2a79 100644 --- a/sei-tendermint/libs/utils/option.go +++ b/sei-tendermint/libs/utils/option.go @@ -33,7 +33,7 @@ func (o Option[T]) IsPresent() bool { } // Or returns the value if present, otherwise returns the default value. -func (o *Option[T]) Or(def T) T { +func (o Option[T]) Or(def T) T { if o.isPresent { return o.value } From 49271a02dab7e07a564d2699fc1ba780ed84ccc9 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 28 May 2026 11:52:08 +0200 Subject: [PATCH 076/100] regression fix --- sei-tendermint/internal/blocksync/pool.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sei-tendermint/internal/blocksync/pool.go b/sei-tendermint/internal/blocksync/pool.go index 7a021bb3af..b27e658b9a 100644 --- a/sei-tendermint/internal/blocksync/pool.go +++ b/sei-tendermint/internal/blocksync/pool.go @@ -698,6 +698,9 @@ func (bpr *bpRequester) reset(peerID types.NodeID, force bool) bool { if inner.peerID != utils.Some(peerID) || (inner.block.IsPresent() && !force) { return false } + if inner.block.IsPresent() { + bpr.pool.numPending.Add(1) + } inner.peerID = utils.None[types.NodeID]() inner.block = utils.None[*types.Block]() ctrl.Updated() From 93f2ecd0ba416f675452c15b62b8eddddfb1a7cd Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 28 May 2026 11:57:11 +0200 Subject: [PATCH 077/100] another regression --- sei-tendermint/internal/blocksync/pool.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sei-tendermint/internal/blocksync/pool.go b/sei-tendermint/internal/blocksync/pool.go index b27e658b9a..6f4a7d4b42 100644 --- a/sei-tendermint/internal/blocksync/pool.go +++ b/sei-tendermint/internal/blocksync/pool.go @@ -652,12 +652,13 @@ func (bpr *bpRequester) run(ctx context.Context) error { return err } // Wait for response with timeout - for inner, ctrl := range bpr.inner.Lock() { - if err := utils.WithTimeout(ctx, peerTimeout, func(ctx context.Context) error { + if err := utils.WithTimeout(ctx, peerTimeout, func(ctx context.Context) error { + for inner, ctrl := range bpr.inner.Lock() { return ctrl.WaitUntil(ctx, func() bool { return inner.block.IsPresent() }) - }); err != nil { - inner.peerID = utils.None[types.NodeID]() } + panic("unreachable") + }); err != nil { + bpr.reset(peer.id, false) } } } From 244347b0b939996c5f57c52e8ce3f0fde47033ca Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 28 May 2026 12:02:44 +0200 Subject: [PATCH 078/100] mempool test flake fix --- .../internal/mempool/reactor/reactor_test.go | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/sei-tendermint/internal/mempool/reactor/reactor_test.go b/sei-tendermint/internal/mempool/reactor/reactor_test.go index a866b91516..100692c723 100644 --- a/sei-tendermint/internal/mempool/reactor/reactor_test.go +++ b/sei-tendermint/internal/mempool/reactor/reactor_test.go @@ -3,7 +3,6 @@ package reactor import ( "context" "fmt" - "math/rand" "os" "runtime" "strings" @@ -53,21 +52,17 @@ func setupMempool(t testing.TB, app *proxy.Proxy, cacheSize int, txConstraintsFe return mempool.NewTxMempool(cfg.Mempool.ToMempoolConfig(), app, mempool.NopMetrics(), txConstraintsFetcher) } -func checkTxs(ctx context.Context, t *testing.T, txmp *mempool.TxMempool, numTxs int) []testTx { +func checkTxs(ctx context.Context, t *testing.T, rng utils.Rng, txmp *mempool.TxMempool, numTxs int) []testTx { t.Helper() txs := make([]testTx, numTxs) - rng := rand.New(rand.NewSource(time.Now().UnixNano())) for i := range numTxs { - prefix := make([]byte, 20) - _, err := rng.Read(prefix) - require.NoError(t, err) - + prefix := utils.GenBytes(rng, 20) txs[i] = testTx{ tx: []byte(fmt.Sprintf("sender-%d=%X=%d", i, prefix, i+1000)), } - _, err = txmp.CheckTx(ctx, txs[i].tx) + _, err := txmp.CheckTx(ctx, txs[i].tx) require.NoError(t, err) } @@ -199,6 +194,7 @@ func TestReactorBroadcastTxs(t *testing.T) { numTxs := 512 numNodes := 4 ctx := t.Context() + rng := utils.TestRng() rts := setupReactors(ctx, t, numNodes) t.Cleanup(leaktest.Check(t)) @@ -206,7 +202,7 @@ func TestReactorBroadcastTxs(t *testing.T) { primary := rts.nodes[0] secondaries := rts.nodes[1:] - txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs) + txs := checkTxs(ctx, t, rng, rts.reactors[primary].mempool, numTxs) require.Equal(t, numTxs, rts.reactors[primary].mempool.Size()) @@ -344,6 +340,7 @@ func TestReactorConcurrency(t *testing.T) { numTxs := 10 numNodes := 2 ctx := t.Context() + rng := utils.TestRng() rts := setupReactors(ctx, t, numNodes) t.Cleanup(leaktest.Check(t)) @@ -358,8 +355,9 @@ func TestReactorConcurrency(t *testing.T) { var secondaryHeight int64 for range runtime.NumCPU() * 2 { + primaryRng := rng.Split() wg.Go(func() { - txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs) + txs := checkTxs(ctx, t, primaryRng, rts.reactors[primary].mempool, numTxs) txmp := rts.mempools[primary] txmp.Lock() @@ -375,8 +373,9 @@ func TestReactorConcurrency(t *testing.T) { require.NoError(t, txmp.Update(ctx, height, convertTex(txs), deliverTxResponses, mempool.NopTxConstraints(), true)) }) + secondaryRng := rng.Split() wg.Go(func() { - _ = checkTxs(ctx, t, rts.reactors[secondary].mempool, numTxs) + _ = checkTxs(ctx, t, secondaryRng, rts.reactors[secondary].mempool, numTxs) txmp := rts.mempools[secondary] txmp.Lock() @@ -426,6 +425,7 @@ func TestBroadcastTxForPeerStopsWhenPeerStops(t *testing.T) { } ctx := t.Context() + rng := utils.TestRng() rts := setupReactors(ctx, t, 2) t.Cleanup(leaktest.Check(t)) @@ -436,7 +436,7 @@ func TestBroadcastTxForPeerStopsWhenPeerStops(t *testing.T) { rts.start(t) rts.network.Remove(t, secondary) - txs := checkTxs(ctx, t, rts.reactors[primary].mempool, 4) + txs := checkTxs(ctx, t, rng, rts.reactors[primary].mempool, 4) require.Equal(t, 4, len(txs)) require.Equal(t, 4, rts.mempools[primary].Size()) require.Equal(t, 0, rts.mempools[secondary].Size()) From 32060445f316a684f9cccc69aa2e53b96d952486 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 28 May 2026 12:40:34 +0200 Subject: [PATCH 079/100] IgnoreCancel --- sei-tendermint/internal/blocksync/reactor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sei-tendermint/internal/blocksync/reactor.go b/sei-tendermint/internal/blocksync/reactor.go index 62d6b4f87d..f73e393867 100644 --- a/sei-tendermint/internal/blocksync/reactor.go +++ b/sei-tendermint/internal/blocksync/reactor.go @@ -386,7 +386,7 @@ func (s *syncController) run(ctx context.Context) error { handoff, err := scope.Run1(ctx, func(ctx context.Context, session scope.Scope) (consensusHandoff, error) { session.SpawnBgNamed("pool.run", func() error { - return pool.run(ctx) + return utils.IgnoreCancel(pool.run(ctx)) }) return s.poolRoutine(ctx, pool, res.state, res.stateSynced) }) From 4c1559e4668aea32d20149a853bd6aa5bc702ba8 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 28 May 2026 14:51:09 +0200 Subject: [PATCH 080/100] busy loop fix --- sei-tendermint/internal/blocksync/pool.go | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/sei-tendermint/internal/blocksync/pool.go b/sei-tendermint/internal/blocksync/pool.go index 6f4a7d4b42..7c9304f3fd 100644 --- a/sei-tendermint/internal/blocksync/pool.go +++ b/sei-tendermint/internal/blocksync/pool.go @@ -145,22 +145,14 @@ func (pool *BlockPool) run(ctx context.Context) error { return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { for ctx.Err() == nil { - _, numPending, lenRequesters := pool.GetStatus() - if numPending >= maxPendingRequests || lenRequesters >= maxTotalRequesters { - // This is preferable to using a timer because the request interval - // is so small. Larger request intervals may necessitate using a - // timer/ticker. - if err := utils.Sleep(ctx, requestInterval); err != nil { - return err - } - pool.removeTimedoutPeers() - continue - } - - // request for more blocks. if r, ok := pool.makeNextRequester(); ok { s.Spawn(func() error { return r.run(ctx) }) + continue } + if err := utils.Sleep(ctx, requestInterval); err != nil { + return err + } + pool.removeTimedoutPeers() } return ctx.Err() }) @@ -491,9 +483,8 @@ func (pool *BlockPool) pickIncrAvailablePeer(height int64) *bpPeer { func (pool *BlockPool) makeNextRequester() (*bpRequester, bool) { pool.mtx.Lock() defer pool.mtx.Unlock() - nextHeight := pool.height + pool.requestersLen() - if nextHeight > pool.maxPeerHeight { + if pool.requestersLen() >= maxTotalRequesters || pool.numPending.Load() >= maxPendingRequests || nextHeight > pool.maxPeerHeight { return nil, false } From ffcdfa6d18e50e01d9951bf5ffc428be4e998f5f Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 28 May 2026 15:19:48 +0200 Subject: [PATCH 081/100] WIP --- sei-tendermint/node/node.go | 265 +++++++++++++++++------------------- 1 file changed, 126 insertions(+), 139 deletions(-) diff --git a/sei-tendermint/node/node.go b/sei-tendermint/node/node.go index c87e316998..7bdf1ecf48 100644 --- a/sei-tendermint/node/node.go +++ b/sei-tendermint/node/node.go @@ -210,19 +210,6 @@ func makeNode( node.router = router node.rpcEnv.Router = router - // Mempool gossiping is not compatible with Giga, - // so we disable the mempool reactor. - if !gigaEnabled { - mp := mempool.NewTxMempool(cfg.Mempool.ToMempoolConfig(), proxyApp, nodeMetrics.mempool, sm.TxConstraintsFetcherFromStore(stateStore)) - node.mempool = utils.Some(mp) - mpReactor, err := mempoolreactor.NewReactor(cfg.Mempool, mp, router) - if err != nil { - return nil, fmt.Errorf("mempoolreactor.NewReactor(): %w", err) - } - mpReactor.MarkReadyToStart() - node.services = append(node.services, mpReactor) - } - evReactor, evPool, edbCloser, err := createEvidenceReactor(cfg, dbProvider, stateStore, blockStore, node.router, nodeMetrics.evidence, eventBus) closers = append(closers, edbCloser) @@ -232,66 +219,75 @@ func makeNode( node.services = append(node.services, evReactor) node.rpcEnv.EvidencePool = evPool node.evPool = evPool - - node.rpcEnv.Mempool = mp - - // make block executor for consensus and blockchain reactors to execute blocks - blockExec := sm.NewBlockExecutor( - stateStore, - proxyApp, - mp, - evPool, - blockStore, - eventBus, - nodeMetrics.state, - consensusPolicy, - ) - - // Determine whether we should attempt state sync. - stateSync := cfg.StateSync.Enable && !onlyValidatorIsUs(state, pubKey) - if stateSync && state.LastBlockHeight > 0 { - logger.Info("Found local state with non-zero height, skipping state sync") - stateSync = false - } - - // Determine whether we should do block sync. This must happen after the handshake, since the - // app may modify the validator set, specifying ourself as the only validator. - blockSync := !onlyValidatorIsUs(state, pubKey) - if gigaEnabled { - // TODO(autobahn-recovery): handles only restart with local disk intact. - // A node that lost its WAL + app CMS (new validator, disk wipe) needs both - // app state sync and an autobahn WAL sync to catch up. Not yet supported. - stateSync = false - blockSync = false + + if cfg.P2P.PexReactor { + pxReactor, err := pex.NewReactor( + node.router, + pex.DefaultSendInterval, + ) + if err != nil { + return nil, fmt.Errorf("pex.NewReactor(): %w", err) + } + node.services = append(node.services, pxReactor) } - waitSync := stateSync || blockSync - consensusWAL, err := consensus.OpenWAL(cfg.Consensus.WalFile()) - if err != nil { - return nil, fmt.Errorf("consensus.OpenWAL(): %w", err) - } - closers = append(closers, func() error { - consensusWAL.Close() - return nil - }) + if !gigaEnabled { + mp := mempool.NewTxMempool(cfg.Mempool.ToMempoolConfig(), proxyApp, nodeMetrics.mempool, sm.TxConstraintsFetcherFromStore(stateStore)) + node.mempool = utils.Some(mp) + node.rpcEnv.Mempool = mp + mpReactor, err := mempoolreactor.NewReactor(cfg.Mempool, mp, router) + if err != nil { + return nil, fmt.Errorf("mempoolreactor.NewReactor(): %w", err) + } + mpReactor.MarkReadyToStart() + node.services = append(node.services, mpReactor) + + // make block executor for consensus and blockchain reactors to execute blocks + blockExec := sm.NewBlockExecutor( + stateStore, + proxyApp, + mp, + evPool, + blockStore, + eventBus, + nodeMetrics.state, + consensusPolicy, + ) - csState := consensus.NewState( - cfg.Consensus, - consensusWAL, - stateStore, - blockExec, - blockStore, - mp, - evPool, - eventBus, - tracerProviderOptions, - nodeMetrics.consensus, - ) - node.rpcEnv.ConsensusState = csState + // Determine whether we should attempt state sync. + stateSync := cfg.StateSync.Enable && !onlyValidatorIsUs(state, pubKey) + if stateSync && state.LastBlockHeight > 0 { + logger.Info("Found local state with non-zero height, skipping state sync") + stateSync = false + } + // Determine whether we should do block sync. This must happen after the handshake, since the + // app may modify the validator set, specifying ourself as the only validator. + blockSync := !onlyValidatorIsUs(state, pubKey) + waitSync := stateSync || blockSync + + consensusWAL, err := consensus.OpenWAL(cfg.Consensus.WalFile()) + if err != nil { + return nil, fmt.Errorf("consensus.OpenWAL(): %w", err) + } + closers = append(closers, func() error { + consensusWAL.Close() + return nil + }) + csState := consensus.NewState( + cfg.Consensus, + consensusWAL, + stateStore, + blockExec, + blockStore, + mp, + evPool, + eventBus, + tracerProviderOptions, + nodeMetrics.consensus, + ) + node.rpcEnv.ConsensusState = csState - var csReactor *consensus.Reactor - if !gigaEnabled { - csReactor, err = consensus.NewReactor( + csReactor, err := consensus.NewReactor( csState, node.router, eventBus, @@ -305,77 +301,65 @@ func makeNode( node.services = append(node.services, csReactor) node.rpcEnv.ConsensusReactor = csReactor - } - - // Create the blockchain reactor. Note, we do not start block sync if we're - // doing a state sync first. - bcReactor, err := blocksync.NewReactor( - stateStore, - blockStore, - node.router, - utils.Some(blocksync.SyncerConfig{ - BlockExec: blockExec, - ConsReactor: csReactor, - BlockSync: blockSync && !stateSync, - Metrics: nodeMetrics.consensus, - EventBus: eventBus, - RestartEvent: restartEvent, - SelfRemediationConfig: cfg.SelfRemediation, - }), - ) - if err != nil { - return nil, fmt.Errorf("blocksync.NewReactor(): %w", err) - } - node.services = append(node.services, bcReactor) - node.rpcEnv.BlockSyncReactor = bcReactor - - // Make ConsensusReactor. Don't enable fully if doing a state sync and/or block sync first. - // FIXME We need to update metrics here, since other reactors don't have access to them. - if stateSync { - nodeMetrics.consensus.StateSyncing.Set(1) - } else if blockSync { - nodeMetrics.consensus.BlockSyncing.Set(1) - } - - if cfg.P2P.PexReactor { - pxReactor, err := pex.NewReactor( + + // Create the blockchain reactor. Note, we do not start block sync if we're + // doing a state sync first. + bcReactor, err := blocksync.NewReactor( + stateStore, + blockStore, node.router, - pex.DefaultSendInterval, + utils.Some(blocksync.SyncerConfig{ + BlockExec: blockExec, + ConsReactor: csReactor, + BlockSync: blockSync && !stateSync, + Metrics: nodeMetrics.consensus, + EventBus: eventBus, + RestartEvent: restartEvent, + SelfRemediationConfig: cfg.SelfRemediation, + }), ) if err != nil { - return nil, fmt.Errorf("pex.NewReactor(): %w", err) + return nil, fmt.Errorf("blocksync.NewReactor(): %w", err) } - node.services = append(node.services, pxReactor) - } - - postSyncHook := func(ctx context.Context, state sm.State) error { - csReactor.SetStateSyncingMetrics(0) - - // TODO: Some form of orchestrator is needed here between the state - // advancing reactors to be able to control which one of the three - // is running - // FIXME Very ugly to have these metrics bleed through here. - csReactor.SetBlockSyncingMetrics(1) - if err := bcReactor.SwitchToBlockSync(state); err != nil { - logger.Error("failed to switch to block sync", "err", err) - return err + node.services = append(node.services, bcReactor) + node.rpcEnv.BlockSyncReactor = bcReactor + + // Make ConsensusReactor. Don't enable fully if doing a state sync and/or block sync first. + // FIXME We need to update metrics here, since other reactors don't have access to them. + if stateSync { + nodeMetrics.consensus.StateSyncing.Set(1) + } else if blockSync { + nodeMetrics.consensus.BlockSyncing.Set(1) } + + postSyncHook := func(ctx context.Context, state sm.State) error { + csReactor.SetStateSyncingMetrics(0) + + // TODO: Some form of orchestrator is needed here between the state + // advancing reactors to be able to control which one of the three + // is running + // FIXME Very ugly to have these metrics bleed through here. + csReactor.SetBlockSyncingMetrics(1) + if err := bcReactor.SwitchToBlockSync(state); err != nil { + logger.Error("failed to switch to block sync", "err", err) + return err + } - return nil - } - // Set up state sync reactor, and schedule a sync if requested. - // FIXME The way we do phased startups (e.g. replay -> block sync -> consensus) is very messy, - // we should clean this whole thing up. See: - // https://github.com/tendermint/tendermint/issues/4644 - // The CometBFT handshaker reconciles the block store and state store with the app - // by replaying blocks and calling InitChain at genesis. Autobahn (giga) maintains - // its own data WAL and does not update the CometBFT block/state stores, so on - // restart the handshaker would observe storeHeight=0 < appHeight=N and fail with - // ErrAppBlockHeightTooHigh. We skip the handshaker in giga mode; instead the - // giga router's runExecute owns InitChain on fresh start (appHeight==0) and - // relies on the app's committed CMS to rebuild deliverState on restart. - node.shouldHandshake = !stateSync && !gigaEnabled - if !gigaEnabled { + return nil + } + + // Set up state sync reactor, and schedule a sync if requested. + // FIXME The way we do phased startups (e.g. replay -> block sync -> consensus) is very messy, + // we should clean this whole thing up. See: + // https://github.com/tendermint/tendermint/issues/4644 + // The CometBFT handshaker reconciles the block store and state store with the app + // by replaying blocks and calling InitChain at genesis. Autobahn (giga) maintains + // its own data WAL and does not update the CometBFT block/state stores, so on + // restart the handshaker would observe storeHeight=0 < appHeight=N and fail with + // ErrAppBlockHeightTooHigh. We skip the handshaker in giga mode; instead the + // giga router's runExecute owns InitChain on fresh start (appHeight==0) and + // relies on the app's committed CMS to rebuild deliverState on restart. + node.shouldHandshake = !stateSync ssReactor, err := statesync.NewReactor( genDoc.ChainID, genDoc.InitialHeight, @@ -397,13 +381,14 @@ func makeNode( return nil, fmt.Errorf("statesync.NewReactor(): %w", err) } node.services = append(node.services, ssReactor) - } - if cfg.Mode == config.ModeValidator { - if privValidator != nil { - csState.SetPrivValidator(ctx, utils.Some(privValidator)) + if cfg.Mode == config.ModeValidator { + if privValidator != nil { + csState.SetPrivValidator(ctx, utils.Some(privValidator)) + } } } + node.rpcEnv.PubKey = pubKey node.BaseService = *service.NewBaseService("Node", node) @@ -515,7 +500,9 @@ func (n *nodeImpl) OnStart(ctx context.Context) error { return err } n.rpcEnv.IsListening = true - n.SpawnCritical("mempool", n.mempool.Run) + if m,ok := n.mempool.Get(); ok { + n.SpawnCritical("mempool", m.Run) + } for _, reactor := range n.services { if err := reactor.Start(ctx); err != nil { From 37cf7f70c4554585e8be022fc7ca08f853374a05 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 28 May 2026 16:23:22 +0200 Subject: [PATCH 082/100] almost separated --- sei-tendermint/internal/rpc/core/consensus.go | 18 +++++- sei-tendermint/internal/rpc/core/dev.go | 6 +- sei-tendermint/internal/rpc/core/env.go | 64 +++++++++++++++---- sei-tendermint/internal/rpc/core/events.go | 10 +-- sei-tendermint/internal/rpc/core/evidence.go | 6 +- .../internal/rpc/core/lag_status.go | 6 +- sei-tendermint/internal/rpc/core/mempool.go | 38 ++++++++--- sei-tendermint/internal/rpc/core/status.go | 35 +++++----- sei-tendermint/node/node.go | 37 ++++++----- sei-tendermint/node/seed.go | 3 +- sei-tendermint/rpc/client/local/local.go | 5 +- 11 files changed, 159 insertions(+), 69 deletions(-) diff --git a/sei-tendermint/internal/rpc/core/consensus.go b/sei-tendermint/internal/rpc/core/consensus.go index 6958766883..1c5f4c171f 100644 --- a/sei-tendermint/internal/rpc/core/consensus.go +++ b/sei-tendermint/internal/rpc/core/consensus.go @@ -91,13 +91,21 @@ func (env *Environment) Validators(ctx context.Context, req *coretypes.RequestVa // More: https://docs.tendermint.com/master/rpc/#/Info/dump_consensus_state func (env *Environment) DumpConsensusState(ctx context.Context) (*coretypes.ResultDumpConsensusState, error) { // Get Peer consensus states. + reactor, err := env.requireConsensusReactor() + if err != nil { + return nil, err + } + consensusState, err := env.requireConsensusState() + if err != nil { + return nil, err + } peerStates := map[types.NodeID]coretypes.PeerStateInfo{} for _, info := range env.Router.ConnInfos() { if _, ok := peerStates[info.ID]; ok { continue } - peerState, ok := env.ConsensusReactor.GetPeerState(info.ID) + peerState, ok := reactor.GetPeerState(info.ID) if !ok { continue } @@ -122,7 +130,7 @@ func (env *Environment) DumpConsensusState(ctx context.Context) (*coretypes.Resu } // Get self round state. - roundState, err := env.ConsensusState.GetRoundStateJSON() + roundState, err := consensusState.GetRoundStateJSON() if err != nil { return nil, err } @@ -136,8 +144,12 @@ func (env *Environment) DumpConsensusState(ctx context.Context) (*coretypes.Resu // UNSTABLE // More: https://docs.tendermint.com/master/rpc/#/Info/consensus_state func (env *Environment) GetConsensusState(ctx context.Context) (*coretypes.ResultConsensusState, error) { + consensusState, err := env.requireConsensusState() + if err != nil { + return nil, err + } // Get self round state. - bz, err := env.ConsensusState.GetRoundStateSimpleJSON() + bz, err := consensusState.GetRoundStateSimpleJSON() return &coretypes.ResultConsensusState{RoundState: bz}, err } diff --git a/sei-tendermint/internal/rpc/core/dev.go b/sei-tendermint/internal/rpc/core/dev.go index 8f90497bda..ecd0d58307 100644 --- a/sei-tendermint/internal/rpc/core/dev.go +++ b/sei-tendermint/internal/rpc/core/dev.go @@ -8,6 +8,10 @@ import ( // UnsafeFlushMempool removes all transactions from the mempool. func (env *Environment) UnsafeFlushMempool(ctx context.Context) (*coretypes.ResultUnsafeFlushMempool, error) { - env.Mempool.Flush() + mp, err := env.requireMempool() + if err != nil { + return nil, err + } + mp.Flush() return &coretypes.ResultUnsafeFlushMempool{}, nil } diff --git a/sei-tendermint/internal/rpc/core/env.go b/sei-tendermint/internal/rpc/core/env.go index aafba973e3..a4a8f6c4a2 100644 --- a/sei-tendermint/internal/rpc/core/env.go +++ b/sei-tendermint/internal/rpc/core/env.go @@ -52,7 +52,7 @@ var logger = seilog.NewLogger("tendermint", "internal", "rpc", "core") //---------------------------------------------- // These interfaces are used by RPC and must be thread safe -type consensusState interface { +type ConsensusState interface { GetState() sm.State GetValidators() (int64, []*types.Validator) GetLastHeight() int64 @@ -70,10 +70,10 @@ type Environment struct { // interfaces defined in types and above StateStore sm.Store BlockStore sm.BlockStore - EvidencePool sm.EvidencePool - ConsensusState consensusState - ConsensusReactor *consensus.Reactor - BlockSyncReactor blocksync.Metricer + EvidencePool utils.Option[sm.EvidencePool] + ConsensusState utils.Option[ConsensusState] + ConsensusReactor utils.Option[*consensus.Reactor] + BlockSyncReactor utils.Option[blocksync.Metricer] IsListening bool Listeners []string @@ -86,9 +86,9 @@ type Environment struct { GenDoc *types.GenesisDoc // cache the genesis structure EventSinks []indexer.EventSink EventBus *eventbus.EventBus // thread safe - EventLog *eventlog.Log - Mempool *mempool.TxMempool - StateSyncMetricer statesync.Metricer + EventLog utils.Option[*eventlog.Log] + Mempool utils.Option[*mempool.TxMempool] + StateSyncMetricer utils.Option[statesync.Metricer] Config config.RPCConfig @@ -211,10 +211,10 @@ func (env *Environment) getHeight(latestHeight int64, heightPtr *int64) (int64, } func (env *Environment) latestUncommittedHeight() int64 { - if env.ConsensusReactor != nil { + if reactor, ok := env.ConsensusReactor.Get(); ok { // consensus reactor can be nil in inspect mode. - nodeIsSyncing := env.ConsensusReactor.WaitSync() + nodeIsSyncing := reactor.WaitSync() if nodeIsSyncing { return env.BlockStore.Height() } @@ -222,6 +222,48 @@ func (env *Environment) latestUncommittedHeight() int64 { return env.BlockStore.Height() + 1 } +func (env *Environment) requireMempool() (*mempool.TxMempool, error) { + if mp, ok := env.Mempool.Get(); ok { + return mp, nil + } + return nil, fmt.Errorf("mempool is not available") +} + +func (env *Environment) requireEventLog() (*eventlog.Log, error) { + if lg, ok := env.EventLog.Get(); ok { + return lg, nil + } + return nil, fmt.Errorf("event log is not enabled") +} + +func (env *Environment) requireEvidencePool() (sm.EvidencePool, error) { + if pool, ok := env.EvidencePool.Get(); ok { + return pool, nil + } + return nil, fmt.Errorf("evidence pool is not available") +} + +func (env *Environment) requireConsensusState() (ConsensusState, error) { + if state, ok := env.ConsensusState.Get(); ok { + return state, nil + } + return nil, fmt.Errorf("consensus state is not available") +} + +func (env *Environment) requireConsensusReactor() (*consensus.Reactor, error) { + if reactor, ok := env.ConsensusReactor.Get(); ok { + return reactor, nil + } + return nil, fmt.Errorf("consensus reactor is not available") +} + +func (env *Environment) requireBlockSyncReactor() (blocksync.Metricer, error) { + if reactor, ok := env.BlockSyncReactor.Get(); ok { + return reactor, nil + } + return nil, fmt.Errorf("block sync reactor is not available") +} + // StartService constructs and starts listeners for the RPC service // according to the config object, returning an error if the service // cannot be constructed or started. The listeners, which provide @@ -258,7 +300,7 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config) ( // If the event log is enabled, subscribe to all events published to the // event bus, and forward them to the event log. - if lg := env.EventLog; lg != nil { + if lg, ok := env.EventLog.Get(); ok { // TODO(creachadair): This is kind of a hack, ideally we'd share the // observer with the indexer, but it's tricky to plumb them together. // For now, use a "normal" subscription with a big buffer allowance. diff --git a/sei-tendermint/internal/rpc/core/events.go b/sei-tendermint/internal/rpc/core/events.go index 4323581e2e..3cf83d301a 100644 --- a/sei-tendermint/internal/rpc/core/events.go +++ b/sei-tendermint/internal/rpc/core/events.go @@ -149,8 +149,9 @@ func (env *Environment) UnsubscribeAll(ctx context.Context) (*coretypes.ResultUn // of maxItems and waitTime may be capped to sensible internal maxima without // reporting an error to the caller. func (env *Environment) Events(ctx context.Context, req *coretypes.RequestEvents) (*coretypes.ResultEvents, error) { - if env.EventLog == nil { - return nil, errors.New("the event log is not enabled") + eventLog, err := env.requireEventLog() + if err != nil { + return nil, err } // Parse and validate parameters. @@ -190,7 +191,6 @@ func (env *Environment) Events(ctx context.Context, req *coretypes.RequestEvents var info eventlog.Info var items []*eventlog.Item - var err error accept := func(itm *eventlog.Item) error { // N.B. We accept up to one item more than requested, so we can tell how // to set the "more" flag in the response. @@ -211,7 +211,7 @@ func (env *Environment) Events(ctx context.Context, req *coretypes.RequestEvents // and we want to keep waiting until we have relevant results (or time out). cur := after for len(items) == 0 { - info, err = env.EventLog.WaitScan(ctx, cur, accept) + info, err = eventLog.WaitScan(ctx, cur, accept) if err != nil { // Don't report a timeout as a request failure. if errors.Is(err, context.DeadlineExceeded) { @@ -223,7 +223,7 @@ func (env *Environment) Events(ctx context.Context, req *coretypes.RequestEvents } } else { // Quick poll, return only what is already available. - info, err = env.EventLog.Scan(accept) + info, err = eventLog.Scan(accept) } if err != nil { return nil, err diff --git a/sei-tendermint/internal/rpc/core/evidence.go b/sei-tendermint/internal/rpc/core/evidence.go index 78032599b1..66015b110f 100644 --- a/sei-tendermint/internal/rpc/core/evidence.go +++ b/sei-tendermint/internal/rpc/core/evidence.go @@ -16,7 +16,11 @@ func (env *Environment) BroadcastEvidence(ctx context.Context, req *coretypes.Re if err := req.Evidence.ValidateBasic(); err != nil { return nil, fmt.Errorf("evidence.ValidateBasic failed: %w", err) } - if err := env.EvidencePool.AddEvidence(ctx, req.Evidence); err != nil { + pool, err := env.requireEvidencePool() + if err != nil { + return nil, err + } + if err := pool.AddEvidence(ctx, req.Evidence); err != nil { return nil, fmt.Errorf("failed to add evidence: %w", err) } return &coretypes.ResultBroadcastEvidence{Hash: req.Evidence.Hash()}, nil diff --git a/sei-tendermint/internal/rpc/core/lag_status.go b/sei-tendermint/internal/rpc/core/lag_status.go index 7d57ba1b18..e38c67168c 100644 --- a/sei-tendermint/internal/rpc/core/lag_status.go +++ b/sei-tendermint/internal/rpc/core/lag_status.go @@ -9,7 +9,11 @@ import ( // LagStatus returns Tendermint lag status, if lag is over a certain threshold func (env *Environment) LagStatus(ctx context.Context) (*coretypes.ResultLagStatus, error) { currentHeight := env.BlockStore.Height() - maxPeerBlockHeight := env.BlockSyncReactor.GetMaxPeerBlockHeight() + blockSyncReactor, err := env.requireBlockSyncReactor() + if err != nil { + return nil, err + } + maxPeerBlockHeight := blockSyncReactor.GetMaxPeerBlockHeight() lag := int64(0) // Calculate lag diff --git a/sei-tendermint/internal/rpc/core/mempool.go b/sei-tendermint/internal/rpc/core/mempool.go index b61cdae14c..dfff3851a2 100644 --- a/sei-tendermint/internal/rpc/core/mempool.go +++ b/sei-tendermint/internal/rpc/core/mempool.go @@ -36,7 +36,11 @@ func (env *Environment) EvmProxy(sender common.Address) (*url.URL, bool) { // https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_async // Deprecated and should be removed in 0.37 func (env *Environment) BroadcastTxAsync(ctx context.Context, req *coretypes.RequestBroadcastTx) (*coretypes.ResultBroadcastTx, error) { - go func() { _, _ = env.Mempool.CheckTx(ctx, req.Tx) }() + mp, err := env.requireMempool() + if err != nil { + return nil, err + } + go func() { _, _ = mp.CheckTx(ctx, req.Tx) }() return &coretypes.ResultBroadcastTx{Hash: req.Tx.Hash().Bytes()}, nil } @@ -50,7 +54,11 @@ func (env *Environment) BroadcastTxSync(ctx context.Context, req *coretypes.Requ // DeliverTx result. // More: https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_sync func (env *Environment) BroadcastTx(ctx context.Context, req *coretypes.RequestBroadcastTx) (*coretypes.ResultBroadcastTx, error) { - r, err := env.Mempool.CheckTx(ctx, req.Tx) + mp, err := env.requireMempool() + if err != nil { + return nil, err + } + r, err := mp.CheckTx(ctx, req.Tx) if err != nil { return nil, err } @@ -66,7 +74,11 @@ func (env *Environment) BroadcastTx(ctx context.Context, req *coretypes.RequestB // BroadcastTxCommit returns with the responses from CheckTx and DeliverTx. // More: https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_commit func (env *Environment) BroadcastTxCommit(ctx context.Context, req *coretypes.RequestBroadcastTx) (*coretypes.ResultBroadcastTxCommit, error) { - r, err := env.Mempool.CheckTx(ctx, req.Tx) + mp, err := env.requireMempool() + if err != nil { + return nil, err + } + r, err := mp.CheckTx(ctx, req.Tx) if err != nil { return nil, err } @@ -127,7 +139,11 @@ func (env *Environment) BroadcastTxCommit(ctx context.Context, req *coretypes.Re // UnconfirmedTxs gets unconfirmed transactions from the mempool in order of priority // More: https://docs.tendermint.com/master/rpc/#/Info/unconfirmed_txs func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.RequestUnconfirmedTxs) (*coretypes.ResultUnconfirmedTxs, error) { - totalCount := env.Mempool.Size() + mp, err := env.requireMempool() + if err != nil { + return nil, err + } + totalCount := mp.Size() perPage := env.validatePerPage(req.PerPage.IntPtr()) page, err := validatePage(req.Page.IntPtr(), perPage, totalCount) if err != nil { @@ -136,7 +152,7 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque skipCount := validateSkipCount(page, perPage) - txs, _ := env.Mempool.ReapTxs(mempool.ReapLimits{ + txs, _ := mp.ReapTxs(mempool.ReapLimits{ MaxTxs: utils.Some(uint64(skipCount + tmmath.MinInt(perPage, totalCount-skipCount))), //nolint:gosec // guaranteed to be non-negative }, false) if skipCount > len(txs) { @@ -147,7 +163,7 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque return &coretypes.ResultUnconfirmedTxs{ Count: len(result), Total: totalCount, - TotalBytes: utils.Clamp[int64](env.Mempool.SizeBytes()), + TotalBytes: utils.Clamp[int64](mp.SizeBytes()), Txs: result, }, nil } @@ -155,10 +171,14 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque // NumUnconfirmedTxs gets number of unconfirmed transactions. // More: https://docs.tendermint.com/master/rpc/#/Info/num_unconfirmed_txs func (env *Environment) NumUnconfirmedTxs(ctx context.Context) (*coretypes.ResultUnconfirmedTxs, error) { + mp, err := env.requireMempool() + if err != nil { + return nil, err + } return &coretypes.ResultUnconfirmedTxs{ - Count: env.Mempool.Size(), - Total: env.Mempool.Size(), - TotalBytes: utils.Clamp[int64](env.Mempool.SizeBytes()), + Count: mp.Size(), + Total: mp.Size(), + TotalBytes: utils.Clamp[int64](mp.SizeBytes()), }, nil } diff --git a/sei-tendermint/internal/rpc/core/status.go b/sei-tendermint/internal/rpc/core/status.go index 04b981104b..d7e91c6075 100644 --- a/sei-tendermint/internal/rpc/core/status.go +++ b/sei-tendermint/internal/rpc/core/status.go @@ -131,24 +131,24 @@ func (env *Environment) Status(ctx context.Context) (*coretypes.ResultStatus, er ValidatorInfo: validatorInfo, } - if env.ConsensusReactor != nil { - result.SyncInfo.CatchingUp = env.ConsensusReactor.WaitSync() + if reactor, ok := env.ConsensusReactor.Get(); ok { + result.SyncInfo.CatchingUp = reactor.WaitSync() } - if env.BlockSyncReactor != nil { - result.SyncInfo.MaxPeerBlockHeight = env.BlockSyncReactor.GetMaxPeerBlockHeight() - result.SyncInfo.TotalSyncedTime = env.BlockSyncReactor.GetTotalSyncedTime() - result.SyncInfo.RemainingTime = env.BlockSyncReactor.GetRemainingSyncTime() + if reactor, ok := env.BlockSyncReactor.Get(); ok { + result.SyncInfo.MaxPeerBlockHeight = reactor.GetMaxPeerBlockHeight() + result.SyncInfo.TotalSyncedTime = reactor.GetTotalSyncedTime() + result.SyncInfo.RemainingTime = reactor.GetRemainingSyncTime() } - if env.StateSyncMetricer != nil { - result.SyncInfo.TotalSnapshots = env.StateSyncMetricer.TotalSnapshots() - result.SyncInfo.ChunkProcessAvgTime = env.StateSyncMetricer.ChunkProcessAvgTime() - result.SyncInfo.SnapshotHeight = env.StateSyncMetricer.SnapshotHeight() - result.SyncInfo.SnapshotChunksCount = env.StateSyncMetricer.SnapshotChunksCount() - result.SyncInfo.SnapshotChunksTotal = env.StateSyncMetricer.SnapshotChunksTotal() - result.SyncInfo.BackFilledBlocks = env.StateSyncMetricer.BackFilledBlocks() - result.SyncInfo.BackFillBlocksTotal = env.StateSyncMetricer.BackFillBlocksTotal() + if metricer, ok := env.StateSyncMetricer.Get(); ok { + result.SyncInfo.TotalSnapshots = metricer.TotalSnapshots() + result.SyncInfo.ChunkProcessAvgTime = metricer.ChunkProcessAvgTime() + result.SyncInfo.SnapshotHeight = metricer.SnapshotHeight() + result.SyncInfo.SnapshotChunksCount = metricer.SnapshotChunksCount() + result.SyncInfo.SnapshotChunksTotal = metricer.SnapshotChunksTotal() + result.SyncInfo.BackFilledBlocks = metricer.BackFilledBlocks() + result.SyncInfo.BackFillBlocksTotal = metricer.BackFillBlocksTotal() } return result, nil @@ -164,17 +164,14 @@ func (env *Environment) validatorAtHeight(h int64) utils.Option[*types.Validator if err != nil { return none } - if env.ConsensusState == nil { - return none - } privValAddress := k.Address() // Skip the in-memory consensus-state lookup under Autobahn: the CometBFT // consensus State is never advanced, so GetValidators would nil-deref // on an unpopulated validator set. The state-store lookup below is kept // in sync under both engines. - if !env.gigaRouter().IsPresent() { - lastBlockHeight, vals := env.ConsensusState.GetValidators() + if consensusState, ok := env.ConsensusState.Get(); ok && !env.gigaRouter().IsPresent() { + lastBlockHeight, vals := consensusState.GetValidators() if lastBlockHeight == h { for _, val := range vals { if bytes.Equal(val.Address, privValAddress) { diff --git a/sei-tendermint/node/node.go b/sei-tendermint/node/node.go index 7bdf1ecf48..4d66999ac7 100644 --- a/sei-tendermint/node/node.go +++ b/sei-tendermint/node/node.go @@ -58,7 +58,7 @@ type nodeImpl struct { // network router *p2p.Router - ServiceRestartCh chan []string + ServiceRestartCh utils.Option[chan []string] nodeInfo types.NodeInfo nodeKey types.NodeKey // our node privkey @@ -68,13 +68,13 @@ type nodeImpl struct { stateStore sm.Store blockStore *store.BlockStore // store the blockchain to disk mempool utils.Option[*mempool.TxMempool] - evPool *evidence.Pool + evPool utils.Option[*evidence.Pool] indexerService *indexer.Service services []service.Service rpcListeners []net.Listener // rpc servers shutdownOps closer rpcEnv *rpccore.Environment - prometheusSrv *http.Server + prometheusSrv utils.Option[*http.Server] } // makeNode returns a new, ready to go, Tendermint Node. @@ -158,6 +158,10 @@ func makeNode( } pubKey = utils.Some(key) } + eventLogOpt := utils.None[*eventlog.Log]() + if eventLog != nil { + eventLogOpt = utils.Some(eventLog) + } // TODO construct node here: node := &nodeImpl{ config: cfg, @@ -183,7 +187,7 @@ func makeNode( GenDoc: genDoc, EventSinks: eventSinks, EventBus: eventBus, - EventLog: eventLog, + EventLog: eventLogOpt, Config: *cfg.RPC, }, } @@ -217,8 +221,8 @@ func makeNode( return nil, fmt.Errorf("createEvidenceReactor(): %w", err) } node.services = append(node.services, evReactor) - node.rpcEnv.EvidencePool = evPool - node.evPool = evPool + node.rpcEnv.EvidencePool = utils.Some[sm.EvidencePool](evPool) + node.evPool = utils.Some(evPool) if cfg.P2P.PexReactor { pxReactor, err := pex.NewReactor( @@ -234,7 +238,7 @@ func makeNode( if !gigaEnabled { mp := mempool.NewTxMempool(cfg.Mempool.ToMempoolConfig(), proxyApp, nodeMetrics.mempool, sm.TxConstraintsFetcherFromStore(stateStore)) node.mempool = utils.Some(mp) - node.rpcEnv.Mempool = mp + node.rpcEnv.Mempool = utils.Some(mp) mpReactor, err := mempoolreactor.NewReactor(cfg.Mempool, mp, router) if err != nil { return nil, fmt.Errorf("mempoolreactor.NewReactor(): %w", err) @@ -285,7 +289,7 @@ func makeNode( tracerProviderOptions, nodeMetrics.consensus, ) - node.rpcEnv.ConsensusState = csState + node.rpcEnv.ConsensusState = utils.Some[rpccore.ConsensusState](csState) csReactor, err := consensus.NewReactor( csState, @@ -300,7 +304,7 @@ func makeNode( } node.services = append(node.services, csReactor) - node.rpcEnv.ConsensusReactor = csReactor + node.rpcEnv.ConsensusReactor = utils.Some(csReactor) // Create the blockchain reactor. Note, we do not start block sync if we're // doing a state sync first. @@ -322,7 +326,7 @@ func makeNode( return nil, fmt.Errorf("blocksync.NewReactor(): %w", err) } node.services = append(node.services, bcReactor) - node.rpcEnv.BlockSyncReactor = bcReactor + node.rpcEnv.BlockSyncReactor = utils.Some[blocksync.Metricer](bcReactor) // Make ConsensusReactor. Don't enable fully if doing a state sync and/or block sync first. // FIXME We need to update metrics here, since other reactors don't have access to them. @@ -487,12 +491,14 @@ func (n *nodeImpl) OnStart(ctx context.Context) error { if err != nil { return err } - if err := n.evPool.Start(state); err != nil { - return err + if evPool, ok := n.evPool.Get(); ok { + if err := evPool.Start(state); err != nil { + return err + } } if n.config.Instrumentation.Prometheus && n.config.Instrumentation.PrometheusListenAddr != "" { - n.prometheusSrv = n.startPrometheusServer(ctx, n.config.Instrumentation.PrometheusListenAddr) + n.prometheusSrv = utils.Some(n.startPrometheusServer(ctx, n.config.Instrumentation.PrometheusListenAddr)) } // Start the transport. @@ -554,12 +560,11 @@ func (n *nodeImpl) OnStop() { pvsc.Wait() } - if n.prometheusSrv != nil { - if err := n.prometheusSrv.Shutdown(context.Background()); err != nil { + if srv, ok := n.prometheusSrv.Get(); ok { + if err := srv.Shutdown(context.Background()); err != nil { // Error from closing listeners, or context timeout: logger.Error("Prometheus HTTP server Shutdown", "err", err) } - } if err := n.shutdownOps(); err != nil { if strings.TrimSpace(err.Error()) != "" { diff --git a/sei-tendermint/node/seed.go b/sei-tendermint/node/seed.go index 7f7a077cec..f372d58e8a 100644 --- a/sei-tendermint/node/seed.go +++ b/sei-tendermint/node/seed.go @@ -12,7 +12,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/config" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/eventbus" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/pex" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" @@ -85,7 +84,7 @@ func makeSeedNode( nodeKey, utils.None[atypes.SecretKey](), cfg, - utils.None[*mempool.TxMempool](), + utils.None[*proxy.Proxy](), genDoc, dbProvider, ) diff --git a/sei-tendermint/rpc/client/local/local.go b/sei-tendermint/rpc/client/local/local.go index 2aab8bffa0..6923c10b17 100644 --- a/sei-tendermint/rpc/client/local/local.go +++ b/sei-tendermint/rpc/client/local/local.go @@ -107,7 +107,10 @@ func (c *Local) CheckTx(ctx context.Context, tx types.Tx) (*coretypes.ResultChec } func (c *Local) EvmNextPendingNonce(addr common.Address) uint64 { - return c.Mempool.EvmNextPendingNonce(addr) + if mp, ok := c.Mempool.Get(); ok { + return mp.EvmNextPendingNonce(addr) + } + return 0 } func (c *Local) EvmProxy(sender common.Address) (*url.URL, bool) { From 50ec7fa627ac643c566a6e2e6bcf1d17f45f3e32 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 28 May 2026 16:28:24 +0200 Subject: [PATCH 083/100] WIP --- sei-tendermint/internal/blocksync/reactor.go | 8 -------- sei-tendermint/internal/rpc/core/env.go | 8 +------- sei-tendermint/internal/rpc/core/lag_status.go | 6 +----- sei-tendermint/internal/rpc/core/status.go | 8 +++----- sei-tendermint/node/node.go | 13 ++++++++++++- 5 files changed, 17 insertions(+), 26 deletions(-) diff --git a/sei-tendermint/internal/blocksync/reactor.go b/sei-tendermint/internal/blocksync/reactor.go index f73e393867..3ae559ab66 100644 --- a/sei-tendermint/internal/blocksync/reactor.go +++ b/sei-tendermint/internal/blocksync/reactor.go @@ -39,14 +39,6 @@ const ( syncTimeout = 180 * time.Second ) -// Metricer is the RPC-facing blocksync surface. The facade and any future -// replacement can expose sync progress without leaking the concrete type. -type Metricer interface { - GetMaxPeerBlockHeight() int64 - GetTotalSyncedTime() time.Duration - GetRemainingSyncTime() time.Duration -} - // TODO(gprusak): that's not sufficient - parsing proto requires checking nils everywhere. func wrap[T *pb.BlockRequest | *pb.NoBlockResponse | *pb.BlockResponse | *pb.StatusRequest | *pb.StatusResponse](msg T) *pb.Message { switch msg := any(msg).(type) { diff --git a/sei-tendermint/internal/rpc/core/env.go b/sei-tendermint/internal/rpc/core/env.go index a4a8f6c4a2..0ff7911dad 100644 --- a/sei-tendermint/internal/rpc/core/env.go +++ b/sei-tendermint/internal/rpc/core/env.go @@ -73,7 +73,7 @@ type Environment struct { EvidencePool utils.Option[sm.EvidencePool] ConsensusState utils.Option[ConsensusState] ConsensusReactor utils.Option[*consensus.Reactor] - BlockSyncReactor utils.Option[blocksync.Metricer] + BlockSyncReactor *blocksync.Reactor IsListening bool Listeners []string @@ -257,12 +257,6 @@ func (env *Environment) requireConsensusReactor() (*consensus.Reactor, error) { return nil, fmt.Errorf("consensus reactor is not available") } -func (env *Environment) requireBlockSyncReactor() (blocksync.Metricer, error) { - if reactor, ok := env.BlockSyncReactor.Get(); ok { - return reactor, nil - } - return nil, fmt.Errorf("block sync reactor is not available") -} // StartService constructs and starts listeners for the RPC service // according to the config object, returning an error if the service diff --git a/sei-tendermint/internal/rpc/core/lag_status.go b/sei-tendermint/internal/rpc/core/lag_status.go index e38c67168c..7d57ba1b18 100644 --- a/sei-tendermint/internal/rpc/core/lag_status.go +++ b/sei-tendermint/internal/rpc/core/lag_status.go @@ -9,11 +9,7 @@ import ( // LagStatus returns Tendermint lag status, if lag is over a certain threshold func (env *Environment) LagStatus(ctx context.Context) (*coretypes.ResultLagStatus, error) { currentHeight := env.BlockStore.Height() - blockSyncReactor, err := env.requireBlockSyncReactor() - if err != nil { - return nil, err - } - maxPeerBlockHeight := blockSyncReactor.GetMaxPeerBlockHeight() + maxPeerBlockHeight := env.BlockSyncReactor.GetMaxPeerBlockHeight() lag := int64(0) // Calculate lag diff --git a/sei-tendermint/internal/rpc/core/status.go b/sei-tendermint/internal/rpc/core/status.go index d7e91c6075..e8629e99b7 100644 --- a/sei-tendermint/internal/rpc/core/status.go +++ b/sei-tendermint/internal/rpc/core/status.go @@ -135,11 +135,9 @@ func (env *Environment) Status(ctx context.Context) (*coretypes.ResultStatus, er result.SyncInfo.CatchingUp = reactor.WaitSync() } - if reactor, ok := env.BlockSyncReactor.Get(); ok { - result.SyncInfo.MaxPeerBlockHeight = reactor.GetMaxPeerBlockHeight() - result.SyncInfo.TotalSyncedTime = reactor.GetTotalSyncedTime() - result.SyncInfo.RemainingTime = reactor.GetRemainingSyncTime() - } + result.SyncInfo.MaxPeerBlockHeight = env.BlockSyncReactor.GetMaxPeerBlockHeight() + result.SyncInfo.TotalSyncedTime = env.BlockSyncReactor.GetTotalSyncedTime() + result.SyncInfo.RemainingTime = env.BlockSyncReactor.GetRemainingSyncTime() if metricer, ok := env.StateSyncMetricer.Get(); ok { result.SyncInfo.TotalSnapshots = metricer.TotalSnapshots() diff --git a/sei-tendermint/node/node.go b/sei-tendermint/node/node.go index 4d66999ac7..f3fc6da8d8 100644 --- a/sei-tendermint/node/node.go +++ b/sei-tendermint/node/node.go @@ -326,7 +326,7 @@ func makeNode( return nil, fmt.Errorf("blocksync.NewReactor(): %w", err) } node.services = append(node.services, bcReactor) - node.rpcEnv.BlockSyncReactor = utils.Some[blocksync.Metricer](bcReactor) + node.rpcEnv.BlockSyncReactor = bcReactor // Make ConsensusReactor. Don't enable fully if doing a state sync and/or block sync first. // FIXME We need to update metrics here, since other reactors don't have access to them. @@ -391,6 +391,17 @@ func makeNode( csState.SetPrivValidator(ctx, utils.Some(privValidator)) } } + } else { + node.rpcEnv.BlockSyncReactor, err = blocksync.NewReactor( + stateStore, + blockStore, + node.router, + utils.None[blocksync.SyncerConfig](), + ) + if err != nil { + return nil, fmt.Errorf("blocksync.NewReactor(): %w", err) + } + node.services = append(node.services, node.rpcEnv.BlockSyncReactor) } node.rpcEnv.PubKey = pubKey From d76cd8f70148197e6ad737f285d0d74cba7a94c4 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 28 May 2026 17:02:08 +0200 Subject: [PATCH 084/100] autobahn mempool Insert and Nonce plugged in --- sei-tendermint/internal/p2p/giga_router.go | 6 +++ sei-tendermint/internal/rpc/core/dev.go | 6 +++ sei-tendermint/internal/rpc/core/mempool.go | 46 ++++++++++++++++++++- sei-tendermint/rpc/client/local/local.go | 3 ++ 4 files changed, 60 insertions(+), 1 deletion(-) diff --git a/sei-tendermint/internal/p2p/giga_router.go b/sei-tendermint/internal/p2p/giga_router.go index 6479df5fd1..322e6b36f0 100644 --- a/sei-tendermint/internal/p2p/giga_router.go +++ b/sei-tendermint/internal/p2p/giga_router.go @@ -125,6 +125,12 @@ func (r *GigaRouter) InsertTx(ctx context.Context, tx types.Tx) (*abci.ResponseC return r.producer.InsertTx(ctx,tx) } +// Mempool exposes Autobahn's producer-backed mempool surface to callers that +// need features not shared with CometBFT's TxMempool. +func (r *GigaRouter) Mempool() *producer.State { + return r.producer +} + // LastCommittedBlockNumber returns the highest global block number finalized // by consensus (derived from the latest CommitQC). When no CommitQC has been // recorded yet, atypes.GlobalRangeOpt returns the committee's empty default diff --git a/sei-tendermint/internal/rpc/core/dev.go b/sei-tendermint/internal/rpc/core/dev.go index ecd0d58307..d2726456c4 100644 --- a/sei-tendermint/internal/rpc/core/dev.go +++ b/sei-tendermint/internal/rpc/core/dev.go @@ -2,12 +2,18 @@ package core import ( "context" + "errors" "github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes" ) // UnsafeFlushMempool removes all transactions from the mempool. func (env *Environment) UnsafeFlushMempool(ctx context.Context) (*coretypes.ResultUnsafeFlushMempool, error) { + if _, ok := env.gigaRouter().Get(); ok { + // TODO(autobahn): expose a producer-backed mempool flush/reset operation + // if we want parity with CometBFT's unsafe_flush_mempool RPC. + return nil, errors.New("unsafe_flush_mempool is not supported with autobahn mempool yet") + } mp, err := env.requireMempool() if err != nil { return nil, err diff --git a/sei-tendermint/internal/rpc/core/mempool.go b/sei-tendermint/internal/rpc/core/mempool.go index dfff3851a2..30c1fd1b9b 100644 --- a/sei-tendermint/internal/rpc/core/mempool.go +++ b/sei-tendermint/internal/rpc/core/mempool.go @@ -36,6 +36,10 @@ func (env *Environment) EvmProxy(sender common.Address) (*url.URL, bool) { // https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_async // Deprecated and should be removed in 0.37 func (env *Environment) BroadcastTxAsync(ctx context.Context, req *coretypes.RequestBroadcastTx) (*coretypes.ResultBroadcastTx, error) { + if giga, ok := env.gigaRouter().Get(); ok { + go func() { _, _ = giga.Mempool().InsertTx(ctx, req.Tx) }() + return &coretypes.ResultBroadcastTx{Hash: req.Tx.Hash().Bytes()}, nil + } mp, err := env.requireMempool() if err != nil { return nil, err @@ -54,6 +58,19 @@ func (env *Environment) BroadcastTxSync(ctx context.Context, req *coretypes.Requ // DeliverTx result. // More: https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_sync func (env *Environment) BroadcastTx(ctx context.Context, req *coretypes.RequestBroadcastTx) (*coretypes.ResultBroadcastTx, error) { + if giga, ok := env.gigaRouter().Get(); ok { + r, err := giga.Mempool().InsertTx(ctx, req.Tx) + if err != nil { + return nil, err + } + return &coretypes.ResultBroadcastTx{ + Code: r.Code, + Data: r.Data, + Codespace: r.Codespace, + Hash: req.Tx.Hash().Bytes(), + Log: r.Log, + }, nil + } mp, err := env.requireMempool() if err != nil { return nil, err @@ -74,6 +91,19 @@ func (env *Environment) BroadcastTx(ctx context.Context, req *coretypes.RequestB // BroadcastTxCommit returns with the responses from CheckTx and DeliverTx. // More: https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_commit func (env *Environment) BroadcastTxCommit(ctx context.Context, req *coretypes.RequestBroadcastTx) (*coretypes.ResultBroadcastTxCommit, error) { + if giga, ok := env.gigaRouter().Get(); ok { + r, err := giga.Mempool().InsertTx(ctx, req.Tx) + if err != nil { + return nil, err + } + if r.Code != abci.CodeTypeOK { + return &coretypes.ResultBroadcastTxCommit{ + CheckTx: *r, + Hash: req.Tx.Hash().Bytes(), + }, nil + } + return env.broadcastTxCommitFromCheckTx(ctx, req, r) + } mp, err := env.requireMempool() if err != nil { return nil, err @@ -82,6 +112,10 @@ func (env *Environment) BroadcastTxCommit(ctx context.Context, req *coretypes.Re if err != nil { return nil, err } + return env.broadcastTxCommitFromCheckTx(ctx, req, r) +} + +func (env *Environment) broadcastTxCommitFromCheckTx(ctx context.Context, req *coretypes.RequestBroadcastTx, r *abci.ResponseCheckTx) (*coretypes.ResultBroadcastTxCommit, error) { if r.Code != abci.CodeTypeOK { return &coretypes.ResultBroadcastTxCommit{ CheckTx: *r, @@ -108,7 +142,7 @@ func (env *Environment) BroadcastTxCommit(ctx context.Context, req *coretypes.Re case <-ctx.Done(): logger.Error("error on broadcastTxCommit", "duration", time.Since(startAt), - "err", err) + "err", ctx.Err()) return &coretypes.ResultBroadcastTxCommit{ CheckTx: *r, Hash: req.Tx.Hash().Bytes(), @@ -139,6 +173,11 @@ func (env *Environment) BroadcastTxCommit(ctx context.Context, req *coretypes.Re // UnconfirmedTxs gets unconfirmed transactions from the mempool in order of priority // More: https://docs.tendermint.com/master/rpc/#/Info/unconfirmed_txs func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.RequestUnconfirmedTxs) (*coretypes.ResultUnconfirmedTxs, error) { + if _, ok := env.gigaRouter().Get(); ok { + // TODO(autobahn): expose size/reap semantics from the producer-backed + // mempool so /unconfirmed_txs can report queued transactions. + return nil, errors.New("unconfirmed_txs is not supported with autobahn mempool yet") + } mp, err := env.requireMempool() if err != nil { return nil, err @@ -171,6 +210,11 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque // NumUnconfirmedTxs gets number of unconfirmed transactions. // More: https://docs.tendermint.com/master/rpc/#/Info/num_unconfirmed_txs func (env *Environment) NumUnconfirmedTxs(ctx context.Context) (*coretypes.ResultUnconfirmedTxs, error) { + if _, ok := env.gigaRouter().Get(); ok { + // TODO(autobahn): expose queued-transaction counts/bytes from the + // producer-backed mempool for /num_unconfirmed_txs. + return nil, errors.New("num_unconfirmed_txs is not supported with autobahn mempool yet") + } mp, err := env.requireMempool() if err != nil { return nil, err diff --git a/sei-tendermint/rpc/client/local/local.go b/sei-tendermint/rpc/client/local/local.go index 6923c10b17..73c9978c69 100644 --- a/sei-tendermint/rpc/client/local/local.go +++ b/sei-tendermint/rpc/client/local/local.go @@ -107,6 +107,9 @@ func (c *Local) CheckTx(ctx context.Context, tx types.Tx) (*coretypes.ResultChec } func (c *Local) EvmNextPendingNonce(addr common.Address) uint64 { + if giga, ok := c.Environment.Router.Giga().Get(); ok { + return giga.Mempool().EvmNextPendingNonce(addr) + } if mp, ok := c.Mempool.Get(); ok { return mp.EvmNextPendingNonce(addr) } From b3d6d1924da90d3e9279d94c06f957b9f278e177 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 10:07:06 +0200 Subject: [PATCH 085/100] removed metricers, reverted previousMaxPeerHeight --- sei-tendermint/internal/blocksync/pool.go | 18 ++- .../internal/blocksync/pool_test.go | 24 ++++ sei-tendermint/internal/blocksync/reactor.go | 4 - sei-tendermint/internal/rpc/core/env.go | 14 +-- sei-tendermint/internal/rpc/core/status.go | 16 +-- .../internal/statesync/mocks/Metricer.go | 112 ------------------ sei-tendermint/internal/statesync/reactor.go | 12 -- 7 files changed, 51 insertions(+), 149 deletions(-) delete mode 100644 sei-tendermint/internal/statesync/mocks/Metricer.go diff --git a/sei-tendermint/internal/blocksync/pool.go b/sei-tendermint/internal/blocksync/pool.go index 7c9304f3fd..93cb53afd8 100644 --- a/sei-tendermint/internal/blocksync/pool.go +++ b/sei-tendermint/internal/blocksync/pool.go @@ -90,9 +90,10 @@ type BlockPool struct { requesters map[int64]*bpRequester height int64 // the lowest key in requesters. // peers - peers map[types.NodeID]*bpPeer - router router - maxPeerHeight int64 // the biggest reported height + peers map[types.NodeID]*bpPeer + router router + maxPeerHeight int64 // the biggest reported height among current peers + monotoneMaxPeerHeight int64 // the biggest reported height observed since the pool started // atomic numPending atomic.Int32 // number of requests pending assignment or block response @@ -217,9 +218,11 @@ func (pool *BlockPool) IsCaughtUp() bool { return false } - // NOTE: we use maxPeerHeight - 1 because to sync block H requires block H+1 - // to verify the LastCommit. - return pool.height >= (pool.maxPeerHeight - 1) + // NOTE: we use monotoneMaxPeerHeight - 1 because to sync block H requires + // block H+1 to verify the LastCommit. The monotone maximum prevents us from + // considering ourselves caught up just because peers later retract their + // reported heights. + return pool.height >= (pool.monotoneMaxPeerHeight - 1) } // PeekTwoBlocks returns blocks at pool.height and pool.height+1. We need to @@ -396,6 +399,9 @@ func (pool *BlockPool) SetPeerRange(peerID types.NodeID, base int64, height int6 if height > pool.maxPeerHeight { pool.maxPeerHeight = height } + if height > pool.monotoneMaxPeerHeight { + pool.monotoneMaxPeerHeight = height + } } // RemovePeer removes the peer with peerID from the pool. If there's no peer diff --git a/sei-tendermint/internal/blocksync/pool_test.go b/sei-tendermint/internal/blocksync/pool_test.go index f2c9044d5b..864bed54ec 100644 --- a/sei-tendermint/internal/blocksync/pool_test.go +++ b/sei-tendermint/internal/blocksync/pool_test.go @@ -249,6 +249,30 @@ func TestBlockPoolMaliciousNodeMaxInt64(t *testing.T) { require.Equal(t, int64(initialHeight), pool.maxPeerHeight) } +func TestBlockPoolIsCaughtUpUsesMonotoneMaxPeerHeight(t *testing.T) { + const startHeight = 7 + goodNodeID := types.NodeID(strings.Repeat("a", 40)) + badNodeID := types.NodeID(strings.Repeat("b", 40)) + peers := testPeers{ + goodNodeID: {goodNodeID, 1, startHeight, make(chan inputData)}, + badNodeID: {badNodeID, 1, math.MaxInt64, make(chan inputData)}, + } + pool := NewBlockPool(1, makeRouter(peers)) + + pool.SetPeerRange(goodNodeID, 1, startHeight) + pool.SetPeerRange(badNodeID, 1, math.MaxInt64) + pool.SetPeerRange(badNodeID, 1, startHeight) + + pool.height = startHeight - 1 + require.False(t, pool.IsCaughtUp()) + + pool.height = startHeight + require.False(t, pool.IsCaughtUp()) + + pool.height = math.MaxInt64 - 1 + require.True(t, pool.IsCaughtUp()) +} + func TestBlockPoolRejectsWrongPeerWithoutDiscardingGoodBlock(t *testing.T) { t.Run("good then bad", func(t *testing.T) { testBlockPoolRejectsWrongPeerWithoutDiscardingGoodBlock(t, true) diff --git a/sei-tendermint/internal/blocksync/reactor.go b/sei-tendermint/internal/blocksync/reactor.go index f73e393867..fca376ffcd 100644 --- a/sei-tendermint/internal/blocksync/reactor.go +++ b/sei-tendermint/internal/blocksync/reactor.go @@ -631,7 +631,6 @@ func (s *syncController) autoRestartIfBehind(ctx context.Context, pool *BlockPoo } lastRestartTime := time.Now() - var previousMaxPeerHeight int64 logger.Info("checking if node is behind threshold, auto restarting if its behind", "threshold", s.blocksBehindThreshold, "interval", s.blocksBehindCheckInterval) for { @@ -642,9 +641,6 @@ func (s *syncController) autoRestartIfBehind(ctx context.Context, pool *BlockPoo threshold := int64(s.blocksBehindThreshold) //nolint:gosec // validated in config.ValidateBasic against MaxInt64 behindHeight := maxPeerHeight - selfHeight blockSyncIsSet := s.blockSync.Load() - if maxPeerHeight > previousMaxPeerHeight { - previousMaxPeerHeight = maxPeerHeight - } if maxPeerHeight == 0 || behindHeight < threshold || blockSyncIsSet { logger.Debug("does not exceed threshold or is already in block sync mode", "threshold", threshold, "behindHeight", behindHeight, "maxPeerHeight", maxPeerHeight, "selfHeight", selfHeight, "blockSyncIsSet", blockSyncIsSet) diff --git a/sei-tendermint/internal/rpc/core/env.go b/sei-tendermint/internal/rpc/core/env.go index aafba973e3..7fd6b6ff0c 100644 --- a/sei-tendermint/internal/rpc/core/env.go +++ b/sei-tendermint/internal/rpc/core/env.go @@ -82,13 +82,13 @@ type Environment struct { Router *p2p.Router // objects - PubKey utils.Option[crypto.PubKey] - GenDoc *types.GenesisDoc // cache the genesis structure - EventSinks []indexer.EventSink - EventBus *eventbus.EventBus // thread safe - EventLog *eventlog.Log - Mempool *mempool.TxMempool - StateSyncMetricer statesync.Metricer + PubKey utils.Option[crypto.PubKey] + GenDoc *types.GenesisDoc // cache the genesis structure + EventSinks []indexer.EventSink + EventBus *eventbus.EventBus // thread safe + EventLog *eventlog.Log + Mempool *mempool.TxMempool + StateSyncReactor *statesync.Reactor Config config.RPCConfig diff --git a/sei-tendermint/internal/rpc/core/status.go b/sei-tendermint/internal/rpc/core/status.go index 04b981104b..5fc69b0ea3 100644 --- a/sei-tendermint/internal/rpc/core/status.go +++ b/sei-tendermint/internal/rpc/core/status.go @@ -141,14 +141,14 @@ func (env *Environment) Status(ctx context.Context) (*coretypes.ResultStatus, er result.SyncInfo.RemainingTime = env.BlockSyncReactor.GetRemainingSyncTime() } - if env.StateSyncMetricer != nil { - result.SyncInfo.TotalSnapshots = env.StateSyncMetricer.TotalSnapshots() - result.SyncInfo.ChunkProcessAvgTime = env.StateSyncMetricer.ChunkProcessAvgTime() - result.SyncInfo.SnapshotHeight = env.StateSyncMetricer.SnapshotHeight() - result.SyncInfo.SnapshotChunksCount = env.StateSyncMetricer.SnapshotChunksCount() - result.SyncInfo.SnapshotChunksTotal = env.StateSyncMetricer.SnapshotChunksTotal() - result.SyncInfo.BackFilledBlocks = env.StateSyncMetricer.BackFilledBlocks() - result.SyncInfo.BackFillBlocksTotal = env.StateSyncMetricer.BackFillBlocksTotal() + if env.StateSyncReactor != nil { + result.SyncInfo.TotalSnapshots = env.StateSyncReactor.TotalSnapshots() + result.SyncInfo.ChunkProcessAvgTime = env.StateSyncReactor.ChunkProcessAvgTime() + result.SyncInfo.SnapshotHeight = env.StateSyncReactor.SnapshotHeight() + result.SyncInfo.SnapshotChunksCount = env.StateSyncReactor.SnapshotChunksCount() + result.SyncInfo.SnapshotChunksTotal = env.StateSyncReactor.SnapshotChunksTotal() + result.SyncInfo.BackFilledBlocks = env.StateSyncReactor.BackFilledBlocks() + result.SyncInfo.BackFillBlocksTotal = env.StateSyncReactor.BackFillBlocksTotal() } return result, nil diff --git a/sei-tendermint/internal/statesync/mocks/Metricer.go b/sei-tendermint/internal/statesync/mocks/Metricer.go deleted file mode 100644 index c4721b304e..0000000000 --- a/sei-tendermint/internal/statesync/mocks/Metricer.go +++ /dev/null @@ -1,112 +0,0 @@ -// Code generated by mockery 2.9.4. DO NOT EDIT. - -package mocks - -import ( - mock "github.com/stretchr/testify/mock" - - time "time" -) - -// Metricer is an autogenerated mock type for the Metricer type -type Metricer struct { - mock.Mock -} - -// BackFillBlocksTotal provides a mock function with given fields: -func (_m *Metricer) BackFillBlocksTotal() int64 { - ret := _m.Called() - - var r0 int64 - if rf, ok := ret.Get(0).(func() int64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int64) - } - - return r0 -} - -// BackFilledBlocks provides a mock function with given fields: -func (_m *Metricer) BackFilledBlocks() int64 { - ret := _m.Called() - - var r0 int64 - if rf, ok := ret.Get(0).(func() int64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int64) - } - - return r0 -} - -// ChunkProcessAvgTime provides a mock function with given fields: -func (_m *Metricer) ChunkProcessAvgTime() time.Duration { - ret := _m.Called() - - var r0 time.Duration - if rf, ok := ret.Get(0).(func() time.Duration); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(time.Duration) - } - - return r0 -} - -// SnapshotChunksCount provides a mock function with given fields: -func (_m *Metricer) SnapshotChunksCount() int64 { - ret := _m.Called() - - var r0 int64 - if rf, ok := ret.Get(0).(func() int64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int64) - } - - return r0 -} - -// SnapshotChunksTotal provides a mock function with given fields: -func (_m *Metricer) SnapshotChunksTotal() int64 { - ret := _m.Called() - - var r0 int64 - if rf, ok := ret.Get(0).(func() int64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int64) - } - - return r0 -} - -// SnapshotHeight provides a mock function with given fields: -func (_m *Metricer) SnapshotHeight() int64 { - ret := _m.Called() - - var r0 int64 - if rf, ok := ret.Get(0).(func() int64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int64) - } - - return r0 -} - -// TotalSnapshots provides a mock function with given fields: -func (_m *Metricer) TotalSnapshots() int64 { - ret := _m.Called() - - var r0 int64 - if rf, ok := ret.Get(0).(func() int64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int64) - } - - return r0 -} diff --git a/sei-tendermint/internal/statesync/reactor.go b/sei-tendermint/internal/statesync/reactor.go index 18ccc6d83f..a324f1f4d1 100644 --- a/sei-tendermint/internal/statesync/reactor.go +++ b/sei-tendermint/internal/statesync/reactor.go @@ -146,18 +146,6 @@ func GetParamsChannelDescriptor() p2p.ChannelDescriptor[*pb.Message] { } } -// Metricer defines an interface used for the rpc sync info query, please see statesync.metrics -// for the details. -type Metricer interface { - TotalSnapshots() int64 - ChunkProcessAvgTime() time.Duration - SnapshotHeight() int64 - SnapshotChunksCount() int64 - SnapshotChunksTotal() int64 - BackFilledBlocks() int64 - BackFillBlocksTotal() int64 -} - // Reactor handles state sync, both restoring snapshots for the local node and // serving snapshots for other nodes. type Reactor struct { From dbe40d6eaf810a9d821b2641f21f61c91df86799 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 10:13:37 +0200 Subject: [PATCH 086/100] test fix --- sei-tendermint/internal/blocksync/pool_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sei-tendermint/internal/blocksync/pool_test.go b/sei-tendermint/internal/blocksync/pool_test.go index 864bed54ec..0dd141894a 100644 --- a/sei-tendermint/internal/blocksync/pool_test.go +++ b/sei-tendermint/internal/blocksync/pool_test.go @@ -253,14 +253,17 @@ func TestBlockPoolIsCaughtUpUsesMonotoneMaxPeerHeight(t *testing.T) { const startHeight = 7 goodNodeID := types.NodeID(strings.Repeat("a", 40)) badNodeID := types.NodeID(strings.Repeat("b", 40)) + otherGoodNodeID := types.NodeID(strings.Repeat("c", 40)) peers := testPeers{ - goodNodeID: {goodNodeID, 1, startHeight, make(chan inputData)}, - badNodeID: {badNodeID, 1, math.MaxInt64, make(chan inputData)}, + goodNodeID: {goodNodeID, 1, startHeight, make(chan inputData)}, + badNodeID: {badNodeID, 1, math.MaxInt64, make(chan inputData)}, + otherGoodNodeID: {otherGoodNodeID, 1, startHeight, make(chan inputData)}, } pool := NewBlockPool(1, makeRouter(peers)) pool.SetPeerRange(goodNodeID, 1, startHeight) pool.SetPeerRange(badNodeID, 1, math.MaxInt64) + pool.SetPeerRange(otherGoodNodeID, 1, startHeight) pool.SetPeerRange(badNodeID, 1, startHeight) pool.height = startHeight - 1 From 58919586e4a494a6e2edc32992d605ece172cf0d Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 10:16:07 +0200 Subject: [PATCH 087/100] adjusted doc --- sei-tendermint/internal/blocksync/doc.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sei-tendermint/internal/blocksync/doc.go b/sei-tendermint/internal/blocksync/doc.go index 2c32741034..694d21930d 100644 --- a/sei-tendermint/internal/blocksync/doc.go +++ b/sei-tendermint/internal/blocksync/doc.go @@ -18,7 +18,6 @@ top-level Reactor owns the single blocksync p2p channel and the always-on query serving path: - serve inbound BlockRequest and StatusRequest messages -- advertise local status to newly connected peers Active syncing itself is handled by a separate sync controller that manages the block pool, requests blocks, applies them locally, and hands off to consensus From 4f6226501179bf2ca5000f86eace9908e58a2524 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 12:10:54 +0200 Subject: [PATCH 088/100] removed deadcode --- sei-tendermint/internal/blocksync/pool.go | 11 ----------- sei-tendermint/internal/blocksync/reactor.go | 8 -------- sei-tendermint/internal/rpc/core/env.go | 2 +- 3 files changed, 1 insertion(+), 20 deletions(-) diff --git a/sei-tendermint/internal/blocksync/pool.go b/sei-tendermint/internal/blocksync/pool.go index 93cb53afd8..51cf4a2449 100644 --- a/sei-tendermint/internal/blocksync/pool.go +++ b/sei-tendermint/internal/blocksync/pool.go @@ -49,10 +49,6 @@ const ( // Maximum difference between current and new block's height. maxDiffBetweenCurrentAndReceivedBlockHeight = 100 - // Used to indicate the reason of the redo - PeerRemoved RetryReason = "PeerRemoved" - BadBlock RetryReason = "BadBlock" - peerTimeout = 2 * time.Second ) @@ -595,13 +591,6 @@ type bpRequester struct { inner utils.Watch[*bpRequesterInner] } -type RetryReason string - -type RedoOp struct { - PeerId types.NodeID - Reason RetryReason -} - func newBPRequester(pool *BlockPool, height int64) *bpRequester { return &bpRequester{ pool: pool, diff --git a/sei-tendermint/internal/blocksync/reactor.go b/sei-tendermint/internal/blocksync/reactor.go index fca376ffcd..c7de9cfb20 100644 --- a/sei-tendermint/internal/blocksync/reactor.go +++ b/sei-tendermint/internal/blocksync/reactor.go @@ -39,14 +39,6 @@ const ( syncTimeout = 180 * time.Second ) -// Metricer is the RPC-facing blocksync surface. The facade and any future -// replacement can expose sync progress without leaking the concrete type. -type Metricer interface { - GetMaxPeerBlockHeight() int64 - GetTotalSyncedTime() time.Duration - GetRemainingSyncTime() time.Duration -} - // TODO(gprusak): that's not sufficient - parsing proto requires checking nils everywhere. func wrap[T *pb.BlockRequest | *pb.NoBlockResponse | *pb.BlockResponse | *pb.StatusRequest | *pb.StatusResponse](msg T) *pb.Message { switch msg := any(msg).(type) { diff --git a/sei-tendermint/internal/rpc/core/env.go b/sei-tendermint/internal/rpc/core/env.go index 7fd6b6ff0c..972ba9110d 100644 --- a/sei-tendermint/internal/rpc/core/env.go +++ b/sei-tendermint/internal/rpc/core/env.go @@ -73,7 +73,7 @@ type Environment struct { EvidencePool sm.EvidencePool ConsensusState consensusState ConsensusReactor *consensus.Reactor - BlockSyncReactor blocksync.Metricer + BlockSyncReactor *blocksync.Reactor IsListening bool Listeners []string From 3b4d15e9d7eede7d61a37c91744f4bd8967971da Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 12:46:20 +0200 Subject: [PATCH 089/100] low effort unconfirmed txs --- .../internal/autobahn/producer/mempool.go | 9 ++++ sei-tendermint/internal/rpc/core/mempool.go | 46 +++++++++++++++---- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/sei-tendermint/internal/autobahn/producer/mempool.go b/sei-tendermint/internal/autobahn/producer/mempool.go index 010669dc6e..7fe06a730b 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool.go +++ b/sei-tendermint/internal/autobahn/producer/mempool.go @@ -45,6 +45,15 @@ func (m *mempool) PushBlock() { } } +// TODO(gprusak): this rpc is probably unused, but if it is +// consider whether unsequenced/unexecuted lane txs should be included here. +func (s *State) UnconfirmedTxs() [][]byte { + for m := range s.mempool.Lock() { + return m.nextBlock.txs + } + panic("uneachable") +} + // (addr,nonce) -> tx // tracking of what is in progress diff --git a/sei-tendermint/internal/rpc/core/mempool.go b/sei-tendermint/internal/rpc/core/mempool.go index 30c1fd1b9b..156ddfe0a4 100644 --- a/sei-tendermint/internal/rpc/core/mempool.go +++ b/sei-tendermint/internal/rpc/core/mempool.go @@ -8,6 +8,7 @@ import ( "net/url" "time" + "github.com/sei-protocol/sei-chain/sei-tendermint/types" "github.com/ethereum/go-ethereum/common" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" @@ -173,10 +174,31 @@ func (env *Environment) broadcastTxCommitFromCheckTx(ctx context.Context, req *c // UnconfirmedTxs gets unconfirmed transactions from the mempool in order of priority // More: https://docs.tendermint.com/master/rpc/#/Info/unconfirmed_txs func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.RequestUnconfirmedTxs) (*coretypes.ResultUnconfirmedTxs, error) { - if _, ok := env.gigaRouter().Get(); ok { - // TODO(autobahn): expose size/reap semantics from the producer-backed - // mempool so /unconfirmed_txs can report queued transactions. - return nil, errors.New("unconfirmed_txs is not supported with autobahn mempool yet") + if giga, ok := env.gigaRouter().Get(); ok { + // NOTE: this pagination seems to be useless, given that the mempool content is + // constantly changing and we don't have any snapshot marker in the request. + rawTxs := giga.Mempool().UnconfirmedTxs() + perPage := env.validatePerPage(req.PerPage.IntPtr()) + page, err := validatePage(req.Page.IntPtr(), perPage, len(rawTxs)) + if err != nil { + return nil, err + } + skipCount := validateSkipCount(page, perPage) + + sizeBytes := 0 + for _, tx := range rawTxs { + sizeBytes += len(tx) + } + var txs types.Txs + for _, tx := range rawTxs[skipCount:min(skipCount+perPage,len(rawTxs))] { + txs = append(txs,tx) + } + return &coretypes.ResultUnconfirmedTxs{ + Count: len(txs), + Total: len(rawTxs), + TotalBytes: int64(sizeBytes), + Txs: txs, + }, nil } mp, err := env.requireMempool() if err != nil { @@ -188,7 +210,6 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque if err != nil { return nil, err } - skipCount := validateSkipCount(page, perPage) txs, _ := mp.ReapTxs(mempool.ReapLimits{ @@ -210,10 +231,17 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque // NumUnconfirmedTxs gets number of unconfirmed transactions. // More: https://docs.tendermint.com/master/rpc/#/Info/num_unconfirmed_txs func (env *Environment) NumUnconfirmedTxs(ctx context.Context) (*coretypes.ResultUnconfirmedTxs, error) { - if _, ok := env.gigaRouter().Get(); ok { - // TODO(autobahn): expose queued-transaction counts/bytes from the - // producer-backed mempool for /num_unconfirmed_txs. - return nil, errors.New("num_unconfirmed_txs is not supported with autobahn mempool yet") + if giga, ok := env.gigaRouter().Get(); ok { + rawTxs := giga.Mempool().UnconfirmedTxs() + sizeBytes := 0 + for _, tx := range rawTxs { + sizeBytes += len(tx) + } + return &coretypes.ResultUnconfirmedTxs{ + Count: len(rawTxs), + Total: len(rawTxs), + TotalBytes: int64(sizeBytes), + },nil } mp, err := env.requireMempool() if err != nil { From 97d052ef0dffaa48c6faaa1ff3e2480f05b249c7 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 19:31:58 +0200 Subject: [PATCH 090/100] flush for autobahn is not supported --- sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go | 2 -- sei-tendermint/internal/rpc/core/dev.go | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go b/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go index c269e91ca0..a8e3ad36da 100644 --- a/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go +++ b/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go @@ -84,9 +84,7 @@ Output is written to the file specified by --output.`, cfg := config.AutobahnFileConfig{ Validators: validators, MaxTxsPerBlock: 5_000, - MempoolSize: 5_000, BlockInterval: utils.Duration(400 * time.Millisecond), - AllowEmptyBlocks: true, ViewTimeout: utils.Duration(1500 * time.Millisecond), DialInterval: utils.Duration(10 * time.Second), } diff --git a/sei-tendermint/internal/rpc/core/dev.go b/sei-tendermint/internal/rpc/core/dev.go index d2726456c4..374c2142f2 100644 --- a/sei-tendermint/internal/rpc/core/dev.go +++ b/sei-tendermint/internal/rpc/core/dev.go @@ -10,9 +10,7 @@ import ( // UnsafeFlushMempool removes all transactions from the mempool. func (env *Environment) UnsafeFlushMempool(ctx context.Context) (*coretypes.ResultUnsafeFlushMempool, error) { if _, ok := env.gigaRouter().Get(); ok { - // TODO(autobahn): expose a producer-backed mempool flush/reset operation - // if we want parity with CometBFT's unsafe_flush_mempool RPC. - return nil, errors.New("unsafe_flush_mempool is not supported with autobahn mempool yet") + return nil, errors.New("unsafe_flush_mempool is not supported with autobahn mempool") } mp, err := env.requireMempool() if err != nil { From 3200bea53eb5c6544b49aab7a430a31d7afc8fd5 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 19:50:38 +0200 Subject: [PATCH 091/100] some fixes --- .../internal/autobahn/producer/mempool.go | 23 ++++--------------- .../internal/autobahn/producer/state.go | 2 ++ 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/sei-tendermint/internal/autobahn/producer/mempool.go b/sei-tendermint/internal/autobahn/producer/mempool.go index 7fe06a730b..dfec46f937 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool.go +++ b/sei-tendermint/internal/autobahn/producer/mempool.go @@ -11,6 +11,9 @@ import ( abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" ) +var errTooLarge = errors.New("transaction too large") +var errBadNonce = errors.New("bad nonce") + type blockSpec struct { gasEstimated uint64 gasWanted uint64 @@ -40,6 +43,7 @@ func (m *mempool) CanPushBlock() bool { func (m *mempool) PushBlock() { m.blocks[m.next] = m.nextBlock + m.next += 1 m.nextBlock = &blockSpec{ evmNonces: map[common.Address]uint64{}, } @@ -54,21 +58,6 @@ func (s *State) UnconfirmedTxs() [][]byte { panic("uneachable") } - -// (addr,nonce) -> tx -// tracking of what is in progress -// on startup -// * read data.State and avail.State from executed until the end (even across gaps) -// * parse all of these transactions -// * consider only our lane blocks (we are guaranteed to have all of our lane blocks) -// * we are interested only in evm nonces - ignore txs with nonces after a gap -// every time execution progresses -// * we check if nonces progressed as expected. -// * if not - just drop all the non-included txs of the given address -// for testnet -// * accept only ready txs -// * don't drop ready txs (unless some tx was unexpectedly dropped) -// * drop over capacity. func (s *State) EvmNextPendingNonce(addr common.Address) uint64 { for m := range s.mempool.Lock() { if nonce,ok := m.evmNonces[addr]; ok { @@ -78,9 +67,6 @@ func (s *State) EvmNextPendingNonce(addr common.Address) uint64 { return s.cfg.App.EvmNonce(addr) } -var errTooLarge = errors.New("transaction too large") -var errBadNonce = errors.New("bad nonce") - func (s *State) mempoolFirst() types.BlockNumber { for m := range s.mempool.Lock() { return m.first @@ -88,6 +74,7 @@ func (s *State) mempoolFirst() types.BlockNumber { panic("unreachable") } +// Removes txs from mempool assigned to lane blocks Date: Fri, 29 May 2026 20:24:49 +0200 Subject: [PATCH 092/100] some codex tests --- .../internal/autobahn/autobahn.proto | 2 +- .../autobahn/producer/mempool_test.go | 384 ++++++++++++++++++ .../internal/autobahn/producer/state.go | 2 +- 3 files changed, 386 insertions(+), 2 deletions(-) create mode 100644 sei-tendermint/internal/autobahn/producer/mempool_test.go diff --git a/sei-tendermint/internal/autobahn/autobahn.proto b/sei-tendermint/internal/autobahn/autobahn.proto index 6db13dfe78..9d4f052feb 100644 --- a/sei-tendermint/internal/autobahn/autobahn.proto +++ b/sei-tendermint/internal/autobahn/autobahn.proto @@ -75,7 +75,7 @@ message BlockHeader { message Payload { reserved "edge_count", "coinbase", "basefee"; - reserved 3,4,5; + reserved 3, 4, 5; option (hashable.hashable) = true; optional Timestamp created_at = 1; // required optional uint64 total_gas = 2; // required diff --git a/sei-tendermint/internal/autobahn/producer/mempool_test.go b/sei-tendermint/internal/autobahn/producer/mempool_test.go new file mode 100644 index 0000000000..53cb2eb431 --- /dev/null +++ b/sei-tendermint/internal/autobahn/producer/mempool_test.go @@ -0,0 +1,384 @@ +package producer + +import ( + "context" + "encoding/binary" + "errors" + "testing" + "time" + + abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" + "github.com/stretchr/testify/require" +) + +type sealTrigger uint8 + +const ( + sealByCount sealTrigger = iota + sealByBytes + sealByGas +) + +type txSpec struct { + tx []byte + gasWanted int64 + gasEstimated int64 +} + +type overflowSpec struct { + count bool + bytes bool + gas bool +} + +type sealScenario struct { + countLimit uint64 + gasLimit uint64 + sealed [][]txSpec + overflow []overflowSpec + open []txSpec + allTxs [][]byte + specsByTx map[string]txSpec +} + +type gasWantedApp struct { + abci.BaseApplication + gasWanted int64 +} + +func (a gasWantedApp) CheckTx(_ context.Context, _ *abci.RequestCheckTxV2) *abci.ResponseCheckTxV2 { + return &abci.ResponseCheckTxV2{ + ResponseCheckTx: &abci.ResponseCheckTx{ + Code: abci.CodeTypeOK, + GasWanted: a.gasWanted, + }, + } +} + +type rejectingApp struct { + abci.BaseApplication + resp *abci.ResponseCheckTx +} + +func (a rejectingApp) CheckTx(_ context.Context, _ *abci.RequestCheckTxV2) *abci.ResponseCheckTxV2 { + return &abci.ResponseCheckTxV2{ResponseCheckTx: a.resp} +} + +type acceptingApp struct { + abci.BaseApplication + resp *abci.ResponseCheckTx +} + +func (a acceptingApp) CheckTx(_ context.Context, _ *abci.RequestCheckTxV2) *abci.ResponseCheckTxV2 { + return &abci.ResponseCheckTxV2{ResponseCheckTx: a.resp} +} + +type txSpecApp struct { + abci.BaseApplication + specs map[string]txSpec +} + +func (a txSpecApp) CheckTx(_ context.Context, req *abci.RequestCheckTxV2) *abci.ResponseCheckTxV2 { + spec := a.specs[string(req.Tx)] + return &abci.ResponseCheckTxV2{ + ResponseCheckTx: &abci.ResponseCheckTx{ + Code: abci.CodeTypeOK, + GasWanted: spec.gasWanted, + GasEstimated: spec.gasEstimated, + }, + } +} + +func newSealScenario(t *testing.T, rng utils.Rng) sealScenario { + const ( + wantSealedBlocks = 24 + minTriggerCoverage = 3 + maxAttempts = 64 + gasScale = 5_000 + unitMax = 16 + ) + for range maxAttempts { + countLimit := uint64(8 + rng.Intn(5)) + avgUnitsPerTx := (unitMax + 1) / 2 + byteBudgetUnits := max(unitMax+1, int(countLimit)*avgUnitsPerTx+rng.Intn(2*int(countLimit)+1)-int(countLimit)) + gasBudgetUnits := max(unitMax+1, int(countLimit)*avgUnitsPerTx+rng.Intn(2*int(countLimit)+1)-int(countLimit)) + byteScale := max(1, int(types.MaxTxsBytesPerBlock)/byteBudgetUnits) + gasLimit := uint64(gasScale * gasBudgetUnits) + var seq uint64 + specs := make([]txSpec, 0, wantSealedBlocks*int(countLimit)) + for len(specs) < wantSealedBlocks*int(countLimit)*4 { + specs = append(specs, randomTxSpec(rng, &seq, byteScale, gasScale, unitMax)) + } + scenario := simulateSealScenario(countLimit, gasLimit, specs) + if len(scenario.sealed) < wantSealedBlocks { + continue + } + scenario = truncateSealScenario(scenario, wantSealedBlocks) + countHits, byteHits, gasHits := 0, 0, 0 + for _, overflow := range scenario.overflow { + if overflow.count { + countHits += 1 + } + if overflow.bytes { + byteHits += 1 + } + if overflow.gas { + gasHits += 1 + } + } + if countHits >= minTriggerCoverage && byteHits >= minTriggerCoverage && gasHits >= minTriggerCoverage { + return scenario + } + } + t.Fatal("failed to generate seal scenario with sufficient trigger coverage") + return sealScenario{} +} + +func truncateSealScenario(scenario sealScenario, wantSealedBlocks int) sealScenario { + sealed := append([][]txSpec(nil), scenario.sealed[:wantSealedBlocks]...) + overflow := append([]overflowSpec(nil), scenario.overflow[:wantSealedBlocks]...) + open := scenario.open + if wantSealedBlocks < len(scenario.sealed) { + open = scenario.sealed[wantSealedBlocks] + } + allTxs := make([][]byte, 0) + specsByTx := map[string]txSpec{} + for _, block := range sealed { + for _, spec := range block { + allTxs = append(allTxs, spec.tx) + specsByTx[string(spec.tx)] = spec + } + } + for _, spec := range open { + allTxs = append(allTxs, spec.tx) + specsByTx[string(spec.tx)] = spec + } + return sealScenario{ + countLimit: scenario.countLimit, + gasLimit: scenario.gasLimit, + sealed: sealed, + overflow: overflow, + open: append([]txSpec(nil), open...), + allTxs: allTxs, + specsByTx: specsByTx, + } +} + +func randomTxSpec(rng utils.Rng, seq *uint64, byteScale, gasScale, unitMax int) txSpec { + sizeUnits := 1 + rng.Intn(unitMax) + gasUnits := 1 + rng.Intn(unitMax) + size := sizeUnits * byteScale + gasWanted := int64(gasUnits * gasScale) + tx := make([]byte, size) + binary.BigEndian.PutUint64(tx[:8], *seq) + copy(tx[8:], utils.GenBytes(rng, size-8)) + *seq += 1 + gasEstimated := gasWanted + if gasWanted > 1 { + gasEstimated = gasWanted - int64(rng.Intn(int(gasWanted-1))) + } + return txSpec{tx: tx, gasWanted: gasWanted, gasEstimated: gasEstimated} +} + +func simulateSealScenario(countLimit, gasLimit uint64, specs []txSpec) sealScenario { + current := make([]txSpec, 0, countLimit) + sealed := make([][]txSpec, 0) + overflow := make([]overflowSpec, 0) + allTxs := make([][]byte, 0, len(specs)) + specsByTx := make(map[string]txSpec, len(specs)) + + for _, spec := range specs { + allTxs = append(allTxs, spec.tx) + specsByTx[string(spec.tx)] = spec + if len(current) > 0 { + o := blockOverflow(current, spec, countLimit, gasLimit) + if o.count || o.bytes || o.gas { + sealed = append(sealed, append([]txSpec(nil), current...)) + overflow = append(overflow, o) + current = current[:0] + } + } + current = append(current, spec) + } + return sealScenario{ + countLimit: countLimit, + gasLimit: gasLimit, + sealed: sealed, + overflow: overflow, + open: append([]txSpec(nil), current...), + allTxs: allTxs, + specsByTx: specsByTx, + } +} + +func blockOverflow(block []txSpec, next txSpec, countLimit, gasLimit uint64) overflowSpec { + size, gas := blockTotals(block) + return overflowSpec{ + count: uint64(len(block))+1 > countLimit, + bytes: size+uint64(len(next.tx)) > uint64(types.MaxTxsBytesPerBlock), + gas: gas+uint64(next.gasWanted) > gasLimit, + } +} + +func blockTotals(block []txSpec) (size uint64, gas uint64) { + for _, spec := range block { + size += uint64(len(spec.tx)) + gas += uint64(spec.gasWanted) + } + return size, gas +} + +func txsOf(block []txSpec) [][]byte { + txs := make([][]byte, len(block)) + for i, spec := range block { + txs[i] = spec.tx + } + return txs +} + +func assertSealScenario( + t *testing.T, + state *State, + firstBlock types.BlockNumber, + scenario sealScenario, +) { + t.Helper() + + lane := state.consensus.Avail().PublicKey() + for i := range scenario.sealed { + block, err := state.consensus.Avail().Block(t.Context(), lane, firstBlock+types.BlockNumber(i)) + require.NoError(t, err) + require.Equal(t, txsOf(scenario.sealed[i]), block.Msg().Block().Payload().Txs()) + } + require.Equal(t, txsOf(scenario.open), state.UnconfirmedTxs()) + require.Len(t, scenario.overflow, len(scenario.sealed)) + for i, sealed := range scenario.sealed { + size, gas := blockTotals(sealed) + require.LessOrEqual(t, uint64(len(sealed)), scenario.countLimit) + require.LessOrEqual(t, size, uint64(types.MaxTxsBytesPerBlock)) + require.LessOrEqual(t, gas, scenario.gasLimit) + var nextFirst txSpec + if i+1 < len(scenario.sealed) { + nextFirst = scenario.sealed[i+1][0] + } else { + nextFirst = scenario.open[0] + } + require.Equal(t, blockOverflow(sealed, nextFirst, scenario.countLimit, scenario.gasLimit), scenario.overflow[i]) + require.True(t, scenario.overflow[i].count || scenario.overflow[i].bytes || scenario.overflow[i].gas) + } + + for m := range state.mempool.Lock() { + require.Equal(t, firstBlock, m.first) + require.Equal(t, firstBlock+types.BlockNumber(len(scenario.sealed)), m.next) + require.Len(t, m.blocks, len(scenario.sealed)) + for i := range scenario.sealed { + require.Equal(t, txsOf(scenario.sealed[i]), m.blocks[firstBlock+types.BlockNumber(i)].txs) + } + require.Equal(t, txsOf(scenario.open), m.nextBlock.txs) + return + } + t.Fatal("unreachable") +} + +func newTestState(t *testing.T, app abci.Application) *State { + t.Helper() + + rng := utils.TestRng() + committee, keys := types.GenCommittee(rng, 3) + dataState := utils.OrPanic1(data.NewState( + &data.Config{Committee: committee}, + utils.OrPanic1(data.NewDataWAL(utils.None[string](), committee)), + )) + consensusState := utils.OrPanic1(consensus.NewState(&consensus.Config{ + Key: keys[0], + ViewTimeout: func(types.View) time.Duration { return time.Hour }, + PersistentStateDir: utils.None[string](), + }, dataState)) + + return NewState(&Config{ + MaxGasPerBlock: types.MaxTxsBytesPerBlock, + MaxTxsPerBlock: types.MaxTxsPerBlock, + BlockInterval: time.Hour, + MaxTxsPerSecond: utils.None[uint64](), + App: proxy.New(app, proxy.NopMetrics()), + }, consensusState) +} + +func TestInsertTxRejectsTooLargeTransaction(t *testing.T) { + state := newTestState(t, abci.BaseApplication{}) + tx := make([]byte, types.MaxTxsBytesPerBlock+1) + + resp, err := state.InsertTx(t.Context(), tx) + + require.Nil(t, resp) + require.ErrorIs(t, err, errTooLarge) + require.True(t, errors.Is(err, errTooLarge)) +} + +func TestInsertTxRejectsGasWantedAboveBlockLimit(t *testing.T) { + state := newTestState(t, gasWantedApp{gasWanted: 101}) + state.cfg.MaxGasPerBlock = 100 + + resp, err := state.InsertTx(t.Context(), []byte("tx")) + + require.Nil(t, resp) + require.ErrorIs(t, err, errTooLarge) +} + +func TestInsertTxReturnsRejectedCheckTxWithoutEnqueueing(t *testing.T) { + wantResp := &abci.ResponseCheckTx{ + Code: 1, + Log: "rejected", + } + state := newTestState(t, rejectingApp{resp: wantResp}) + + gotResp, err := state.InsertTx(t.Context(), []byte("tx")) + + require.NoError(t, err) + require.Same(t, wantResp, gotResp) + require.Empty(t, state.UnconfirmedTxs()) +} + +func TestInsertTxAppendsAcceptedTransactionToOpenBlock(t *testing.T) { + wantResp := &abci.ResponseCheckTx{ + Code: abci.CodeTypeOK, + GasWanted: 50, + GasEstimated: 40, + } + state := newTestState(t, acceptingApp{resp: wantResp}) + tx := []byte("tx1") + + gotResp, err := state.InsertTx(t.Context(), tx) + + require.NoError(t, err) + require.Same(t, wantResp, gotResp) + require.Equal(t, [][]byte{tx}, state.UnconfirmedTxs()) +} + +func TestInsertTxSealsCurrentBlockWhenTxCountWouldOverflow(t *testing.T) { + rng := utils.TestRng() + scenario := newSealScenario(t, rng) + state := newTestState(t, txSpecApp{specs: scenario.specsByTx}) + state.cfg.MaxTxsPerBlock = scenario.countLimit + state.cfg.MaxGasPerBlock = scenario.gasLimit + lane := state.consensus.Avail().PublicKey() + firstBlock := state.consensus.Avail().NextBlock(lane) + err := scope.Run(t.Context(), func(ctx context.Context, s scope.Scope) error { + s.SpawnBg(func() error { return utils.IgnoreCancel(state.Run(ctx)) }) + + for _, tx := range scenario.allTxs { + resp, err := state.InsertTx(ctx, tx) + require.NoError(t, err) + require.Equal(t, scenario.specsByTx[string(tx)].gasWanted, resp.GasWanted) + } + assertSealScenario(t, state, firstBlock, scenario) + return nil + }) + require.NoError(t, err) +} diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index 92f2ef9dd6..c6e5d314d1 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -92,7 +92,7 @@ func (s *State) Run(ctx context.Context) error { } limiter := rate.NewLimiter(limit, burst) lastBlockTime := time.Now() - for toProduce:=firstBlock;; firstBlock += 1 { + for toProduce:=firstBlock;; toProduce += 1 { if err := availState.WaitForCapacity(ctx,toProduce); err != nil { return fmt.Errorf("availState.WaitForCapacity(): %w", err) } From 6da742676b5e9d22452463fa56d4b7b416baaf19 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 20:26:43 +0200 Subject: [PATCH 093/100] fmt --- sei-tendermint/abci/types/types.go | 2 +- .../commands/gen_autobahn_config.go | 10 +-- .../internal/autobahn/avail/state.go | 2 +- .../internal/autobahn/data/state.go | 29 +++++--- .../internal/autobahn/producer/mempool.go | 72 ++++++++++--------- .../internal/autobahn/producer/state.go | 67 ++++++++--------- sei-tendermint/internal/p2p/giga_router.go | 2 +- .../internal/p2p/giga_router_test.go | 24 +++---- sei-tendermint/internal/p2p/router_test.go | 10 +-- sei-tendermint/internal/rpc/core/blocks.go | 2 +- sei-tendermint/internal/rpc/core/env.go | 13 ++-- sei-tendermint/internal/rpc/core/mempool.go | 8 +-- sei-tendermint/node/node.go | 16 ++--- sei-tendermint/node/setup.go | 15 ++-- 14 files changed, 144 insertions(+), 128 deletions(-) diff --git a/sei-tendermint/abci/types/types.go b/sei-tendermint/abci/types/types.go index 3d5125dc77..cb4edbb6ee 100644 --- a/sei-tendermint/abci/types/types.go +++ b/sei-tendermint/abci/types/types.go @@ -232,7 +232,7 @@ type ResponseCheckTxV2 struct { *ResponseCheckTx // helper properties for prioritization in mempool - IsEVM bool + IsEVM bool EVMNonce uint64 // EVM and sei addresses are both derived from the sender's public key. // TODO(gprusak): include just the secp256k1 public key and let the CheckTx caller derive evm/sei address on their own. diff --git a/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go b/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go index a8e3ad36da..b7e7c88215 100644 --- a/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go +++ b/sei-tendermint/cmd/tendermint/commands/gen_autobahn_config.go @@ -82,11 +82,11 @@ Output is written to the file specified by --output.`, } cfg := config.AutobahnFileConfig{ - Validators: validators, - MaxTxsPerBlock: 5_000, - BlockInterval: utils.Duration(400 * time.Millisecond), - ViewTimeout: utils.Duration(1500 * time.Millisecond), - DialInterval: utils.Duration(10 * time.Second), + Validators: validators, + MaxTxsPerBlock: 5_000, + BlockInterval: utils.Duration(400 * time.Millisecond), + ViewTimeout: utils.Duration(1500 * time.Millisecond), + DialInterval: utils.Duration(10 * time.Second), } // The flag defaults to "data/autobahn" so persistence is on without // operator action. node/setup.go rootifies the relative path against diff --git a/sei-tendermint/internal/autobahn/avail/state.go b/sei-tendermint/internal/autobahn/avail/state.go index 8ce78ff311..356b776106 100644 --- a/sei-tendermint/internal/autobahn/avail/state.go +++ b/sei-tendermint/internal/autobahn/avail/state.go @@ -585,7 +585,7 @@ func (s *State) produceBlock(n types.BlockNumber, key types.SecretKey, payload * return nil, fmt.Errorf("lane full") } if q.next != n { - return nil, fmt.Errorf("unexpected block number: got %v, want %v",n,q.next) + return nil, fmt.Errorf("unexpected block number: got %v, want %v", n, q.next) } var parent types.BlockHeaderHash if q.first < q.next { diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index 04b0c6a1e7..e539d442ce 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -145,8 +145,8 @@ type appProposalWithTimestamp struct { } type inner struct { - qcs map[types.GlobalBlockNumber]*types.FullCommitQC // [first,nextQC) - blocks map[types.GlobalBlockNumber]*types.Block // [first,nextBlock) + subset of [nextBlock,nextQC) + qcs map[types.GlobalBlockNumber]*types.FullCommitQC // [first,nextQC) + blocks map[types.GlobalBlockNumber]*types.Block // [first,nextBlock) + subset of [nextBlock,nextQC) // appProposal[n] contains appProposal block >=n. appProposals map[types.GlobalBlockNumber]appProposalWithTimestamp // [first,nextAppProposal) @@ -600,8 +600,10 @@ func (s *State) AppProposal(ctx context.Context, n types.GlobalBlockNumber) (*ty func (i *inner) nextToExecute(lane types.LaneID) types.BlockNumber { // TODO(gprusak): decide whether 0 is a good result in this case in general. - if i.first == i.nextAppProposal { return 0 } - n := i.nextAppProposal-1 + if i.first == i.nextAppProposal { + return 0 + } + n := i.nextAppProposal - 1 r := i.qcs[n].QC().LaneRange(lane) // TODO: this header can be actually extracted from FullCommitQC, so consider moving all this logic there. h := i.blocks[n].Header() @@ -609,20 +611,25 @@ func (i *inner) nextToExecute(lane types.LaneID) types.BlockNumber { // NOTE: here we assume the specific ordering of lane blocks in the CommitQC: // TODO(gprusak): move this logic closer to CommitQC switch { - case x<0: return r.Next() - case x>0: return r.First() - default: return h.BlockNumber()+1 + case x < 0: + return r.Next() + case x > 0: + return r.First() + default: + return h.BlockNumber() + 1 } } // Waits until lane block n is executed, returns the next block of this lane to be exectued (>n) -func (s *State) WaitUntilExecuted(ctx context.Context, lane types.LaneID, n types.BlockNumber) (types.BlockNumber,error) { - for inner,ctrl := range s.inner.Lock() { +func (s *State) WaitUntilExecuted(ctx context.Context, lane types.LaneID, n types.BlockNumber) (types.BlockNumber, error) { + for inner, ctrl := range s.inner.Lock() { for { if next := inner.nextToExecute(lane); n < next { - return next,nil + return next, nil + } + if err := ctrl.Wait(ctx); err != nil { + return 0, err } - if err:=ctrl.Wait(ctx); err!=nil { return 0,err } } } panic("unreachable") diff --git a/sei-tendermint/internal/autobahn/producer/mempool.go b/sei-tendermint/internal/autobahn/producer/mempool.go index dfec46f937..4f331960b5 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool.go +++ b/sei-tendermint/internal/autobahn/producer/mempool.go @@ -1,14 +1,14 @@ package producer - + import ( "context" "errors" "fmt" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" "github.com/ethereum/go-ethereum/common" - tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) var errTooLarge = errors.New("transaction too large") @@ -18,17 +18,17 @@ type blockSpec struct { gasEstimated uint64 gasWanted uint64 sizeBytes uint64 - txs [][]byte + txs [][]byte // nonces of accounts which are expected to be bumped by this block. // They are checked against the app state after the block is executed. evmNonces map[common.Address]uint64 } type mempool struct { - capacity uint64 - first types.BlockNumber - next types.BlockNumber - blocks map[types.BlockNumber]*blockSpec + capacity uint64 + first types.BlockNumber + next types.BlockNumber + blocks map[types.BlockNumber]*blockSpec nextBlock *blockSpec evmNonces map[common.Address]uint64 } @@ -60,7 +60,7 @@ func (s *State) UnconfirmedTxs() [][]byte { func (s *State) EvmNextPendingNonce(addr common.Address) uint64 { for m := range s.mempool.Lock() { - if nonce,ok := m.evmNonces[addr]; ok { + if nonce, ok := m.evmNonces[addr]; ok { return nonce } } @@ -76,24 +76,26 @@ func (s *State) mempoolFirst() types.BlockNumber { // Removes txs from mempool assigned to lane blocks m.next shouldn't really happen, // because local mempool is the only source of local lane blocks, // but we handle it gracefully anyway. - m.next = max(m.next,n) + m.next = max(m.next, n) } } @@ -111,21 +113,27 @@ func (s *State) InsertTx(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseChec return nil, errTooLarge } resp, err := s.cfg.App.CheckTxSafe(ctx, &abci.RequestCheckTxV2{Tx: tx}) - if err!=nil { return nil, err } - if !resp.IsOK() { return resp.ResponseCheckTx, nil } + if err != nil { + return nil, err + } + if !resp.IsOK() { + return resp.ResponseCheckTx, nil + } gasWanted := utils.Clamp[uint64](resp.GasWanted) - if gasWanted > s.cfg.MaxGasPerBlock { return nil, errTooLarge } - - for m,ctrl := range s.mempool.Lock() { + if gasWanted > s.cfg.MaxGasPerBlock { + return nil, errTooLarge + } + + for m, ctrl := range s.mempool.Lock() { // mempool is constructed as a FIFO - we do not delay insertions of large txs (going over cap) // in favor of waiting for smaller txs. This simple algorithm allows us to cap // pending txs to size of a single block. We can refine this rule later if needed. - if err:=ctrl.WaitUntil(ctx, func() bool { return !m.IsFull() }); err!=nil { + if err := ctrl.WaitUntil(ctx, func() bool { return !m.IsFull() }); err != nil { return nil, err } if resp.IsEVM { addr := resp.EVMSenderAddress - nonce,ok := m.evmNonces[addr] + nonce, ok := m.evmNonces[addr] if !ok { nonce = s.cfg.App.EvmNonce(addr) } @@ -136,11 +144,11 @@ func (s *State) InsertTx(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseChec m.evmNonces[addr] = nonce + 1 } // If any limit would be exceeded, then construct a payload. - ok := uint64(len(m.nextBlock.txs)) + 1 <= s.cfg.maxTxsPerBlock() - ok = ok && m.nextBlock.sizeBytes + uint64(len(tx)) <= types.MaxTxsBytesPerBlock - ok = ok && m.nextBlock.gasWanted + gasWanted <= s.cfg.MaxGasPerBlock + ok := uint64(len(m.nextBlock.txs))+1 <= s.cfg.maxTxsPerBlock() + ok = ok && m.nextBlock.sizeBytes+uint64(len(tx)) <= types.MaxTxsBytesPerBlock + ok = ok && m.nextBlock.gasWanted+gasWanted <= s.cfg.MaxGasPerBlock if !ok { - m.PushBlock() + m.PushBlock() ctrl.Updated() } @@ -153,7 +161,7 @@ func (s *State) InsertTx(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseChec b.gasEstimated += utils.Clamp[uint64](gasEstimated) b.gasWanted += utils.Clamp[uint64](resp.GasWanted) b.sizeBytes += uint64(len(tx)) - b.txs = append(b.txs,tx) + b.txs = append(b.txs, tx) } - return resp.ResponseCheckTx,nil + return resp.ResponseCheckTx, nil } diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index c6e5d314d1..e6a6c44c00 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -5,27 +5,27 @@ import ( "fmt" "time" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" + "github.com/ethereum/go-ethereum/common" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/avail" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/scope" - "github.com/ethereum/go-ethereum/common" "golang.org/x/time/rate" ) // Config is the config of the block scope. type Config struct { - MaxGasPerBlock uint64 - MaxTxsPerBlock uint64 + MaxGasPerBlock uint64 + MaxTxsPerBlock uint64 // Delay after which a non-full block can be produced. - BlockInterval time.Duration + BlockInterval time.Duration // TESTONLY: max rate at which lane is produced. It can be used to do // benchmarks with stable throughput, in case execution performance degrades // when overloaded. - MaxTxsPerSecond utils.Option[uint64] - App *proxy.Proxy + MaxTxsPerSecond utils.Option[uint64] + App *proxy.Proxy } const minTxGas = 21000 @@ -36,24 +36,25 @@ func (c *Config) maxTxsPerBlock() uint64 { // State is the block producer state. type State struct { - cfg *Config + cfg *Config mempool utils.Watch[*mempool] // consensus state to which published blocks will be reported. consensus *consensus.State } + // NewState constructs a new block producer state. // Returns an error if the current node is NOT a producer. func NewState(cfg *Config, consensus *consensus.State) *State { lane := consensus.Avail().PublicKey() n := consensus.Avail().NextBlock(lane) return &State{ - cfg: cfg, - mempool: utils.NewWatch(&mempool { - capacity: avail.BlocksPerLane, - first: n, - next: n, - blocks: map[types.BlockNumber]*blockSpec{}, - nextBlock: &blockSpec{evmNonces:map[common.Address]uint64{}}, + cfg: cfg, + mempool: utils.NewWatch(&mempool{ + capacity: avail.BlocksPerLane, + first: n, + next: n, + blocks: map[types.BlockNumber]*blockSpec{}, + nextBlock: &blockSpec{evmNonces: map[common.Address]uint64{}}, evmNonces: map[common.Address]uint64{}, }), consensus: consensus, @@ -65,9 +66,9 @@ func NewState(cfg *Config, consensus *consensus.State) *State { // * pushes new lane blocks from mempool to avail state // Note that mempool capacity bounds the number of unexecuted blocks of the local lane. // This is needed so that we can track the evm nonces of sequenced txs - mempool admits txs -// sequentially in the nonce order. +// sequentially in the nonce order. func (s *State) Run(ctx context.Context) error { - return scope.Run(ctx, func(ctx context.Context, scope scope.Scope) error { + return scope.Run(ctx, func(ctx context.Context, scope scope.Scope) error { availState := s.consensus.Avail() lane := availState.PublicKey() firstBlock := s.mempoolFirst() @@ -75,8 +76,8 @@ func (s *State) Run(ctx context.Context) error { // Task pruning executed lane blocks from the mempool dataState := s.consensus.Data() var err error - for toExecute := firstBlock ;; { - if toExecute,err = dataState.WaitUntilExecuted(ctx,lane,toExecute); err!=nil { + for toExecute := firstBlock; ; { + if toExecute, err = dataState.WaitUntilExecuted(ctx, lane, toExecute); err != nil { return err } s.pruneMempool(toExecute) @@ -92,34 +93,34 @@ func (s *State) Run(ctx context.Context) error { } limiter := rate.NewLimiter(limit, burst) lastBlockTime := time.Now() - for toProduce:=firstBlock;; toProduce += 1 { - if err := availState.WaitForCapacity(ctx,toProduce); err != nil { + for toProduce := firstBlock; ; toProduce += 1 { + if err := availState.WaitForCapacity(ctx, toProduce); err != nil { return fmt.Errorf("availState.WaitForCapacity(): %w", err) } var payload *types.Payload // Wait until either // * there is a full proposal in mempool // * BlockInterval since the last block passed AND mempool is non-empty - for m,ctrl := range s.mempool.Lock() { + for m, ctrl := range s.mempool.Lock() { // Wait for full payload with timeout. if err := utils.WithDeadline(ctx, utils.Some(lastBlockTime.Add(s.cfg.BlockInterval)), func(ctx context.Context) error { return ctrl.WaitUntil(ctx, func() bool { return toProduce < m.next }) - }); err!=nil { - if ctx.Err()!=nil { + }); err != nil { + if ctx.Err() != nil { return ctx.Err() } - // Wait for non-empty payload. - if err:=ctrl.WaitUntil(ctx, func() bool { - return toProduce < m.next || (toProduce==m.next && m.CanPushBlock()) - }); err!=nil { + // Wait for non-empty payload. + if err := ctrl.WaitUntil(ctx, func() bool { + return toProduce < m.next || (toProduce == m.next && m.CanPushBlock()) + }); err != nil { return err } // Seal the payload if needed. - if toProduce==m.next { + if toProduce == m.next { m.PushBlock() } } - b,ok := m.blocks[toProduce] + b, ok := m.blocks[toProduce] if !ok { // Block number tracking should always be in sync between avail state and mempool: // * mempool keeps blocks until they are executed. @@ -132,12 +133,12 @@ func (s *State) Run(ctx context.Context) error { CreatedAt: time.Now(), TotalGas: b.gasEstimated, Txs: b.txs, - }.Build() + }.Build() if err != nil { // This should never happen: we construct the payload from correctly sized data. panic(fmt.Errorf("PayloadBuilder{}.Build(): %w", err)) } - } + } if _, err := availState.ProduceBlock(toProduce, payload); err != nil { return fmt.Errorf("availState.ProduceBlock(): %w", err) } diff --git a/sei-tendermint/internal/p2p/giga_router.go b/sei-tendermint/internal/p2p/giga_router.go index 322e6b36f0..3b15a941f4 100644 --- a/sei-tendermint/internal/p2p/giga_router.go +++ b/sei-tendermint/internal/p2p/giga_router.go @@ -122,7 +122,7 @@ func NewGigaRouter(cfg *GigaRouterConfig, key NodeSecretKey) (*GigaRouter, error } func (r *GigaRouter) InsertTx(ctx context.Context, tx types.Tx) (*abci.ResponseCheckTx, error) { - return r.producer.InsertTx(ctx,tx) + return r.producer.InsertTx(ctx, tx) } // Mempool exposes Autobahn's producer-backed mempool surface to callers that diff --git a/sei-tendermint/internal/p2p/giga_router_test.go b/sei-tendermint/internal/p2p/giga_router_test.go index 919855629a..bb32f2ba2f 100644 --- a/sei-tendermint/internal/p2p/giga_router_test.go +++ b/sei-tendermint/internal/p2p/giga_router_test.go @@ -308,13 +308,13 @@ func TestGigaRouter_FinalizeBlocks(t *testing.T) { PersistentStateDir: utils.None[string](), }, Producer: &producer.Config{ - App: proxyApp, - MaxGasPerBlock: txGasUsed * maxTxsPerBlock, - MaxTxsPerBlock: maxTxsPerBlock, - MaxTxsPerSecond: utils.None[uint64](), - BlockInterval: 100 * time.Millisecond, + App: proxyApp, + MaxGasPerBlock: txGasUsed * maxTxsPerBlock, + MaxTxsPerBlock: maxTxsPerBlock, + MaxTxsPerSecond: utils.None[uint64](), + BlockInterval: 100 * time.Millisecond, }, - GenDoc: genDoc, + GenDoc: genDoc, }), }, ) @@ -449,13 +449,13 @@ func TestGigaRouter_EvmProxy(t *testing.T) { PersistentStateDir: utils.None[string](), }, Producer: &producer.Config{ - App: proxy.New(newTestApp(), proxy.NopMetrics()), - MaxGasPerBlock: 1, - MaxTxsPerBlock: 1, - MaxTxsPerSecond: utils.None[uint64](), - BlockInterval: time.Second, + App: proxy.New(newTestApp(), proxy.NopMetrics()), + MaxGasPerBlock: 1, + MaxTxsPerBlock: 1, + MaxTxsPerSecond: utils.None[uint64](), + BlockInterval: time.Second, }, - GenDoc: genDoc, + GenDoc: genDoc, }, nodeKeys[0]) require.NoError(t, err) diff --git a/sei-tendermint/internal/p2p/router_test.go b/sei-tendermint/internal/p2p/router_test.go index a2f1520391..04a7906559 100644 --- a/sei-tendermint/internal/p2p/router_test.go +++ b/sei-tendermint/internal/p2p/router_test.go @@ -334,11 +334,11 @@ func TestRouter_GigaSetWhenConfigured(t *testing.T) { PersistentStateDir: utils.None[string](), }, Producer: &producer.Config{ - App: proxyApp, - MaxGasPerBlock: 77_000_000, - MaxTxsPerBlock: 7_777, - MaxTxsPerSecond: utils.Some(uint64(999)), - BlockInterval: 777 * time.Millisecond, + App: proxyApp, + MaxGasPerBlock: 77_000_000, + MaxTxsPerBlock: 7_777, + MaxTxsPerSecond: utils.Some(uint64(999)), + BlockInterval: 777 * time.Millisecond, }, GenDoc: &types.GenesisDoc{ ChainID: "giga-e2e-test", diff --git a/sei-tendermint/internal/rpc/core/blocks.go b/sei-tendermint/internal/rpc/core/blocks.go index 7322493782..14e77a9878 100644 --- a/sei-tendermint/internal/rpc/core/blocks.go +++ b/sei-tendermint/internal/rpc/core/blocks.go @@ -269,7 +269,7 @@ func (env *Environment) BlockResults(ctx context.Context, req *coretypes.Request return &coretypes.ResultBlockResults{ Height: height, ConsensusParamUpdates: &tmproto.ConsensusParams{ - Block: &tmproto.BlockParams { + Block: &tmproto.BlockParams{ MaxGas: utils.Clamp[int64](giga.MaxGasPerBlock()), }, }, diff --git a/sei-tendermint/internal/rpc/core/env.go b/sei-tendermint/internal/rpc/core/env.go index 5fb33d44bf..7b60727e3d 100644 --- a/sei-tendermint/internal/rpc/core/env.go +++ b/sei-tendermint/internal/rpc/core/env.go @@ -82,12 +82,12 @@ type Environment struct { Router *p2p.Router // objects - PubKey utils.Option[crypto.PubKey] - GenDoc *types.GenesisDoc // cache the genesis structure - EventSinks []indexer.EventSink - EventBus *eventbus.EventBus // thread safe - EventLog utils.Option[*eventlog.Log] - Mempool utils.Option[*mempool.TxMempool] + PubKey utils.Option[crypto.PubKey] + GenDoc *types.GenesisDoc // cache the genesis structure + EventSinks []indexer.EventSink + EventBus *eventbus.EventBus // thread safe + EventLog utils.Option[*eventlog.Log] + Mempool utils.Option[*mempool.TxMempool] StateSyncReactor utils.Option[statesync.Reactor] Config config.RPCConfig @@ -257,7 +257,6 @@ func (env *Environment) requireConsensusReactor() (*consensus.Reactor, error) { return nil, fmt.Errorf("consensus reactor is not available") } - // StartService constructs and starts listeners for the RPC service // according to the config object, returning an error if the service // cannot be constructed or started. The listeners, which provide diff --git a/sei-tendermint/internal/rpc/core/mempool.go b/sei-tendermint/internal/rpc/core/mempool.go index 156ddfe0a4..78b15e6606 100644 --- a/sei-tendermint/internal/rpc/core/mempool.go +++ b/sei-tendermint/internal/rpc/core/mempool.go @@ -8,7 +8,6 @@ import ( "net/url" "time" - "github.com/sei-protocol/sei-chain/sei-tendermint/types" "github.com/ethereum/go-ethereum/common" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" @@ -16,6 +15,7 @@ import ( tmmath "github.com/sei-protocol/sei-chain/sei-tendermint/libs/math" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes" + "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) // EvmProxy returns the EVM RPC URL of the autobahn validator that owns the @@ -190,8 +190,8 @@ func (env *Environment) UnconfirmedTxs(ctx context.Context, req *coretypes.Reque sizeBytes += len(tx) } var txs types.Txs - for _, tx := range rawTxs[skipCount:min(skipCount+perPage,len(rawTxs))] { - txs = append(txs,tx) + for _, tx := range rawTxs[skipCount:min(skipCount+perPage, len(rawTxs))] { + txs = append(txs, tx) } return &coretypes.ResultUnconfirmedTxs{ Count: len(txs), @@ -241,7 +241,7 @@ func (env *Environment) NumUnconfirmedTxs(ctx context.Context) (*coretypes.Resul Count: len(rawTxs), Total: len(rawTxs), TotalBytes: int64(sizeBytes), - },nil + }, nil } mp, err := env.requireMempool() if err != nil { diff --git a/sei-tendermint/node/node.go b/sei-tendermint/node/node.go index f3fc6da8d8..bddcd2f00d 100644 --- a/sei-tendermint/node/node.go +++ b/sei-tendermint/node/node.go @@ -223,7 +223,7 @@ func makeNode( node.services = append(node.services, evReactor) node.rpcEnv.EvidencePool = utils.Some[sm.EvidencePool](evPool) node.evPool = utils.Some(evPool) - + if cfg.P2P.PexReactor { pxReactor, err := pex.NewReactor( node.router, @@ -245,7 +245,7 @@ func makeNode( } mpReactor.MarkReadyToStart() node.services = append(node.services, mpReactor) - + // make block executor for consensus and blockchain reactors to execute blocks blockExec := sm.NewBlockExecutor( stateStore, @@ -268,7 +268,7 @@ func makeNode( // app may modify the validator set, specifying ourself as the only validator. blockSync := !onlyValidatorIsUs(state, pubKey) waitSync := stateSync || blockSync - + consensusWAL, err := consensus.OpenWAL(cfg.Consensus.WalFile()) if err != nil { return nil, fmt.Errorf("consensus.OpenWAL(): %w", err) @@ -305,7 +305,7 @@ func makeNode( node.services = append(node.services, csReactor) node.rpcEnv.ConsensusReactor = utils.Some(csReactor) - + // Create the blockchain reactor. Note, we do not start block sync if we're // doing a state sync first. bcReactor, err := blocksync.NewReactor( @@ -335,7 +335,7 @@ func makeNode( } else if blockSync { nodeMetrics.consensus.BlockSyncing.Set(1) } - + postSyncHook := func(ctx context.Context, state sm.State) error { csReactor.SetStateSyncingMetrics(0) @@ -351,7 +351,7 @@ func makeNode( return nil } - + // Set up state sync reactor, and schedule a sync if requested. // FIXME The way we do phased startups (e.g. replay -> block sync -> consensus) is very messy, // we should clean this whole thing up. See: @@ -403,7 +403,7 @@ func makeNode( } node.services = append(node.services, node.rpcEnv.BlockSyncReactor) } - + node.rpcEnv.PubKey = pubKey node.BaseService = *service.NewBaseService("Node", node) @@ -517,7 +517,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) error { return err } n.rpcEnv.IsListening = true - if m,ok := n.mempool.Get(); ok { + if m, ok := n.mempool.Get(); ok { n.SpawnCritical("mempool", m.Run) } diff --git a/sei-tendermint/node/setup.go b/sei-tendermint/node/setup.go index 4ce7cecfef..380e8b759d 100644 --- a/sei-tendermint/node/setup.go +++ b/sei-tendermint/node/setup.go @@ -21,11 +21,11 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/consensus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/eventbus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/evidence" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" mempoolreactor "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool/reactor" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/conn" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/pex" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" sm "github.com/sei-protocol/sei-chain/sei-tendermint/internal/state" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/state/indexer" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/statesync" @@ -253,12 +253,13 @@ func buildGigaConfig( PersistentStateDir: fc.PersistentStateDir, }, Producer: &producer.Config{ - MaxGasPerBlock: maxGasPerBlock, - MaxTxsPerBlock: fc.MaxTxsPerBlock, - MaxTxsPerSecond: fc.MaxTxsPerSecond, - BlockInterval: time.Duration(fc.BlockInterval), + App: app, + MaxGasPerBlock: maxGasPerBlock, + MaxTxsPerBlock: fc.MaxTxsPerBlock, + MaxTxsPerSecond: fc.MaxTxsPerSecond, + BlockInterval: time.Duration(fc.BlockInterval), }, - GenDoc: genDoc, + GenDoc: genDoc, }, nil } @@ -360,7 +361,7 @@ func createRouter( if !ok { return nil, closer, fmt.Errorf("autobahn non-validator nodes are not supported yet; a local validator key is required") } - app,ok := app.Get() + app, ok := app.Get() if !ok { return nil, closer, fmt.Errorf("autobahn requires app") } From 52dde738003c31dc5b1f40dd576a3203a58ffb39 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 20:27:41 +0200 Subject: [PATCH 094/100] lastBlockTime --- sei-tendermint/internal/autobahn/producer/state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sei-tendermint/internal/autobahn/producer/state.go b/sei-tendermint/internal/autobahn/producer/state.go index e6a6c44c00..d2617e1e70 100644 --- a/sei-tendermint/internal/autobahn/producer/state.go +++ b/sei-tendermint/internal/autobahn/producer/state.go @@ -142,6 +142,7 @@ func (s *State) Run(ctx context.Context) error { if _, err := availState.ProduceBlock(toProduce, payload); err != nil { return fmt.Errorf("availState.ProduceBlock(): %w", err) } + lastBlockTime = time.Now() if err := limiter.WaitN(ctx, len(payload.Txs())); err != nil { return fmt.Errorf("limiter(): %w", err) } From 2c82f803a0e05136f63fd78b9bfaf973850af7b0 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 20:41:16 +0200 Subject: [PATCH 095/100] codex nonce test --- .../internal/autobahn/data/state.go | 2 +- .../internal/autobahn/producer/mempool.go | 1 + .../autobahn/producer/mempool_test.go | 164 ++++++++++++++++++ 3 files changed, 166 insertions(+), 1 deletion(-) diff --git a/sei-tendermint/internal/autobahn/data/state.go b/sei-tendermint/internal/autobahn/data/state.go index e539d442ce..c971386b48 100644 --- a/sei-tendermint/internal/autobahn/data/state.go +++ b/sei-tendermint/internal/autobahn/data/state.go @@ -620,7 +620,7 @@ func (i *inner) nextToExecute(lane types.LaneID) types.BlockNumber { } } -// Waits until lane block n is executed, returns the next block of this lane to be exectued (>n) +// Waits until lane block n is executed, returns the next block of this lane to be executed (>n) func (s *State) WaitUntilExecuted(ctx context.Context, lane types.LaneID, n types.BlockNumber) (types.BlockNumber, error) { for inner, ctrl := range s.inner.Lock() { for { diff --git a/sei-tendermint/internal/autobahn/producer/mempool.go b/sei-tendermint/internal/autobahn/producer/mempool.go index 4f331960b5..504f890112 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool.go +++ b/sei-tendermint/internal/autobahn/producer/mempool.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/ethereum/go-ethereum/common" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" diff --git a/sei-tendermint/internal/autobahn/producer/mempool_test.go b/sei-tendermint/internal/autobahn/producer/mempool_test.go index 53cb2eb431..284ac3d92a 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool_test.go +++ b/sei-tendermint/internal/autobahn/producer/mempool_test.go @@ -4,9 +4,11 @@ import ( "context" "encoding/binary" "errors" + "math/big" "testing" "time" + "github.com/ethereum/go-ethereum/common" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/consensus" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/data" @@ -31,6 +33,12 @@ type txSpec struct { gasEstimated int64 } +type evmTxSpec struct { + tx []byte + sender common.Address + nonce uint64 +} + type overflowSpec struct { count bool bytes bool @@ -95,6 +103,46 @@ func (a txSpecApp) CheckTx(_ context.Context, req *abci.RequestCheckTxV2) *abci. } } +type evmTxSpecApp struct { + abci.BaseApplication + baseNonces map[common.Address]uint64 +} + +func (a evmTxSpecApp) CheckTx(_ context.Context, req *abci.RequestCheckTxV2) *abci.ResponseCheckTxV2 { + spec := decodeEvmTxSpec(req.Tx) + return &abci.ResponseCheckTxV2{ + ResponseCheckTx: &abci.ResponseCheckTx{ + Code: abci.CodeTypeOK, + GasWanted: 50, + GasEstimated: 40, + }, + IsEVM: true, + EVMNonce: spec.nonce, + EVMSenderAddress: spec.sender, + SeiSenderAddress: []byte("sender"), + EVMRequiredBalance: big.NewInt(1), + } +} + +func (a evmTxSpecApp) EvmNonce(addr common.Address) uint64 { + return a.baseNonces[addr] +} + +func encodeEvmTx(sender common.Address, nonce uint64) []byte { + tx := make([]byte, common.AddressLength+8) + copy(tx, sender.Bytes()) + binary.BigEndian.PutUint64(tx[common.AddressLength:], nonce) + return tx +} + +func decodeEvmTxSpec(tx []byte) evmTxSpec { + return evmTxSpec{ + tx: tx, + sender: common.BytesToAddress(tx[:common.AddressLength]), + nonce: binary.BigEndian.Uint64(tx[common.AddressLength:]), + } +} + func newSealScenario(t *testing.T, rng utils.Rng) sealScenario { const ( wantSealedBlocks = 24 @@ -382,3 +430,119 @@ func TestInsertTxSealsCurrentBlockWhenTxCountWouldOverflow(t *testing.T) { }) require.NoError(t, err) } + +func TestInsertTxRequiresEVMNonceOrderAcrossAccountsAndBlocks(t *testing.T) { + rng := utils.TestRng() + accountCount := 3 + rng.Intn(2) + blockSize := 2 + rng.Intn(2) + goodCount := 2*blockSize + 1 + rng.Intn(blockSize+1) + + accounts := make([]common.Address, accountCount) + baseNonces := make(map[common.Address]uint64, accountCount) + expectedNonces := make(map[common.Address]uint64, accountCount) + perAccountAccepted := make(map[common.Address]int, accountCount) + for i := range accountCount { + accounts[i] = common.BytesToAddress(utils.GenBytes(rng, common.AddressLength)) + baseNonces[accounts[i]] = uint64(rng.Intn(20)) + expectedNonces[accounts[i]] = baseNonces[accounts[i]] + } + + type attempt struct { + spec evmTxSpec + isBad bool + } + attempts := make([]attempt, 0, 2*goodCount) + good := make([]evmTxSpec, 0, goodCount) + + newTx := func(sender common.Address, nonce uint64, label byte) evmTxSpec { + _ = label + return evmTxSpec{ + tx: encodeEvmTx(sender, nonce), + sender: sender, + nonce: nonce, + } + } + badNonce := func(sender common.Address, want uint64) uint64 { + switch rng.Intn(3) { + case 0: + return want + 1 + uint64(rng.Intn(3)) + case 1: + if want == 0 { + return 1 + uint64(rng.Intn(3)) + } + return want - 1 + default: + if want > baseNonces[sender] { + return baseNonces[sender] + uint64(rng.Intn(int(want-baseNonces[sender]))) + } + return want + 2 + uint64(rng.Intn(2)) + } + } + + for i := range goodCount { + sender := accounts[rng.Intn(len(accounts))] + want := expectedNonces[sender] + if i > 0 || want > 0 { + bad := newTx(sender, badNonce(sender, want), 'b') + attempts = append(attempts, attempt{spec: bad, isBad: true}) + } + ok := newTx(sender, want, 'g') + attempts = append(attempts, attempt{spec: ok}) + good = append(good, ok) + expectedNonces[sender] = want + 1 + perAccountAccepted[sender] += 1 + } + + state := newTestState(t, evmTxSpecApp{baseNonces: baseNonces}) + state.cfg.MaxTxsPerBlock = uint64(blockSize) + + currentExpected := make(map[common.Address]uint64, len(baseNonces)) + for addr, nonce := range baseNonces { + currentExpected[addr] = nonce + } + assertPendingNonces := func() { + t.Helper() + for addr, nonce := range currentExpected { + require.Equal(t, nonce, state.EvmNextPendingNonce(addr)) + } + } + for _, x := range attempts { + resp, err := state.InsertTx(t.Context(), x.spec.tx) + if x.isBad { + require.Nil(t, resp) + require.ErrorIs(t, err, errBadNonce) + assertPendingNonces() + continue + } + require.NoError(t, err) + require.EqualValues(t, 50, resp.GasWanted) + currentExpected[x.spec.sender] += 1 + assertPendingNonces() + } + + assertPendingNonces() + + sealedBlocks := (len(good) - 1) / blockSize + openStart := sealedBlocks * blockSize + require.Equal(t, txsOfEVM(good[openStart:]), state.UnconfirmedTxs()) + + for m := range state.mempool.Lock() { + require.Equal(t, sealedBlocks, len(m.blocks)) + for i := range sealedBlocks { + from := i * blockSize + to := from + blockSize + require.Equal(t, txsOfEVM(good[from:to]), m.blocks[m.first+types.BlockNumber(i)].txs) + } + require.Equal(t, txsOfEVM(good[openStart:]), m.nextBlock.txs) + return + } + t.Fatal("unreachable") +} + +func txsOfEVM(specs []evmTxSpec) [][]byte { + txs := make([][]byte, len(specs)) + for i, spec := range specs { + txs[i] = spec.tx + } + return txs +} From de00149221cba508eb8c7fbc3cf8cadf1e56f2b0 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 20:47:15 +0200 Subject: [PATCH 096/100] some fixes --- sei-tendermint/internal/autobahn/producer/mempool.go | 7 +++++-- sei-tendermint/internal/rpc/core/env.go | 2 +- sei-tendermint/internal/rpc/core/lag_status.go | 5 ++++- sei-tendermint/internal/rpc/core/status.go | 8 +++++--- sei-tendermint/node/node.go | 7 ++++--- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/sei-tendermint/internal/autobahn/producer/mempool.go b/sei-tendermint/internal/autobahn/producer/mempool.go index 504f890112..2a21c9c4c7 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool.go +++ b/sei-tendermint/internal/autobahn/producer/mempool.go @@ -141,7 +141,6 @@ func (s *State) InsertTx(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseChec if nonce != resp.EVMNonce { return nil, fmt.Errorf("%w: got %v, want %v", errBadNonce, resp.EVMNonce, nonce) } - m.nextBlock.evmNonces[addr] = nonce + 1 m.evmNonces[addr] = nonce + 1 } // If any limit would be exceeded, then construct a payload. @@ -151,7 +150,7 @@ func (s *State) InsertTx(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseChec if !ok { m.PushBlock() ctrl.Updated() - } + } // Normalize the gas estimate. gasEstimated := resp.GasEstimated @@ -163,6 +162,10 @@ func (s *State) InsertTx(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseChec b.gasWanted += utils.Clamp[uint64](resp.GasWanted) b.sizeBytes += uint64(len(tx)) b.txs = append(b.txs, tx) + if resp.IsEVM { + addr := resp.EVMSenderAddress + b.evmNonces[addr] = m.evmNonces[addr] + } } return resp.ResponseCheckTx, nil } diff --git a/sei-tendermint/internal/rpc/core/env.go b/sei-tendermint/internal/rpc/core/env.go index 7b60727e3d..f539cb7d0b 100644 --- a/sei-tendermint/internal/rpc/core/env.go +++ b/sei-tendermint/internal/rpc/core/env.go @@ -73,7 +73,7 @@ type Environment struct { EvidencePool utils.Option[sm.EvidencePool] ConsensusState utils.Option[ConsensusState] ConsensusReactor utils.Option[*consensus.Reactor] - BlockSyncReactor *blocksync.Reactor + BlockSyncReactor utils.Option[*blocksync.Reactor] IsListening bool Listeners []string diff --git a/sei-tendermint/internal/rpc/core/lag_status.go b/sei-tendermint/internal/rpc/core/lag_status.go index 7d57ba1b18..8761bb3095 100644 --- a/sei-tendermint/internal/rpc/core/lag_status.go +++ b/sei-tendermint/internal/rpc/core/lag_status.go @@ -9,7 +9,10 @@ import ( // LagStatus returns Tendermint lag status, if lag is over a certain threshold func (env *Environment) LagStatus(ctx context.Context) (*coretypes.ResultLagStatus, error) { currentHeight := env.BlockStore.Height() - maxPeerBlockHeight := env.BlockSyncReactor.GetMaxPeerBlockHeight() + maxPeerBlockHeight := int64(0) + if reactor, ok := env.BlockSyncReactor.Get(); ok { + maxPeerBlockHeight = reactor.GetMaxPeerBlockHeight() + } lag := int64(0) // Calculate lag diff --git a/sei-tendermint/internal/rpc/core/status.go b/sei-tendermint/internal/rpc/core/status.go index 602376e71d..712f110ede 100644 --- a/sei-tendermint/internal/rpc/core/status.go +++ b/sei-tendermint/internal/rpc/core/status.go @@ -135,9 +135,11 @@ func (env *Environment) Status(ctx context.Context) (*coretypes.ResultStatus, er result.SyncInfo.CatchingUp = reactor.WaitSync() } - result.SyncInfo.MaxPeerBlockHeight = env.BlockSyncReactor.GetMaxPeerBlockHeight() - result.SyncInfo.TotalSyncedTime = env.BlockSyncReactor.GetTotalSyncedTime() - result.SyncInfo.RemainingTime = env.BlockSyncReactor.GetRemainingSyncTime() + if reactor, ok := env.BlockSyncReactor.Get(); ok { + result.SyncInfo.MaxPeerBlockHeight = reactor.GetMaxPeerBlockHeight() + result.SyncInfo.TotalSyncedTime = reactor.GetTotalSyncedTime() + result.SyncInfo.RemainingTime = reactor.GetRemainingSyncTime() + } if reactor, ok := env.StateSyncReactor.Get(); ok { result.SyncInfo.TotalSnapshots = reactor.TotalSnapshots() diff --git a/sei-tendermint/node/node.go b/sei-tendermint/node/node.go index bddcd2f00d..b358a29120 100644 --- a/sei-tendermint/node/node.go +++ b/sei-tendermint/node/node.go @@ -326,7 +326,7 @@ func makeNode( return nil, fmt.Errorf("blocksync.NewReactor(): %w", err) } node.services = append(node.services, bcReactor) - node.rpcEnv.BlockSyncReactor = bcReactor + node.rpcEnv.BlockSyncReactor = utils.Some(bcReactor) // Make ConsensusReactor. Don't enable fully if doing a state sync and/or block sync first. // FIXME We need to update metrics here, since other reactors don't have access to them. @@ -392,7 +392,7 @@ func makeNode( } } } else { - node.rpcEnv.BlockSyncReactor, err = blocksync.NewReactor( + bcReactor, err := blocksync.NewReactor( stateStore, blockStore, node.router, @@ -401,7 +401,8 @@ func makeNode( if err != nil { return nil, fmt.Errorf("blocksync.NewReactor(): %w", err) } - node.services = append(node.services, node.rpcEnv.BlockSyncReactor) + node.rpcEnv.BlockSyncReactor = utils.Some(bcReactor) + node.services = append(node.services, bcReactor) } node.rpcEnv.PubKey = pubKey From 35564b9242fd347132a2e8faa8d0a087c625324d Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 20:49:03 +0200 Subject: [PATCH 097/100] test fixes --- sei-tendermint/node/setup_test.go | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/sei-tendermint/node/setup_test.go b/sei-tendermint/node/setup_test.go index 091117038c..facb9cdd4b 100644 --- a/sei-tendermint/node/setup_test.go +++ b/sei-tendermint/node/setup_test.go @@ -14,7 +14,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/config" "github.com/sei-protocol/sei-chain/sei-tendermint/crypto/ed25519" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/tcp" @@ -57,9 +57,7 @@ func defaultFileConfig(validators []config.AutobahnValidator) *config.AutobahnFi Validators: validators, MaxTxsPerBlock: 5_000, MaxTxsPerSecond: utils.None[uint64](), - MempoolSize: 5_000, BlockInterval: utils.Duration(400 * time.Millisecond), - AllowEmptyBlocks: false, ViewTimeout: utils.Duration(1500 * time.Millisecond), PersistentStateDir: utils.None[string](), DialInterval: utils.Duration(10 * time.Second), @@ -69,13 +67,8 @@ func defaultFileConfig(validators []config.AutobahnValidator) *config.AutobahnFi // testGenesisMaxGas is the gas limit baked into the test genesis doc. const testGenesisMaxGas int64 = 50_000_000 -func makeTestGigaDeps() (*mempool.TxMempool, *types.GenesisDoc) { - txMempool := mempool.NewTxMempool( - mempool.TestConfig(), - kvstore.NewProxy(), - mempool.NopMetrics(), - mempool.NopTxConstraintsFetcher, - ) +func makeTestGigaDeps() (*proxy.Proxy, *types.GenesisDoc) { + app := kvstore.NewProxy() genDoc := &types.GenesisDoc{ ChainID: "test-chain", // Nontrivial InitialHeight so any future code that assumes the @@ -85,7 +78,7 @@ func makeTestGigaDeps() (*mempool.TxMempool, *types.GenesisDoc) { Block: types.BlockParams{MaxGas: testGenesisMaxGas}, }, } - return txMempool, genDoc + return app, genDoc } func TestBuildGigaConfig_EmptyPathErrors(t *testing.T) { @@ -106,9 +99,7 @@ func TestBuildGigaConfig_EnabledWithValidators(t *testing.T) { Validators: []config.AutobahnValidator{v1, v2, v3}, MaxTxsPerBlock: 5_000, MaxTxsPerSecond: utils.Some(uint64(1_000)), - MempoolSize: 20_000, BlockInterval: utils.Duration(200 * time.Millisecond), - AllowEmptyBlocks: true, ViewTimeout: utils.Duration(3 * time.Second), PersistentStateDir: utils.Some("/tmp/autobahn-state"), DialInterval: utils.Duration(5 * time.Second), @@ -144,9 +135,7 @@ func TestBuildGigaConfig_EnabledWithValidators(t *testing.T) { maxTps, ok := result.Producer.MaxTxsPerSecond.Get() require.True(t, ok) assert.Equal(t, uint64(1_000), maxTps) - assert.Equal(t, uint64(20_000), result.Producer.MempoolSize) assert.Equal(t, 200*time.Millisecond, result.Producer.BlockInterval) - assert.True(t, result.Producer.AllowEmptyBlocks) assert.Equal(t, genDoc, result.GenDoc) } From 43c1bc2128a0c3abbae840f83e1885eac875706a Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 20:51:28 +0200 Subject: [PATCH 098/100] fixes --- sei-tendermint/rpc/client/eventstream/eventstream_test.go | 3 ++- sei-tendermint/rpc/client/rpc_test.go | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sei-tendermint/rpc/client/eventstream/eventstream_test.go b/sei-tendermint/rpc/client/eventstream/eventstream_test.go index 252dd97276..2a4413c985 100644 --- a/sei-tendermint/rpc/client/eventstream/eventstream_test.go +++ b/sei-tendermint/rpc/client/eventstream/eventstream_test.go @@ -13,6 +13,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/eventlog" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/eventlog/cursor" rpccore "github.com/sei-protocol/sei-chain/sei-tendermint/internal/rpc/core" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/rpc/client/eventstream" "github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes" "github.com/sei-protocol/sei-chain/sei-tendermint/types" @@ -192,7 +193,7 @@ func newStreamTester(t *testing.T, query string, logOpts eventlog.LogSettings, s t.Fatalf("Creating event log: %v", err) } s.log = lg - s.env = &rpccore.Environment{EventLog: lg} + s.env = &rpccore.Environment{EventLog: utils.Some(lg)} s.stream = eventstream.New(s, query, streamOpts) return s } diff --git a/sei-tendermint/rpc/client/rpc_test.go b/sei-tendermint/rpc/client/rpc_test.go index ec39af03e2..8da3411ef3 100644 --- a/sei-tendermint/rpc/client/rpc_test.go +++ b/sei-tendermint/rpc/client/rpc_test.go @@ -569,7 +569,9 @@ func getMempool(t *testing.T, srv service.Service) *mempool.TxMempool { RPCEnvironment() *rpccore.Environment }) require.True(t, ok) - return n.RPCEnvironment().Mempool + mp, ok := n.RPCEnvironment().Mempool.Get() + require.True(t, ok) + return mp } // these cases are roughly the same as the TestClientMethodCalls, but From 2161b70b4c32b4f7430da0ee4848c4dbb344bc5b Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 21:05:00 +0200 Subject: [PATCH 099/100] fix --- sei-tendermint/internal/autobahn/producer/mempool.go | 7 +++++-- sei-tendermint/node/setup_test.go | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sei-tendermint/internal/autobahn/producer/mempool.go b/sei-tendermint/internal/autobahn/producer/mempool.go index 2a21c9c4c7..ea95aaf2e6 100644 --- a/sei-tendermint/internal/autobahn/producer/mempool.go +++ b/sei-tendermint/internal/autobahn/producer/mempool.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - + "github.com/ethereum/go-ethereum/common" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" @@ -149,8 +149,11 @@ func (s *State) InsertTx(ctx context.Context, tx tmtypes.Tx) (*abci.ResponseChec ok = ok && m.nextBlock.gasWanted+gasWanted <= s.cfg.MaxGasPerBlock if !ok { m.PushBlock() + } + if len(m.nextBlock.txs) == 0 { + // We notify that we start a new block. ctrl.Updated() - } + } // Normalize the gas estimate. gasEstimated := resp.GasEstimated diff --git a/sei-tendermint/node/setup_test.go b/sei-tendermint/node/setup_test.go index facb9cdd4b..0f57c7198c 100644 --- a/sei-tendermint/node/setup_test.go +++ b/sei-tendermint/node/setup_test.go @@ -14,8 +14,8 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/config" "github.com/sei-protocol/sei-chain/sei-tendermint/crypto/ed25519" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/types" - "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/proxy" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/tcp" "github.com/sei-protocol/sei-chain/sei-tendermint/types" From a081eb7bf7a29ae896f24ed7a5aeac13f4f173a9 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 29 May 2026 21:16:28 +0200 Subject: [PATCH 100/100] test fix --- sei-tendermint/internal/p2p/giga/avail_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sei-tendermint/internal/p2p/giga/avail_test.go b/sei-tendermint/internal/p2p/giga/avail_test.go index 14410556f8..4e614af244 100644 --- a/sei-tendermint/internal/p2p/giga/avail_test.go +++ b/sei-tendermint/internal/p2p/giga/avail_test.go @@ -49,7 +49,11 @@ func TestAvailClientServer(t *testing.T) { a := node.consensus.Avail() lane := a.PublicKey() for range totalBlocks { - if _, err := a.ProduceBlock(a.NextBlock(lane), types.GenPayload(rng)); err != nil { + n := a.NextBlock(lane) + if err := a.WaitForCapacity(ctx, n); err != nil { + return fmt.Errorf("waitForCapacity(): %w", err) + } + if _, err := a.ProduceBlock(n, types.GenPayload(rng)); err != nil { return fmt.Errorf("produceBlock(): %w", err) } }