8000 Remove priority mempool by hvanz · Pull Request #281 · cometbft/cometbft · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Remove priority mempool #281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
Feb 23, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
- [mempool] Remove priority mempool.
([\#260](https://github.com/cometbft/cometbft/issues/260))
- [config] Remove `Version` field from `MempoolConfig`.
([\#260](https://github.com/cometbft/cometbft/issues/260))
- [protobuf] Remove fields `sender`, `priority`, and `mempool_error` from
`ResponseCheckTx`. ([\#260](https://github.com/cometbft/cometbft/issues/260))
15 changes: 15 additions & 0 deletions UPGRADING.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ This guide provides instructions for upgrading to specific versions of CometBFT.
adding a list of addresses to the addressbook upon initializing a node. This is an
alternative to `PersistentPeers`. `PersistentPeers` shold be only used for
nodes that you want to keep a constant connection with i.e. sentry nodes
* The field `Version` in the mempool section has been removed. The priority
mempool (what was called version `v1`) has been removed (see below), thus
there is only one implementation of the mempool available (what was called
`v0`).
* Config fields `TTLDuration` and `TTLNumBlocks`, which were only used by the priority
mempool, have been removed.

### ABCI Changes

Expand Down Expand Up @@ -37,3 +43,12 @@ This guide provides instructions for upgrading to specific versions of CometBFT.
guidelines](https://developers.google.com/protocol-buffers/docs/proto3#updating),
this should have no effect on the wire-level encoding for UTF8-encoded
strings.

### Mempool Changes

* The priority mempool (what was referred in the code as version `v1`) has been
removed. There is now only one mempool (what was called version `v0`), that
is, the default implementation as a queue of transactions.
* In the protobuf message `ResponseCheckTx`, fields `sender`, `priority`, and
`mempool_error`, which were only used by the priority mempool, were removed
but still kept in the message as "reserved".
516 changes: 188 additions & 328 deletions abci/types/types.pb.go

Large diffs are not rendered by default.

41 changes: 9 additions & 32 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,6 @@ const (
// DefaultLogLevel defines a default log level as INFO.
DefaultLogLevel = "info"

// Mempool versions. V1 is prioritized mempool, v0 is regular mempool.
// Default is v0.
MempoolV0 = "v0"
MempoolV1 = "v1"

DefaultTendermintDir = ".cometbft"
DefaultConfigDir = "config"
DefaultDataDir = "data"
Expand Down Expand Up @@ -709,11 +704,12 @@ func DefaultFuzzConnConfig() *FuzzConnConfig {
// MempoolConfig

// MempoolConfig defines the configuration options for the CometBFT mempool
//
// Note: Until v0.37 there was a `Version` field to select which implementation
// of the mempool to use. Two versions used to exist: the current, default
// implementation (previously called v0), and a prioritized mempool (v1), which
// was removed (see https://github.com/cometbft/cometbft/issues/260).
type MempoolConfig struct {
// Mempool version to use:
// 1) "v0" - (default) FIFO mempool.
// 2) "v1" - prioritized mempool.
Version string `mapstructure:"version"`
// RootDir is the root directory for all data. This should be configured via
// the $CMTHOME env variable or --home cmd flag rather than overriding this
// struct field.
Expand Down Expand Up @@ -754,39 +750,20 @@ type MempoolConfig struct {
// Including space needed by encoding (one varint per transaction).
// XXX: Unused due to https://github.com/tendermint/tendermint/issues/5796
MaxBatchBytes int `mapstructure:"max_batch_bytes"`

// TTLDuration, if non-zero, defines the maximum amount of time a transaction
// can exist for in the mempool.
//
// Note, if TTLNumBlocks is also defined, a transaction will be removed if it
// has existed in the mempool at least TTLNumBlocks number of blocks or if it's
// insertion time into the mempool is beyond TTLDuration.
TTLDuration time.Duration `mapstructure:"ttl-duration"`

// TTLNumBlocks, if non-zero, defines the maximum number of blocks a transaction
// can exist for in the mempool.
//
// Note, if TTLDuration is also defined, a transaction will be removed if it
// has existed in the mempool at least TTLNumBlocks number of blocks or if
// it's insertion time into the mempool is beyond TTLDuration.
TTLNumBlocks int64 `mapstructure:"ttl-num-blocks"`
}

// DefaultMempoolConfig returns a default configuration for the CometBFT mempool
func DefaultMempoolConfig() *MempoolConfig {
return &MempoolConfig{
Version: MempoolV0,
Recheck: true,
Broadcast: true,
WalPath: "",
// Each signature verification takes .5ms, Size reduced until we implement
// ABCI Recheck
Size: 5000,
MaxTxsBytes: 1024 * 1024 * 1024, // 1GB
CacheSize: 10000,
MaxTxBytes: 1024 * 1024, // 1MB
TTLDuration: 0 * time.Second,
TTLNumBlocks: 0,
Size: 5000,
MaxTxsBytes: 1024 * 1024 * 1024, // 1GB
CacheSize: 10000,
MaxTxBytes: 1024 * 1024, // 1MB
}
}

Expand Down
21 changes: 0 additions & 21 deletions config/toml.go
Original file line number Diff line number Diff line change
Expand Up @@ -349,11 +349,6 @@ dial_timeout = "{{ .P2P.DialTimeout }}"
#######################################################
[mempool]

# Mempool version to use:
# 1) "v0" - (default) FIFO mempool.
# 2) "v1" - prioritized mempool.
version = "{{ .Mempool.Version }}"

# Recheck (default: true) defines whether CometBFT should recheck the
# validity for all remaining transaction in the mempool after a block.
# Since a block affects the application state, some transactions in the
Expand Down Expand Up @@ -399,22 +394,6 @@ max_tx_bytes = {{ .Mempool.MaxTxBytes }}
# XXX: Unused due to https://github.com/tendermint/tendermint/issues/5796
max_batch_bytes = {{ .Mempool.MaxBatchBytes }}

# ttl-duration, if non-zero, defines the maximum amount of time a transaction
# can exist for in the mempool.
#
# Note, if ttl-num-blocks is also defined, a transaction will be removed if it
# has existed in the mempool at least ttl-num-blocks number of blocks or if it's
# insertion time into the mempool is beyond ttl-duration.
ttl-duration = "{{ .Mempool.TTLDuration }}"

# ttl-num-blocks, if non-zero, defines the maximum number of blocks a transaction
# can exist for in the mempool.
#
# Note, if ttl-duration is also defined, a transaction will be removed if it
# has existed in the mempool at least ttl-num-blocks number of blocks or if
# it's insertion time into the mempool is beyond ttl-duration.
ttl-num-blocks = {{ .Mempool.TTLNumBlocks }}

#######################################################
### State Sync Configuration Options ###
#######################################################
Expand Down
26 changes: 5 additions & 21 deletions consensus/byzantine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,6 @@ import (
cmtsync "github.com/cometbft/cometbft/libs/sync"
mempl "github.com/cometbft/cometbft/mempool"

cfg "github.com/cometbft/cometbft/config"
mempoolv0 "github.com/cometbft/cometbft/mempool/v0"
mempoolv1 "github.com/cometbft/cometbft/mempool/v1"
"github.com/cometbft/cometbft/p2p"
cmtcons "github.com/cometbft/cometbft/proto/tendermint/consensus"
cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
Expand Down Expand Up @@ -71,24 +68,11 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
proxyAppConnConMem := abcicli.NewLocalClient(mtx, app)

// Make Mempool
var mempool mempl.Mempool

switch thisConfig.Mempool.Version {
case cfg.MempoolV0:
mempool = mempoolv0.NewCListMempool(config.Mempool,
proxyAppConnConMem,
state.LastBlockHeight,
mempoolv0.WithPreCheck(sm.TxPreCheck(state)),
mempoolv0.WithPostCheck(sm.TxPostCheck(state)))
case cfg.MempoolV1:
mempool = mempoolv1.NewTxMempool(logger,
config.Mempool,
proxyAppConnConMem,
state.LastBlockHeight,
mempoolv1.WithPreCheck(sm.TxPreCheck(state)),
mempoolv1.WithPostCheck(sm.TxPostCheck(state)),
)
}
mempool := mempl.NewCListMempool(config.Mempool,
proxyAppConnConMem,
state.LastBlockHeight,
mempl.WithPreCheck(sm.TxPreCheck(state)),
mempl.WithPostCheck(sm.TxPostCheck(state)))

if thisConfig.Consensus.WaitForTxs() {
mempool.EnableTxsAvailable()
Expand Down
30 changes: 7 additions & 23 deletions consensus/common_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@ import (
cmtpubsub "github.com/cometbft/cometbft/libs/pubsub"
cmtsync "github.com/cometbft/cometbft/libs/sync"
mempl "github.com/cometbft/cometbft/mempool"
mempoolv0 "github.com/cometbft/cometbft/mempool/v0"
mempoolv1 "github.com/cometbft/cometbft/mempool/v1"
"github.com/cometbft/cometbft/p2p"
"github.com/cometbft/cometbft/privval"
cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
Expand Down Expand Up @@ -403,27 +401,13 @@ func newStateWithConfigAndBlockStore(
memplMetrics := mempl.NopMetrics()

// Make Mempool
var mempool mempl.Mempool

switch config.Mempool.Version {
case cfg.MempoolV0:
mempool = mempoolv0.NewCListMempool(config.Mempool,
proxyAppConnConMem,
state.LastBlockHeight,
mempoolv0.WithMetrics(memplMetrics),
mempoolv0.WithPreCheck(sm.TxPreCheck(state)),
mempoolv0.WithPostCheck(sm.TxPostCheck(state)))
case cfg.MempoolV1:
logger := consensusLogger()
mempool = mempoolv1.NewTxMempool(logger,
config.Mempool,
proxyAppConnConMem,
state.LastBlockHeight,
mempoolv1.WithMetrics(memplMetrics),
mempoolv1.WithPreCheck(sm.TxPreCheck(state)),
mempoolv1.WithPostCheck(sm.TxPostCheck(state)),
)
}
mempool := mempl.NewCListMempool(config.Mempool,
proxyAppConnConMem,
state.LastBlockHeight,
mempl.WithMetrics(memplMetrics),
mempl.WithPreCheck(sm.TxPreCheck(state)),
mempl.WithPostCheck(sm.TxPostCheck(state)))

if thisConfig.Consensus.WaitForTxs() {
mempool.EnableTxsAvailable()
}
Expand Down
29 changes: 7 additions & 22 deletions consensus/reactor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,6 @@ import (
"github.com/cometbft/cometbft/libs/log"
cmtsync "github.com/cometbft/cometbft/libs/sync"
mempl "github.com/cometbft/cometbft/mempool"
mempoolv0 "github.com/cometbft/cometbft/mempool/v0"
mempoolv1 "github.com/cometbft/cometbft/mempool/v1"
"github.com/cometbft/cometbft/p2p"
p2pmock "github.com/cometbft/cometbft/p2p/mock"
cmtcons "github.com/cometbft/cometbft/proto/tendermint/consensus"
Expand Down Expand Up @@ -163,26 +161,13 @@ func TestReactorWithEvidence(t *testing.T) {
proxyAppConnConMem := abcicli.NewLocalClient(mtx, app)

// Make Mempool
var mempool mempl.Mempool

switch config.Mempool.Version {
case cfg.MempoolV0:
mempool = mempoolv0.NewCListMempool(config.Mempool,
proxyAppConnConMem,
state.LastBlockHeight,
mempoolv0.WithMetrics(memplMetrics),
mempoolv0.WithPreCheck(sm.TxPreCheck(state)),
mempoolv0.WithPostCheck(sm.TxPostCheck(state)))
case cfg.MempoolV1:
mempool = mempoolv1.NewTxMempool(logger,
config.Mempool,
proxyAppConnConMem,
state.LastBlockHeight,
mempoolv1.WithMetrics(memplMetrics),
mempoolv1.WithPreCheck(sm.TxPreCheck(state)),
mempoolv1.WithPostCheck(sm.TxPostCheck(state)),
)
}
mempool := mempl.NewCListMempool(config.Mempool,
proxyAppConnConMem,
state.LastBlockHeight,
mempl.WithMetrics(memplMetrics),
mempl.WithPreCheck(sm.TxPreCheck(state)),
mempl.WithPostCheck(sm.TxPostCheck(state)))

if thisConfig.Consensus.WaitForTxs() {
mempool.EnableTxsAvailable()
}
Expand Down
28 changes: 27 additions & 1 deletion consensus/replay_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -904,6 +904,32 @@ func buildTMStateFromChain(
return state
}

func makeBlocks(n int, state sm.State, privVals []types.PrivValidator) ([]*types.Block, error) {
blockID := test.MakeBlockID()
blocks := make([]*types.Block, n)

for i := 0; i < n; i++ {
height := state.LastBlockHeight + 1 + int64(i)
lastCommit, err := test.MakeCommit(blockID, height-1, 0, state.LastValidators, privVals, state.ChainID, state.LastBlockTime)
if err != nil {
return nil, err
}
block := state.MakeBlock(height, test.MakeNTxs(height, 10), lastCommit, nil, state.LastValidators.Proposer.Address)
blocks[i] = block
state.LastBlockID = blockID
state.LastBlockHeight = height
state.LastBlockTime = state.LastBlockTime.Add(1 * time.Second)
state.LastValidators = state.Validators.Copy()
state.Validators = state.NextValidators.Copy()
state.NextValidators = state.NextValidators.CopyIncrementProposerPriority(1)
state.AppHash = test.RandomHash()

blockID = test.MakeBlockIDWithHash(block.Hash())
}

return blocks, nil
}

func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) {
// 1. Initialize CometBFT and commit 3 blocks with the following app hashes:
// - 0x01
Expand All @@ -922,7 +948,7 @@ func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) {
genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile())
state.LastValidators = state.Validators.Copy()
// mode = 0 for committing all the blocks
blocks, err := test.MakeBlocks(3, state, []types.PrivValidator{privVal})
blocks, err := makeBlocks(3, state, []types.PrivValidator{privVal})
require.NoError(t, err)

store.chain = blocks
Expand Down
31 changes: 0 additions & 31 deletions internal/test/block.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (

"github.com/cometbft/cometbft/crypto"
"github.com/cometbft/cometbft/crypto/tmhash"
sm "github.com/cometbft/cometbft/state"
"github.com/cometbft/cometbft/types"
"github.com/cometbft/cometbft/version"
)
Expand Down Expand Up @@ -91,33 +90,3 @@ func MakeHeader(t *testing.T, h *types.Header) *types.Header {

return h
}

func MakeBlock(state sm.State) *types.Block {
return state.MakeBlock(state.LastBlockHeight+1, MakeNTxs(state.LastBlockHeight+1, 10), new(types.Commit), nil, state.NextValidators.Proposer.Address)
}

func MakeBlocks(n int, state sm.State, privVals []types.PrivValidator) ([]*types.Block, error) {
blockID := MakeBlockID()
blocks := make([]*types.Block, n)

for i := 0; i < n; i++ {
height := state.LastBlockHeight + 1 + int64(i)
lastCommit, err := MakeCommit(blockID, height-1, 0, state.LastValidators, privVals, state.ChainID, state.LastBlockTime)
if err != nil {
return nil, err
}
block := state.MakeBlock(height, MakeNTxs(height, 10), lastCommit, nil, state.LastValidators.Proposer.Address)
blocks[i] = block
state.LastBlockID = blockID
state.LastBlockHeight = height
state.LastBlockTime = state.LastBlockTime.Add(1 * time.Second)
state.LastValidators = state.Validators.Copy()
state.Validators = state.NextValidators.Copy()
state.NextValidators = state.NextValidators.CopyIncrementProposerPriority(1)
state.AppHash = RandomHash()

blockID = MakeBlockIDWithHash(block.Hash())
}

return blocks, nil
}
5 changes: 0 additions & 5 deletions internal/test/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"time"

cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
sm "github.com/cometbft/cometbft/state"
"github.com/cometbft/cometbft/types"
)

Expand Down Expand Up @@ -40,10 +39,6 @@ func MakeCommitFromVoteSet(blockID types.BlockID, voteSet *types.VoteSet, valida
return voteSet.MakeCommit(), nil
}

func MakeVoteSet(lastState sm.State, round int32) *types.VoteSet {
return types.NewVoteSet(lastState.ChainID, lastState.LastBlockHeight+1, round, cmtproto.PrecommitType, lastState.NextValidators)
}

func MakeCommit(blockID types.BlockID, height int64, round int32, valSet *types.ValidatorSet, privVals []types.PrivValidator, chainID string, now time.Time) (*types.Commit, error) {
sigs := make([]types.CommitSig, len(valSet.Validators))
for i := 0; i < len(valSet.Validators); i++ {
Expand Down
Loading
0