From b7c5a1877634c188fd40e8b9aa40f5fb40778fbc Mon Sep 17 00:00:00 2001 From: Lockwarr Date: Fri, 5 Jul 2024 11:28:57 +0300 Subject: [PATCH 1/3] fix: querying events containing dash(-) issue 3401 fix: tests and docs --- docs/guides/app-dev/indexing-transactions.md | 1 + libs/pubsub/query/syntax/doc.go | 2 +- libs/pubsub/query/syntax/scanner.go | 2 +- libs/pubsub/query/syntax/syntax_test.go | 2 ++ 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/guides/app-dev/indexing-transactions.md b/docs/guides/app-dev/indexing-transactions.md index 10072d9ea70..4fac3628645 100644 --- a/docs/guides/app-dev/indexing-transactions.md +++ b/docs/guides/app-dev/indexing-transactions.md @@ -221,6 +221,7 @@ func (app *Application) FinalizeBlock(_ context.Context, req *types.FinalizeBloc If the indexer is not `null`, the transaction will be indexed. Each event is indexed using a composite key in the form of `{eventType}.{eventAttribute}={eventValue}`, e.g. `transfer.sender=bob`. +or `wasm-transfer.sender=bob`. ## Querying Transactions Events diff --git a/libs/pubsub/query/syntax/doc.go b/libs/pubsub/query/syntax/doc.go index b9fb1afede2..e64df08b799 100644 --- a/libs/pubsub/query/syntax/doc.go +++ b/libs/pubsub/query/syntax/doc.go @@ -17,7 +17,7 @@ // The lexical terms are defined here using RE2 regular expression notation: // // // The name of an event attribute (type.value) -// tag = #'\w+(\.\w+)*' +// tag = #`^[\w-]+(\.\w+)?$` // // // A datestamp (YYYY-MM-DD) // date = #'DATE \d{4}-\d{2}-\d{2}' diff --git a/libs/pubsub/query/syntax/scanner.go b/libs/pubsub/query/syntax/scanner.go index f8c0583f813..767afe1933d 100644 --- a/libs/pubsub/query/syntax/scanner.go +++ b/libs/pubsub/query/syntax/scanner.go @@ -303,7 +303,7 @@ func (s *Scanner) invalid(ch rune) error { func isDigit(r rune) bool { return '0' <= r && r <= '9' } func isTagRune(r rune) bool { - return r == '.' || r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) + return r == '.' || r == '_' || r == '-' || unicode.IsLetter(r) || unicode.IsDigit(r) } func isTimeRune(r rune) bool { diff --git a/libs/pubsub/query/syntax/syntax_test.go b/libs/pubsub/query/syntax/syntax_test.go index 29a85aa9ec6..4b1aa5e3213 100644 --- a/libs/pubsub/query/syntax/syntax_test.go +++ b/libs/pubsub/query/syntax/syntax_test.go @@ -25,6 +25,8 @@ func TestScanner(t *testing.T) { // Tags {`foo foo.bar`, []syntax.Token{syntax.TTag, syntax.TTag}}, + {`foo foo-foo.bar`, []syntax.Token{syntax.TTag, syntax.TTag}}, + {`foo foo._bar_bar`, []syntax.Token{syntax.TTag, syntax.TTag}}, // Strings (values) {` '' x 'x' 'x y'`, []syntax.Token{syntax.TString, syntax.TTag, syntax.TString, syntax.TString}}, From 2d212afb16654e75f3e59ced567c9b30dcd63e34 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 8 Jul 2024 10:49:41 +0400 Subject: [PATCH 2/3] add more docs and changelog --- .../3401-allow-dash-in-event-tags.md | 2 + docs/guides/app-dev/indexing-transactions.md | 38 +++++++++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 .changelog/unreleased/improvements/3401-allow-dash-in-event-tags.md diff --git a/.changelog/unreleased/improvements/3401-allow-dash-in-event-tags.md b/.changelog/unreleased/improvements/3401-allow-dash-in-event-tags.md new file mode 100644 index 00000000000..6de79f5e09a --- /dev/null +++ b/.changelog/unreleased/improvements/3401-allow-dash-in-event-tags.md @@ -0,0 +1,2 @@ +- `[libs/pubsub]` Allow dash (`-`) in event tags + ([\#3401](https://github.com/cometbft/cometbft/issues/3401)) diff --git a/docs/guides/app-dev/indexing-transactions.md b/docs/guides/app-dev/indexing-transactions.md index 4fac3628645..ecccc74daad 100644 --- a/docs/guides/app-dev/indexing-transactions.md +++ b/docs/guides/app-dev/indexing-transactions.md @@ -221,7 +221,6 @@ func (app *Application) FinalizeBlock(_ context.Context, req *types.FinalizeBloc If the indexer is not `null`, the transaction will be indexed. Each event is indexed using a composite key in the form of `{eventType}.{eventAttribute}={eventValue}`, e.g. `transfer.sender=bob`. -or `wasm-transfer.sender=bob`. ## Querying Transactions Events @@ -264,22 +263,39 @@ curl "localhost:26657/block_search?query=\"block.height > 10\"" ``` -Storing the event sequence was introduced in CometBFT 0.34.26. Before that, up until Tendermint Core 0.34.26, -the event sequence was not stored in the kvstore and events were stored only by height. That means that queries -returned blocks and transactions whose event attributes match within the height but can match across different -events on that height. -This behavior was fixed with CometBFT 0.34.26+. However, if the data was indexed with earlier versions of -Tendermint Core and not re-indexed, that data will be queried as if all the attributes within a height -occurred within the same event. +Storing the event sequence was introduced in CometBFT 0.34.26. Before that, up +until Tendermint Core 0.34.26, the event sequence was not stored in the kvstore +and events were stored only by height. That means that queries returned blocks +and transactions whose event attributes match within the height but can match +across different events on that height. + +This behavior was fixed with CometBFT 0.34.26+. However, if the data was +indexed with earlier versions of Tendermint Core and not re-indexed, that data +will be queried as if all the attributes within a height occurred within the +same event. ## Event attribute value types -Users can use anything as an event value. However, if the event attribute value is a number, the following needs to be taken into account: +Users can use anything as an event value. However, if the event attribute value +is a number, the following needs to be taken into account: - Negative numbers will not be properly retrieved when querying the indexer. -- Event values are converted to big floats (from the `big/math` package). The precision of the floating point number is set to the bit length -of the integer it is supposed to represent, so that there is no loss of information due to insufficient precision. This was not present before CometBFT v0.38.x and all float values were ignored. +- Event values are converted to big floats (from the `big/math` package). The + precision of the floating point number is set to the bit length of the + integer it is supposed to represent, so that there is no loss of information + due to insufficient precision. This was not present before CometBFT v0.38.x + and all float values were ignored. - As of CometBFT v0.38.x, queries can contain floating point numbers as well. - Note that comparing to floats can be imprecise with a high number of decimals. +## Event type and attribute key format + +An event type/attribute key is a string that can contain any Unicode letter or +digit, as well as the following characters: `.` (dot), `-` (dash), `_` +(underscore): + +``` +^[\w-]+(\.\w+)?$ +``` + [abci-events]: https://github.com/cometbft/cometbft/blob/main/spec/abci/abci++_basic_concepts.md#events From 5a045c8921c86bdc1d233677a547d75056cd1a53 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 8 Jul 2024 11:19:55 +0400 Subject: [PATCH 3/3] doesn't allow first rune in tag be `.` or `-` --- docs/guides/app-dev/indexing-transactions.md | 5 +++-- libs/pubsub/query/syntax/doc.go | 2 +- libs/pubsub/query/syntax/scanner.go | 6 +++++- libs/pubsub/query/syntax/syntax_test.go | 2 ++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/guides/app-dev/indexing-transactions.md b/docs/guides/app-dev/indexing-transactions.md index ecccc74daad..98a45c0ccd4 100644 --- a/docs/guides/app-dev/indexing-transactions.md +++ b/docs/guides/app-dev/indexing-transactions.md @@ -292,10 +292,11 @@ is a number, the following needs to be taken into account: An event type/attribute key is a string that can contain any Unicode letter or digit, as well as the following characters: `.` (dot), `-` (dash), `_` -(underscore): +(underscore). The event type/attribute key must not start with `-` (dash) or +`.` (dot). ``` -^[\w-]+(\.\w+)?$ +^[\w]+[\.-\w]?$ ``` [abci-events]: https://github.com/cometbft/cometbft/blob/main/spec/abci/abci++_basic_concepts.md#events diff --git a/libs/pubsub/query/syntax/doc.go b/libs/pubsub/query/syntax/doc.go index e64df08b799..e60423abfcc 100644 --- a/libs/pubsub/query/syntax/doc.go +++ b/libs/pubsub/query/syntax/doc.go @@ -17,7 +17,7 @@ // The lexical terms are defined here using RE2 regular expression notation: // // // The name of an event attribute (type.value) -// tag = #`^[\w-]+(\.\w+)?$` +// tag = #`^[\w]+[\.-\w]?$` // // // A datestamp (YYYY-MM-DD) // date = #'DATE \d{4}-\d{2}-\d{2}' diff --git a/libs/pubsub/query/syntax/scanner.go b/libs/pubsub/query/syntax/scanner.go index 767afe1933d..b0b9a433691 100644 --- a/libs/pubsub/query/syntax/scanner.go +++ b/libs/pubsub/query/syntax/scanner.go @@ -99,7 +99,7 @@ func (s *Scanner) Next() error { } if '0' <= ch && ch <= '9' { return s.scanNumber(ch) - } else if isTagRune(ch) { + } else if isFirstTagRune(ch) { return s.scanTagLike(ch) } switch ch { @@ -306,6 +306,10 @@ func isTagRune(r rune) bool { return r == '.' || r == '_' || r == '-' || unicode.IsLetter(r) || unicode.IsDigit(r) } +func isFirstTagRune(r rune) bool { + return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) +} + func isTimeRune(r rune) bool { return strings.ContainsRune("-T:+Z", r) || isDigit(r) } diff --git a/libs/pubsub/query/syntax/syntax_test.go b/libs/pubsub/query/syntax/syntax_test.go index 4b1aa5e3213..a097500ff7a 100644 --- a/libs/pubsub/query/syntax/syntax_test.go +++ b/libs/pubsub/query/syntax/syntax_test.go @@ -169,6 +169,8 @@ func TestParseValid(t *testing.T) { {"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true}, {"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false}, + + {"cosm-wasm.transfer_amount=100", true}, } for _, test := range tests {