[v11/forgejo] fix: skip empty tokens in SearchOptions.Tokens() (#8412)
Some checks failed
/ release (push) Has been cancelled
testing / backend-checks (push) Has been cancelled
testing / frontend-checks (push) Has been cancelled
testing / test-unit (push) Has been cancelled
testing / test-e2e (push) Has been cancelled
testing / test-remote-cacher (redis) (push) Has been cancelled
testing / test-remote-cacher (valkey) (push) Has been cancelled
testing / test-remote-cacher (garnet) (push) Has been cancelled
testing / test-remote-cacher (redict) (push) Has been cancelled
testing / test-mysql (push) Has been cancelled
testing / test-pgsql (push) Has been cancelled
testing / test-sqlite (push) Has been cancelled
testing / security-check (push) Has been cancelled

backport of #8261 to v11

Co-authored-by: Danko Aleksejevs <danko@very.lv>
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/8412
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Shiny Nematoda <snematoda@noreply.codeberg.org>
Co-committed-by: Shiny Nematoda <snematoda@noreply.codeberg.org>
This commit is contained in:
Shiny Nematoda 2025-07-06 10:42:45 +02:00 committed by Earl Warren
parent 0dc2bed2dd
commit 86b6553f3a
5 changed files with 146 additions and 16 deletions

View file

@ -155,11 +155,12 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error {
func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) {
var queries []query.Query var queries []query.Query
if options.Keyword != "" { tokens, err := options.Tokens()
tokens, err := options.Tokens() if err != nil {
if err != nil { return nil, err
return nil, err }
}
if len(tokens) > 0 {
q := bleve.NewBooleanQuery() q := bleve.NewBooleanQuery()
for _, token := range tokens { for _, token := range tokens {
innerQ := bleve.NewDisjunctionQuery( innerQ := bleve.NewDisjunctionQuery(

View file

@ -148,12 +148,13 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error {
func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) {
query := elastic.NewBoolQuery() query := elastic.NewBoolQuery()
if options.Keyword != "" { tokens, err := options.Tokens()
if err != nil {
return nil, err
}
if len(tokens) > 0 {
q := elastic.NewBoolQuery() q := elastic.NewBoolQuery()
tokens, err := options.Tokens()
if err != nil {
return nil, err
}
for _, token := range tokens { for _, token := range tokens {
innerQ := elastic.NewMultiMatchQuery(token.Term, "title", "content", "comments") innerQ := elastic.NewMultiMatchQuery(token.Term, "title", "content", "comments")
if token.Fuzzy { if token.Fuzzy {

View file

@ -36,12 +36,9 @@ func (t *Tokenizer) next() (tk Token, err error) {
// skip all leading white space // skip all leading white space
for { for {
if r, _, err = t.in.ReadRune(); err == nil && r == ' ' { if r, _, err = t.in.ReadRune(); err != nil || r != ' ' {
//nolint:staticcheck,wastedassign // SA4006 the variable is used after the loop break
r, _, err = t.in.ReadRune()
continue
} }
break
} }
if err != nil { if err != nil {
return tk, err return tk, err
@ -98,11 +95,17 @@ nextEnd:
// Tokenize the keyword // Tokenize the keyword
func (o *SearchOptions) Tokens() (tokens []Token, err error) { func (o *SearchOptions) Tokens() (tokens []Token, err error) {
if o.Keyword == "" {
return nil, nil
}
in := strings.NewReader(o.Keyword) in := strings.NewReader(o.Keyword)
it := Tokenizer{in: in} it := Tokenizer{in: in}
for token, err := it.next(); err == nil; token, err = it.next() { for token, err := it.next(); err == nil; token, err = it.next() {
tokens = append(tokens, token) if token.Term != "" {
tokens = append(tokens, token)
}
} }
if err != nil && err != io.EOF { if err != nil && err != io.EOF {
return nil, err return nil, err

View file

@ -41,6 +41,36 @@ var testOpts = []testIssueQueryStringOpt{
}, },
}, },
}, },
{
Keyword: "Hello World",
Results: []Token{
{
Term: "Hello",
Fuzzy: true,
Kind: BoolOptShould,
},
{
Term: "World",
Fuzzy: true,
Kind: BoolOptShould,
},
},
},
{
Keyword: " Hello World ",
Results: []Token{
{
Term: "Hello",
Fuzzy: true,
Kind: BoolOptShould,
},
{
Term: "World",
Fuzzy: true,
Kind: BoolOptShould,
},
},
},
{ {
Keyword: "+Hello +World", Keyword: "+Hello +World",
Results: []Token{ Results: []Token{
@ -156,6 +186,68 @@ var testOpts = []testIssueQueryStringOpt{
}, },
}, },
}, },
{
Keyword: "\\",
Results: nil,
},
{
Keyword: "\"",
Results: nil,
},
{
Keyword: "Hello \\",
Results: []Token{
{
Term: "Hello",
Fuzzy: true,
Kind: BoolOptShould,
},
},
},
{
Keyword: "\"\"",
Results: nil,
},
{
Keyword: "\" World \"",
Results: []Token{
{
Term: " World ",
Fuzzy: false,
Kind: BoolOptShould,
},
},
},
{
Keyword: "\"\" World \"\"",
Results: []Token{
{
Term: "World",
Fuzzy: true,
Kind: BoolOptShould,
},
},
},
{
Keyword: "Best \"Hello World\" Ever",
Results: []Token{
{
Term: "Best",
Fuzzy: true,
Kind: BoolOptShould,
},
{
Term: "Hello World",
Fuzzy: false,
Kind: BoolOptShould,
},
{
Term: "Ever",
Fuzzy: true,
Kind: BoolOptShould,
},
},
},
} }
func TestIssueQueryString(t *testing.T) { func TestIssueQueryString(t *testing.T) {

View file

@ -88,6 +88,11 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) {
} }
} }
func allResults(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Len(t, result.Hits, len(data))
assert.Equal(t, len(data), int(result.Total))
}
var cases = []*testIndexerCase{ var cases = []*testIndexerCase{
{ {
Name: "default", Name: "default",
@ -97,6 +102,34 @@ var cases = []*testIndexerCase{
assert.Equal(t, len(data), int(result.Total)) assert.Equal(t, len(data), int(result.Total))
}, },
}, },
{
Name: "empty keyword",
SearchOptions: &internal.SearchOptions{
Keyword: "",
},
Expected: allResults,
},
{
Name: "whitespace keyword",
SearchOptions: &internal.SearchOptions{
Keyword: " ",
},
Expected: allResults,
},
{
Name: "dangling slash in keyword",
SearchOptions: &internal.SearchOptions{
Keyword: "\\",
},
Expected: allResults,
},
{
Name: "dangling quote in keyword",
SearchOptions: &internal.SearchOptions{
Keyword: "\"",
},
Expected: allResults,
},
{ {
Name: "empty", Name: "empty",
SearchOptions: &internal.SearchOptions{ SearchOptions: &internal.SearchOptions{