1
0
Fork 0
forked from forgejo/forgejo

Upgrade blevesearch dependency to v2.0.1 (#14346)

* Upgrade blevesearch dependency to v2.0.1

* Update rupture to v1.0.0

* Fix test
This commit is contained in:
Lauris BH 2021-01-18 03:21:14 +02:00 committed by GitHub
parent 3aa53dc6bc
commit f5abe2f563
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
459 changed files with 7518 additions and 4211 deletions

View file

@ -0,0 +1,70 @@
// Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package analysis
import (
index "github.com/blevesearch/bleve_index_api"
)
func TokenFrequency(tokens TokenStream, arrayPositions []uint64, options index.FieldIndexingOptions) index.TokenFrequencies {
rv := make(map[string]*index.TokenFreq, len(tokens))
if options.IncludeTermVectors() {
tls := make([]index.TokenLocation, len(tokens))
tlNext := 0
for _, token := range tokens {
tls[tlNext] = index.TokenLocation{
ArrayPositions: arrayPositions,
Start: token.Start,
End: token.End,
Position: token.Position,
}
curr, ok := rv[string(token.Term)]
if ok {
curr.Locations = append(curr.Locations, &tls[tlNext])
} else {
curr = &index.TokenFreq{
Term: token.Term,
Locations: []*index.TokenLocation{&tls[tlNext]},
}
rv[string(token.Term)] = curr
}
if !options.SkipFreqNorm() {
curr.SetFrequency(curr.Frequency() + 1)
}
tlNext++
}
} else {
for _, token := range tokens {
curr, exists := rv[string(token.Term)]
if !exists {
curr = &index.TokenFreq{
Term: token.Term,
}
rv[string(token.Term)] = curr
}
if !options.SkipFreqNorm() {
curr.SetFrequency(curr.Frequency() + 1)
}
}
}
return rv
}