Upgrade blevesearch to v0.8.1 (#9177)

For #1441

a91b427b59

vendor/github.com/blevesearch/bleve/search/collector/topn.go

@@ -69,6 +69,7 @@ type TopNCollector struct {
 	lowestMatchOutsideResults *search.DocumentMatch
 	updateFieldVisitor        index.DocumentFieldTermVisitor
 	dvReader                  index.DocValueReader
+	searchAfter               *search.DocumentMatch
 }
 
 // CheckDoneEvery controls how frequently we check the context deadline
@@ -78,6 +79,21 @@ const CheckDoneEvery = uint64(1024)
 // skipping over the first 'skip' hits
 // ordering hits by the provided sort order
 func NewTopNCollector(size int, skip int, sort search.SortOrder) *TopNCollector {
+	return newTopNCollector(size, skip, sort)
+}
+
+// NewTopNCollector builds a collector to find the top 'size' hits
+// skipping over the first 'skip' hits
+// ordering hits by the provided sort order
+func NewTopNCollectorAfter(size int, sort search.SortOrder, after []string) *TopNCollector {
+	rv := newTopNCollector(size, 0, sort)
+	rv.searchAfter = &search.DocumentMatch{
+		Sort: after,
+	}
+	return rv
+}
+
+func newTopNCollector(size int, skip int, sort search.SortOrder) *TopNCollector {
 	hc := &TopNCollector{size: size, skip: skip, sort: sort}
 
 	// pre-allocate space on the store to avoid reslicing
@@ -141,6 +157,7 @@ func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher,
 	searchContext := &search.SearchContext{
 		DocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), len(hc.sort)),
 		Collector:         hc,
+		IndexReader:       reader,
 	}
 
 	hc.dvReader, err = reader.DocValueReader(hc.neededFields)
@@ -265,6 +282,19 @@ func MakeTopNDocumentMatchHandler(
 			if d == nil {
 				return nil
 			}
+
+			// support search after based pagination,
+			// if this hit is <= the search after sort key
+			// we should skip it
+			if hc.searchAfter != nil {
+				// exact sort order matches use hit number to break tie
+				// but we want to allow for exact match, so we pretend
+				hc.searchAfter.HitNumber = d.HitNumber
+				if hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, d, hc.searchAfter) <= 0 {
+					return nil
+				}
+			}
+
 			// optimization, we track lowest sorting hit already removed from heap
 			// with this one comparison, we can avoid all heap operations if
 			// this hit would have been added and then immediately removed

vendor/github.com/blevesearch/bleve/search/query/date_range.go

@@ -41,6 +41,14 @@ type BleveQueryTime struct {
 	time.Time
 }
 
+var MinRFC3339CompatibleTime time.Time
+var MaxRFC3339CompatibleTime time.Time
+
+func init() {
+	MinRFC3339CompatibleTime, _ = time.Parse(time.RFC3339, "1677-12-01T00:00:00Z")
+	MaxRFC3339CompatibleTime, _ = time.Parse(time.RFC3339, "2262-04-11T11:59:59Z")
+}
+
 func queryTimeFromString(t string) (time.Time, error) {
 	dateTimeParser, err := cache.DateTimeParserNamed(QueryDateTimeParser)
 	if err != nil {
@@ -143,10 +151,20 @@ func (q *DateRangeQuery) parseEndpoints() (*float64, *float64, error) {
 	min := math.Inf(-1)
 	max := math.Inf(1)
 	if !q.Start.IsZero() {
-		min = numeric.Int64ToFloat64(q.Start.UnixNano())
+		if !isDatetimeCompatible(q.Start) {
+			// overflow
+			return nil, nil, fmt.Errorf("invalid/unsupported date range, start: %v", q.Start)
+		}
+		startInt64 := q.Start.UnixNano()
+		min = numeric.Int64ToFloat64(startInt64)
 	}
 	if !q.End.IsZero() {
-		max = numeric.Int64ToFloat64(q.End.UnixNano())
+		if !isDatetimeCompatible(q.End) {
+			// overflow
+			return nil, nil, fmt.Errorf("invalid/unsupported date range, end: %v", q.End)
+		}
+		endInt64 := q.End.UnixNano()
+		max = numeric.Int64ToFloat64(endInt64)
 	}
 
 	return &min, &max, nil
@@ -162,3 +180,12 @@ func (q *DateRangeQuery) Validate() error {
 	}
 	return nil
 }
+
+func isDatetimeCompatible(t BleveQueryTime) bool {
+	if QueryDateTimeFormat == time.RFC3339 &&
+		(t.Before(MinRFC3339CompatibleTime) || t.After(MaxRFC3339CompatibleTime)) {
+		return false
+	}
+
+	return true
+}

vendor/github.com/blevesearch/bleve/search/query/disjunction.go

@@ -80,12 +80,6 @@ func (q *DisjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping,
 
 	if len(ss) < 1 {
 		return searcher.NewMatchNoneSearcher(i)
-	} else if len(ss) == 1 && int(q.Min) == ss[0].Min() {
-		// apply optimization only if both conditions below are satisfied:
-		// - disjunction searcher has only 1 child searcher
-		// - parent searcher's min setting is equal to child searcher's min
-
-		return ss[0], nil
 	}
 
 	return searcher.NewDisjunctionSearcher(i, ss, q.Min, options)

vendor/github.com/blevesearch/bleve/search/query/geo_boundingpolygon.go

@@ -0,0 +1,94 @@
+//  Copyright (c) 2019 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package query
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/blevesearch/bleve/geo"
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/mapping"
+	"github.com/blevesearch/bleve/search"
+	"github.com/blevesearch/bleve/search/searcher"
+)
+
+type GeoBoundingPolygonQuery struct {
+	Points   []geo.Point `json:"polygon_points"`
+	FieldVal string      `json:"field,omitempty"`
+	BoostVal *Boost      `json:"boost,omitempty"`
+}
+
+func NewGeoBoundingPolygonQuery(points []geo.Point) *GeoBoundingPolygonQuery {
+	return &GeoBoundingPolygonQuery{
+		Points: points}
+}
+
+func (q *GeoBoundingPolygonQuery) SetBoost(b float64) {
+	boost := Boost(b)
+	q.BoostVal = &boost
+}
+
+func (q *GeoBoundingPolygonQuery) Boost() float64 {
+	return q.BoostVal.Value()
+}
+
+func (q *GeoBoundingPolygonQuery) SetField(f string) {
+	q.FieldVal = f
+}
+
+func (q *GeoBoundingPolygonQuery) Field() string {
+	return q.FieldVal
+}
+
+func (q *GeoBoundingPolygonQuery) Searcher(i index.IndexReader,
+	m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
+	field := q.FieldVal
+	if q.FieldVal == "" {
+		field = m.DefaultSearchField()
+	}
+
+	return searcher.NewGeoBoundedPolygonSearcher(i, q.Points, field, q.BoostVal.Value(), options)
+}
+
+func (q *GeoBoundingPolygonQuery) Validate() error {
+	return nil
+}
+
+func (q *GeoBoundingPolygonQuery) UnmarshalJSON(data []byte) error {
+	tmp := struct {
+		Points   []interface{} `json:"polygon_points"`
+		FieldVal string        `json:"field,omitempty"`
+		BoostVal *Boost        `json:"boost,omitempty"`
+	}{}
+	err := json.Unmarshal(data, &tmp)
+	if err != nil {
+		return err
+	}
+
+	q.Points = make([]geo.Point, 0, len(tmp.Points))
+	for _, i := range tmp.Points {
+		// now use our generic point parsing code from the geo package
+		lon, lat, found := geo.ExtractGeoPoint(i)
+		if !found {
+			return fmt.Errorf("geo polygon point: %v is not in a valid format", i)
+		}
+		q.Points = append(q.Points, geo.Point{Lon: lon, Lat: lat})
+	}
+
+	q.FieldVal = tmp.FieldVal
+	q.BoostVal = tmp.BoostVal
+	return nil
+}

vendor/github.com/blevesearch/bleve/search/query/query.go

@@ -273,6 +273,15 @@ func ParseQuery(input []byte) (Query, error) {
 		}
 		return &rv, nil
 	}
+	_, hasPoints := tmp["polygon_points"]
+	if hasPoints {
+		var rv GeoBoundingPolygonQuery
+		err := json.Unmarshal(input, &rv)
+		if err != nil {
+			return nil, err
+		}
+		return &rv, nil
+	}
 	return nil, fmt.Errorf("unknown query type")
 }
 

vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go

@@ -40,6 +40,7 @@ type TermQueryScorer struct {
 	idf                    float64
 	options                search.SearcherOptions
 	idfExplanation         *search.Explanation
+	includeScore           bool
 	queryNorm              float64
 	queryWeight            float64
 	queryWeightExplanation *search.Explanation
@@ -62,14 +63,15 @@ func (s *TermQueryScorer) Size() int {
 
 func NewTermQueryScorer(queryTerm []byte, queryField string, queryBoost float64, docTotal, docTerm uint64, options search.SearcherOptions) *TermQueryScorer {
 	rv := TermQueryScorer{
-		queryTerm:   string(queryTerm),
-		queryField:  queryField,
-		queryBoost:  queryBoost,
-		docTerm:     docTerm,
-		docTotal:    docTotal,
-		idf:         1.0 + math.Log(float64(docTotal)/float64(docTerm+1.0)),
-		options:     options,
-		queryWeight: 1.0,
+		queryTerm:    string(queryTerm),
+		queryField:   queryField,
+		queryBoost:   queryBoost,
+		docTerm:      docTerm,
+		docTotal:     docTotal,
+		idf:          1.0 + math.Log(float64(docTotal)/float64(docTerm+1.0)),
+		options:      options,
+		queryWeight:  1.0,
+		includeScore: options.Score != "none",
 	}
 
 	if options.Explain {
@@ -113,56 +115,61 @@ func (s *TermQueryScorer) SetQueryNorm(qnorm float64) {
 }
 
 func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.TermFieldDoc) *search.DocumentMatch {
-	var scoreExplanation *search.Explanation
-
-	// need to compute score
-	var tf float64
-	if termMatch.Freq < MaxSqrtCache {
-		tf = SqrtCache[int(termMatch.Freq)]
-	} else {
-		tf = math.Sqrt(float64(termMatch.Freq))
-	}
-	score := tf * termMatch.Norm * s.idf
-
-	if s.options.Explain {
-		childrenExplanations := make([]*search.Explanation, 3)
-		childrenExplanations[0] = &search.Explanation{
-			Value:   tf,
-			Message: fmt.Sprintf("tf(termFreq(%s:%s)=%d", s.queryField, s.queryTerm, termMatch.Freq),
+	rv := ctx.DocumentMatchPool.Get()
+	// perform any score computations only when needed
+	if s.includeScore || s.options.Explain {
+		var scoreExplanation *search.Explanation
+		var tf float64
+		if termMatch.Freq < MaxSqrtCache {
+			tf = SqrtCache[int(termMatch.Freq)]
+		} else {
+			tf = math.Sqrt(float64(termMatch.Freq))
 		}
-		childrenExplanations[1] = &search.Explanation{
-			Value:   termMatch.Norm,
-			Message: fmt.Sprintf("fieldNorm(field=%s, doc=%s)", s.queryField, termMatch.ID),
-		}
-		childrenExplanations[2] = s.idfExplanation
-		scoreExplanation = &search.Explanation{
-			Value:    score,
-			Message:  fmt.Sprintf("fieldWeight(%s:%s in %s), product of:", s.queryField, s.queryTerm, termMatch.ID),
-			Children: childrenExplanations,
-		}
-	}
+		score := tf * termMatch.Norm * s.idf
 
-	// if the query weight isn't 1, multiply
-	if s.queryWeight != 1.0 {
-		score = score * s.queryWeight
 		if s.options.Explain {
-			childExplanations := make([]*search.Explanation, 2)
-			childExplanations[0] = s.queryWeightExplanation
-			childExplanations[1] = scoreExplanation
+			childrenExplanations := make([]*search.Explanation, 3)
+			childrenExplanations[0] = &search.Explanation{
+				Value:   tf,
+				Message: fmt.Sprintf("tf(termFreq(%s:%s)=%d", s.queryField, s.queryTerm, termMatch.Freq),
+			}
+			childrenExplanations[1] = &search.Explanation{
+				Value:   termMatch.Norm,
+				Message: fmt.Sprintf("fieldNorm(field=%s, doc=%s)", s.queryField, termMatch.ID),
+			}
+			childrenExplanations[2] = s.idfExplanation
 			scoreExplanation = &search.Explanation{
 				Value:    score,
-				Message:  fmt.Sprintf("weight(%s:%s^%f in %s), product of:", s.queryField, s.queryTerm, s.queryBoost, termMatch.ID),
-				Children: childExplanations,
+				Message:  fmt.Sprintf("fieldWeight(%s:%s in %s), product of:", s.queryField, s.queryTerm, termMatch.ID),
+				Children: childrenExplanations,
 			}
 		}
+
+		// if the query weight isn't 1, multiply
+		if s.queryWeight != 1.0 {
+			score = score * s.queryWeight
+			if s.options.Explain {
+				childExplanations := make([]*search.Explanation, 2)
+				childExplanations[0] = s.queryWeightExplanation
+				childExplanations[1] = scoreExplanation
+				scoreExplanation = &search.Explanation{
+					Value:    score,
+					Message:  fmt.Sprintf("weight(%s:%s^%f in %s), product of:", s.queryField, s.queryTerm, s.queryBoost, termMatch.ID),
+					Children: childExplanations,
+				}
+			}
+		}
+
+		if s.includeScore {
+			rv.Score = score
+		}
+
+		if s.options.Explain {
+			rv.Expl = scoreExplanation
+		}
 	}
 
-	rv := ctx.DocumentMatchPool.Get()
 	rv.IndexInternalID = append(rv.IndexInternalID, termMatch.ID...)
-	rv.Score = score
-	if s.options.Explain {
-		rv.Expl = scoreExplanation
-	}
 
 	if len(termMatch.Vectors) > 0 {
 		if cap(rv.FieldTermLocations) < len(termMatch.Vectors) {

vendor/github.com/blevesearch/bleve/search/search.go

@@ -17,6 +17,7 @@ package search
 import (
 	"fmt"
 	"reflect"
+	"sort"
 
 	"github.com/blevesearch/bleve/index"
 	"github.com/blevesearch/bleve/size"
@@ -49,6 +50,24 @@ func (ap ArrayPositions) Equals(other ArrayPositions) bool {
 	return true
 }
 
+func (ap ArrayPositions) Compare(other ArrayPositions) int {
+	for i, p := range ap {
+		if i >= len(other) {
+			return 1
+		}
+		if p < other[i] {
+			return -1
+		}
+		if p > other[i] {
+			return 1
+		}
+	}
+	if len(ap) < len(other) {
+		return -1
+	}
+	return 0
+}
+
 type Location struct {
 	// Pos is the position of the term within the field, starting at 1
 	Pos uint64 `json:"pos"`
@@ -68,6 +87,46 @@ func (l *Location) Size() int {
 
 type Locations []*Location
 
+func (p Locations) Len() int      { return len(p) }
+func (p Locations) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+func (p Locations) Less(i, j int) bool {
+	c := p[i].ArrayPositions.Compare(p[j].ArrayPositions)
+	if c < 0 {
+		return true
+	}
+	if c > 0 {
+		return false
+	}
+	return p[i].Pos < p[j].Pos
+}
+
+func (p Locations) Dedupe() Locations { // destructive!
+	if len(p) <= 1 {
+		return p
+	}
+
+	sort.Sort(p)
+
+	slow := 0
+
+	for _, pfast := range p {
+		pslow := p[slow]
+		if pslow.Pos == pfast.Pos &&
+			pslow.Start == pfast.Start &&
+			pslow.End == pfast.End &&
+			pslow.ArrayPositions.Equals(pfast.ArrayPositions) {
+			continue // duplicate, so only move fast ahead
+		}
+
+		slow++
+
+		p[slow] = pfast
+	}
+
+	return p[:slow+1]
+}
+
 type TermLocationMap map[string]Locations
 
 func (t TermLocationMap) AddLocation(term string, location *Location) {
@@ -208,6 +267,7 @@ func (dm *DocumentMatch) Complete(prealloc []Location) []Location {
 
 		var lastField string
 		var tlm TermLocationMap
+		var needsDedupe bool
 
 		for i, ftl := range dm.FieldTermLocations {
 			if lastField != ftl.Field {
@@ -231,7 +291,19 @@ func (dm *DocumentMatch) Complete(prealloc []Location) []Location {
 				loc.ArrayPositions = append(ArrayPositions(nil), loc.ArrayPositions...)
 			}
 
-			tlm[ftl.Term] = append(tlm[ftl.Term], loc)
+			locs := tlm[ftl.Term]
+
+			// if the loc is before or at the last location, then there
+			// might be duplicates that need to be deduplicated
+			if !needsDedupe && len(locs) > 0 {
+				last := locs[len(locs)-1]
+				cmp := loc.ArrayPositions.Compare(last.ArrayPositions)
+				if cmp < 0 || (cmp == 0 && loc.Pos <= last.Pos) {
+					needsDedupe = true
+				}
+			}
+
+			tlm[ftl.Term] = append(locs, loc)
 
 			dm.FieldTermLocations[i] = FieldTermLocation{ // recycle
 				Location: Location{
@@ -239,6 +311,14 @@ func (dm *DocumentMatch) Complete(prealloc []Location) []Location {
 				},
 			}
 		}
+
+		if needsDedupe {
+			for _, tlm := range dm.Locations {
+				for term, locs := range tlm {
+					tlm[term] = locs.Dedupe()
+				}
+			}
+		}
 	}
 
 	dm.FieldTermLocations = dm.FieldTermLocations[:0] // recycle
@@ -279,6 +359,7 @@ type SearcherOptions struct {
 type SearchContext struct {
 	DocumentMatchPool *DocumentMatchPool
 	Collector         Collector
+	IndexReader       index.IndexReader
 }
 
 func (sc *SearchContext) Size() int {

vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go

@@ -45,6 +45,7 @@ type BooleanSearcher struct {
 	scorer          *scorer.ConjunctionQueryScorer
 	matches         []*search.DocumentMatch
 	initialized     bool
+	done            bool
 }
 
 func NewBooleanSearcher(indexReader index.IndexReader, mustSearcher search.Searcher, shouldSearcher search.Searcher, mustNotSearcher search.Searcher, options search.SearcherOptions) (*BooleanSearcher, error) {
@@ -207,6 +208,10 @@ func (s *BooleanSearcher) SetQueryNorm(qnorm float64) {
 
 func (s *BooleanSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
 
+	if s.done {
+		return nil, nil
+	}
+
 	if !s.initialized {
 		err := s.initSearchers(ctx)
 		if err != nil {
@@ -320,11 +325,19 @@ func (s *BooleanSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch
 		}
 	}
 
+	if rv == nil {
+		s.done = true
+	}
+
 	return rv, nil
 }
 
 func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
 
+	if s.done {
+		return nil, nil
+	}
+
 	if !s.initialized {
 		err := s.initSearchers(ctx)
 		if err != nil {
@@ -332,14 +345,8 @@ func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInter
 		}
 	}
 
-	// Advance the searchers only if the currentID cursor is trailing the lookup ID,
-	// additionally if the mustNotSearcher has been initialized, ensure that the
-	// cursor used to track the mustNotSearcher (currMustNot, which isn't tracked by
-	// currentID) is trailing the lookup ID as well - for in the case where currentID
-	// is nil and currMustNot is already at or ahead of the lookup ID, we MUST NOT
-	// advance the currentID or the currMustNot cursors.
-	if (s.currentID == nil || s.currentID.Compare(ID) < 0) &&
-		(s.currMustNot == nil || s.currMustNot.IndexInternalID.Compare(ID) < 0) {
+	// Advance the searcher only if the cursor is trailing the lookup ID
+	if s.currentID == nil || s.currentID.Compare(ID) < 0 {
 		var err error
 		if s.mustSearcher != nil {
 			if s.currMust != nil {
@@ -362,12 +369,17 @@ func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInter
 		}
 
 		if s.mustNotSearcher != nil {
-			if s.currMustNot != nil {
-				ctx.DocumentMatchPool.Put(s.currMustNot)
-			}
-			s.currMustNot, err = s.mustNotSearcher.Advance(ctx, ID)
-			if err != nil {
-				return nil, err
+			// Additional check for mustNotSearcher, whose cursor isn't tracked by
+			// currentID to prevent it from moving when the searcher's tracked
+			// position is already ahead of or at the requested ID.
+			if s.currMustNot == nil || s.currMustNot.IndexInternalID.Compare(ID) < 0 {
+				if s.currMustNot != nil {
+					ctx.DocumentMatchPool.Put(s.currMustNot)
+				}
+				s.currMustNot, err = s.mustNotSearcher.Advance(ctx, ID)
+				if err != nil {
+					return nil, err
+				}
 			}
 		}
 

vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go

@@ -22,6 +22,11 @@ import (
 	"github.com/blevesearch/bleve/search"
 )
 
+type filterFunc func(key []byte) bool
+
+var GeoBitsShift1 = (geo.GeoBits << 1)
+var GeoBitsShift1Minus1 = GeoBitsShift1 - 1
+
 func NewGeoBoundingBoxSearcher(indexReader index.IndexReader, minLon, minLat,
 	maxLon, maxLat float64, field string, boost float64,
 	options search.SearcherOptions, checkBoundaries bool) (
@@ -36,8 +41,11 @@ func NewGeoBoundingBoxSearcher(indexReader index.IndexReader, minLon, minLat,
 	}
 
 	// do math to produce list of terms needed for this search
-	onBoundaryTerms, notOnBoundaryTerms := ComputeGeoRange(0, (geo.GeoBits<<1)-1,
-		minLon, minLat, maxLon, maxLat, checkBoundaries)
+	onBoundaryTerms, notOnBoundaryTerms, err := ComputeGeoRange(0, GeoBitsShift1Minus1,
+		minLon, minLat, maxLon, maxLat, checkBoundaries, indexReader, field)
+	if err != nil {
+		return nil, err
+	}
 
 	var onBoundarySearcher search.Searcher
 	dvReader, err := indexReader.DocValueReader([]string{field})
@@ -94,59 +102,123 @@ var geoMaxShift = document.GeoPrecisionStep * 4
 var geoDetailLevel = ((geo.GeoBits << 1) - geoMaxShift) / 2
 
 func ComputeGeoRange(term uint64, shift uint,
-	sminLon, sminLat, smaxLon, smaxLat float64,
-	checkBoundaries bool) (
-	onBoundary [][]byte, notOnBoundary [][]byte) {
-	split := term | uint64(0x1)<<shift
-	var upperMax uint64
-	if shift < 63 {
-		upperMax = term | ((uint64(1) << (shift + 1)) - 1)
-	} else {
-		upperMax = 0xffffffffffffffff
-	}
-	lowerMax := split - 1
-	onBoundary, notOnBoundary = relateAndRecurse(term, lowerMax, shift,
-		sminLon, sminLat, smaxLon, smaxLat, checkBoundaries)
-	plusOnBoundary, plusNotOnBoundary := relateAndRecurse(split, upperMax, shift,
-		sminLon, sminLat, smaxLon, smaxLat, checkBoundaries)
-	onBoundary = append(onBoundary, plusOnBoundary...)
-	notOnBoundary = append(notOnBoundary, plusNotOnBoundary...)
-	return
-}
+	sminLon, sminLat, smaxLon, smaxLat float64, checkBoundaries bool,
+	indexReader index.IndexReader, field string) (
+	onBoundary [][]byte, notOnBoundary [][]byte, err error) {
+	preallocBytesLen := 32
+	preallocBytes := make([]byte, preallocBytesLen)
 
-func relateAndRecurse(start, end uint64, res uint,
-	sminLon, sminLat, smaxLon, smaxLat float64,
-	checkBoundaries bool) (
-	onBoundary [][]byte, notOnBoundary [][]byte) {
-	minLon := geo.MortonUnhashLon(start)
-	minLat := geo.MortonUnhashLat(start)
-	maxLon := geo.MortonUnhashLon(end)
-	maxLat := geo.MortonUnhashLat(end)
-
-	level := ((geo.GeoBits << 1) - res) >> 1
-
-	within := res%document.GeoPrecisionStep == 0 &&
-		geo.RectWithin(minLon, minLat, maxLon, maxLat,
-			sminLon, sminLat, smaxLon, smaxLat)
-	if within || (level == geoDetailLevel &&
-		geo.RectIntersects(minLon, minLat, maxLon, maxLat,
-			sminLon, sminLat, smaxLon, smaxLat)) {
-		if !within && checkBoundaries {
-			return [][]byte{
-				numeric.MustNewPrefixCodedInt64(int64(start), res),
-			}, nil
+	makePrefixCoded := func(in int64, shift uint) (rv numeric.PrefixCoded) {
+		if len(preallocBytes) <= 0 {
+			preallocBytesLen = preallocBytesLen * 2
+			preallocBytes = make([]byte, preallocBytesLen)
 		}
-		return nil,
-			[][]byte{
-				numeric.MustNewPrefixCodedInt64(int64(start), res),
-			}
-	} else if level < geoDetailLevel &&
-		geo.RectIntersects(minLon, minLat, maxLon, maxLat,
-			sminLon, sminLat, smaxLon, smaxLat) {
-		return ComputeGeoRange(start, res-1, sminLon, sminLat, smaxLon, smaxLat,
-			checkBoundaries)
+
+		rv, preallocBytes, err =
+			numeric.NewPrefixCodedInt64Prealloc(in, shift, preallocBytes)
+
+		return rv
 	}
-	return nil, nil
+
+	var fieldDict index.FieldDictContains
+	var isIndexed filterFunc
+	if irr, ok := indexReader.(index.IndexReaderContains); ok {
+		fieldDict, err = irr.FieldDictContains(field)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		isIndexed = func(term []byte) bool {
+			found, err := fieldDict.Contains(term)
+			return err == nil && found
+		}
+	}
+
+	defer func() {
+		if fieldDict != nil {
+			if fd, ok := fieldDict.(index.FieldDict); ok {
+				cerr := fd.Close()
+				if cerr != nil {
+					err = cerr
+				}
+			}
+		}
+	}()
+
+	if isIndexed == nil {
+		isIndexed = func(term []byte) bool {
+			if indexReader != nil {
+				reader, err := indexReader.TermFieldReader(term, field, false, false, false)
+				if err != nil || reader == nil {
+					return false
+				}
+				if reader.Count() == 0 {
+					_ = reader.Close()
+					return false
+				}
+				_ = reader.Close()
+			}
+			return true
+		}
+	}
+
+	var computeGeoRange func(term uint64, shift uint) // declare for recursion
+
+	relateAndRecurse := func(start, end uint64, res, level uint) {
+		minLon := geo.MortonUnhashLon(start)
+		minLat := geo.MortonUnhashLat(start)
+		maxLon := geo.MortonUnhashLon(end)
+		maxLat := geo.MortonUnhashLat(end)
+
+		within := res%document.GeoPrecisionStep == 0 &&
+			geo.RectWithin(minLon, minLat, maxLon, maxLat,
+				sminLon, sminLat, smaxLon, smaxLat)
+		if within || (level == geoDetailLevel &&
+			geo.RectIntersects(minLon, minLat, maxLon, maxLat,
+				sminLon, sminLat, smaxLon, smaxLat)) {
+			codedTerm := makePrefixCoded(int64(start), res)
+			if isIndexed(codedTerm) {
+				if !within && checkBoundaries {
+					onBoundary = append(onBoundary, codedTerm)
+				} else {
+					notOnBoundary = append(notOnBoundary, codedTerm)
+				}
+			}
+		} else if level < geoDetailLevel &&
+			geo.RectIntersects(minLon, minLat, maxLon, maxLat,
+				sminLon, sminLat, smaxLon, smaxLat) {
+			computeGeoRange(start, res-1)
+		}
+	}
+
+	computeGeoRange = func(term uint64, shift uint) {
+		if err != nil {
+			return
+		}
+
+		split := term | uint64(0x1)<<shift
+		var upperMax uint64
+		if shift < 63 {
+			upperMax = term | ((uint64(1) << (shift + 1)) - 1)
+		} else {
+			upperMax = 0xffffffffffffffff
+		}
+
+		lowerMax := split - 1
+
+		level := (GeoBitsShift1 - shift) >> 1
+
+		relateAndRecurse(term, lowerMax, shift, level)
+		relateAndRecurse(split, upperMax, shift, level)
+	}
+
+	computeGeoRange(term, shift)
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return onBoundary, notOnBoundary, err
 }
 
 func buildRectFilter(dvReader index.DocValueReader, field string,

vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go

@@ -34,7 +34,7 @@ func NewGeoPointDistanceSearcher(indexReader index.IndexReader, centerLon,
 	// build a searcher for the box
 	boxSearcher, err := boxSearcher(indexReader,
 		topLeftLon, topLeftLat, bottomRightLon, bottomRightLat,
-		field, boost, options)
+		field, boost, options, false)
 	if err != nil {
 		return nil, err
 	}
@@ -54,19 +54,20 @@ func NewGeoPointDistanceSearcher(indexReader index.IndexReader, centerLon,
 // two boxes joined through a disjunction searcher
 func boxSearcher(indexReader index.IndexReader,
 	topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64,
-	field string, boost float64, options search.SearcherOptions) (
+	field string, boost float64, options search.SearcherOptions, checkBoundaries bool) (
 	search.Searcher, error) {
 	if bottomRightLon < topLeftLon {
 		// cross date line, rewrite as two parts
 
 		leftSearcher, err := NewGeoBoundingBoxSearcher(indexReader,
 			-180, bottomRightLat, bottomRightLon, topLeftLat,
-			field, boost, options, false)
+			field, boost, options, checkBoundaries)
 		if err != nil {
 			return nil, err
 		}
 		rightSearcher, err := NewGeoBoundingBoxSearcher(indexReader,
-			topLeftLon, bottomRightLat, 180, topLeftLat, field, boost, options, false)
+			topLeftLon, bottomRightLat, 180, topLeftLat, field, boost, options,
+			checkBoundaries)
 		if err != nil {
 			_ = leftSearcher.Close()
 			return nil, err
@@ -85,7 +86,7 @@ func boxSearcher(indexReader index.IndexReader,
 	// build geoboundinggox searcher for that bounding box
 	boxSearcher, err := NewGeoBoundingBoxSearcher(indexReader,
 		topLeftLon, bottomRightLat, bottomRightLon, topLeftLat, field, boost,
-		options, false)
+		options, checkBoundaries)
 	if err != nil {
 		return nil, err
 	}

vendor/github.com/blevesearch/bleve/search/searcher/search_geopolygon.go

@@ -0,0 +1,110 @@
+//  Copyright (c) 2019 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package searcher
+
+import (
+	"github.com/blevesearch/bleve/geo"
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/numeric"
+	"github.com/blevesearch/bleve/search"
+	"math"
+)
+
+func NewGeoBoundedPolygonSearcher(indexReader index.IndexReader,
+	polygon []geo.Point, field string, boost float64,
+	options search.SearcherOptions) (search.Searcher, error) {
+
+	// compute the bounding box enclosing the polygon
+	topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err :=
+		geo.BoundingRectangleForPolygon(polygon)
+	if err != nil {
+		return nil, err
+	}
+
+	// build a searcher for the bounding box on the polygon
+	boxSearcher, err := boxSearcher(indexReader,
+		topLeftLon, topLeftLat, bottomRightLon, bottomRightLat,
+		field, boost, options, true)
+	if err != nil {
+		return nil, err
+	}
+
+	dvReader, err := indexReader.DocValueReader([]string{field})
+	if err != nil {
+		return nil, err
+	}
+
+	// wrap it in a filtering searcher that checks for the polygon inclusivity
+	return NewFilteringSearcher(boxSearcher,
+		buildPolygonFilter(dvReader, field, polygon)), nil
+}
+
+const float64EqualityThreshold = 1e-6
+
+func almostEqual(a, b float64) bool {
+	return math.Abs(a-b) <= float64EqualityThreshold
+}
+
+// buildPolygonFilter returns true if the point lies inside the
+// polygon. It is based on the ray-casting technique as referred
+// here: https://wrf.ecse.rpi.edu/nikola/pubdetails/pnpoly.html
+func buildPolygonFilter(dvReader index.DocValueReader, field string,
+	polygon []geo.Point) FilterFunc {
+	return func(d *search.DocumentMatch) bool {
+		var lon, lat float64
+		var found bool
+
+		err := dvReader.VisitDocValues(d.IndexInternalID, func(field string, term []byte) {
+			// only consider the values which are shifted 0
+			prefixCoded := numeric.PrefixCoded(term)
+			shift, err := prefixCoded.Shift()
+			if err == nil && shift == 0 {
+				i64, err := prefixCoded.Int64()
+				if err == nil {
+					lon = geo.MortonUnhashLon(uint64(i64))
+					lat = geo.MortonUnhashLat(uint64(i64))
+					found = true
+				}
+			}
+		})
+
+		// Note: this approach works for points which are strictly inside
+		// the polygon. ie it might fail for certain points on the polygon boundaries.
+		if err == nil && found {
+			nVertices := len(polygon)
+			var inside bool
+			// check for a direct vertex match
+			if almostEqual(polygon[0].Lat, lat) &&
+				almostEqual(polygon[0].Lon, lon) {
+				return true
+			}
+
+			for i := 1; i < nVertices; i++ {
+				if almostEqual(polygon[i].Lat, lat) &&
+					almostEqual(polygon[i].Lon, lon) {
+					return true
+				}
+				if (polygon[i].Lat > lat) != (polygon[i-1].Lat > lat) &&
+					lon < (polygon[i-1].Lon-polygon[i].Lon)*(lat-polygon[i].Lat)/
+						(polygon[i-1].Lat-polygon[i].Lat)+polygon[i].Lon {
+					inside = !inside
+				}
+			}
+			return inside
+
+		}
+		return false
+	}
+}

vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go

@@ -53,20 +53,49 @@ func NewNumericRangeSearcher(indexReader index.IndexReader,
 	if !*inclusiveMax && maxInt64 != math.MinInt64 {
 		maxInt64--
 	}
+
+	var fieldDict index.FieldDictContains
+	var isIndexed filterFunc
+	var err error
+	if irr, ok := indexReader.(index.IndexReaderContains); ok {
+		fieldDict, err = irr.FieldDictContains(field)
+		if err != nil {
+			return nil, err
+		}
+
+		isIndexed = func(term []byte) bool {
+			found, err := fieldDict.Contains(term)
+			return err == nil && found
+		}
+	}
+
 	// FIXME hard-coded precision, should match field declaration
 	termRanges := splitInt64Range(minInt64, maxInt64, 4)
-	terms := termRanges.Enumerate()
+	terms := termRanges.Enumerate(isIndexed)
+	if fieldDict != nil {
+		if fd, ok := fieldDict.(index.FieldDict); ok {
+			cerr := fd.Close()
+			if cerr != nil {
+				err = cerr
+			}
+		}
+	}
+
 	if len(terms) < 1 {
 		// cannot return MatchNoneSearcher because of interaction with
 		// commit f391b991c20f02681bacd197afc6d8aed444e132
 		return NewMultiTermSearcherBytes(indexReader, terms, field, boost, options,
 			true)
 	}
-	var err error
-	terms, err = filterCandidateTerms(indexReader, terms, field)
-	if err != nil {
-		return nil, err
+
+	// for upside_down
+	if isIndexed == nil {
+		terms, err = filterCandidateTerms(indexReader, terms, field)
+		if err != nil {
+			return nil, err
+		}
 	}
+
 	if tooManyClauses(len(terms)) {
 		return nil, tooManyClausesErr(len(terms))
 	}
@@ -125,11 +154,17 @@ type termRange struct {
 	endTerm   []byte
 }
 
-func (t *termRange) Enumerate() [][]byte {
+func (t *termRange) Enumerate(filter filterFunc) [][]byte {
 	var rv [][]byte
 	next := t.startTerm
 	for bytes.Compare(next, t.endTerm) <= 0 {
-		rv = append(rv, next)
+		if filter != nil {
+			if filter(next) {
+				rv = append(rv, next)
+			}
+		} else {
+			rv = append(rv, next)
+		}
 		next = incrementBytes(next)
 	}
 	return rv
@@ -150,10 +185,10 @@ func incrementBytes(in []byte) []byte {
 
 type termRanges []*termRange
 
-func (tr termRanges) Enumerate() [][]byte {
+func (tr termRanges) Enumerate(filter filterFunc) [][]byte {
 	var rv [][]byte
 	for _, tri := range tr {
-		trie := tri.Enumerate()
+		trie := tri.Enumerate(filter)
 		rv = append(rv, trie...)
 	}
 	return rv

vendor/github.com/blevesearch/bleve/search/sort.go

@@ -38,6 +38,8 @@ type SearchSort interface {
 	RequiresScoring() bool
 	RequiresFields() []string
 
+	Reverse()
+
 	Copy() SearchSort
 }
 
@@ -293,6 +295,12 @@ func (so SortOrder) CacheDescending() []bool {
 	return rv
 }
 
+func (so SortOrder) Reverse() {
+	for _, soi := range so {
+		soi.Reverse()
+	}
+}
+
 // SortFieldType lets you control some internal sort behavior
 // normally leaving this to the zero-value of SortFieldAuto is fine
 type SortFieldType int
@@ -492,6 +500,15 @@ func (s *SortField) Copy() SearchSort {
 	return &rv
 }
 
+func (s *SortField) Reverse() {
+	s.Desc = !s.Desc
+	if s.Missing == SortFieldMissingFirst {
+		s.Missing = SortFieldMissingLast
+	} else {
+		s.Missing = SortFieldMissingFirst
+	}
+}
+
 // SortDocID will sort results by the document identifier
 type SortDocID struct {
 	Desc bool
@@ -533,6 +550,10 @@ func (s *SortDocID) Copy() SearchSort {
 	return &rv
 }
 
+func (s *SortDocID) Reverse() {
+	s.Desc = !s.Desc
+}
+
 // SortScore will sort results by the document match score
 type SortScore struct {
 	Desc bool
@@ -574,6 +595,10 @@ func (s *SortScore) Copy() SearchSort {
 	return &rv
 }
 
+func (s *SortScore) Reverse() {
+	s.Desc = !s.Desc
+}
+
 var maxDistance = string(numeric.MustNewPrefixCodedInt64(math.MaxInt64, 0))
 
 // NewSortGeoDistance creates SearchSort instance for sorting documents by
@@ -705,6 +730,10 @@ func (s *SortGeoDistance) Copy() SearchSort {
 	return &rv
 }
 
+func (s *SortGeoDistance) Reverse() {
+	s.Desc = !s.Desc
+}
+
 type BytesSlice [][]byte
 
 func (p BytesSlice) Len() int           { return len(p) }