Upgrade blevesearch to v0.8.1 (#9177)

For #1441

a91b427b59

vendor/github.com/couchbase/vellum/levenshtein/parametric_dfa.go

@@ -0,0 +1,349 @@
+//  Copyright (c) 2018 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package levenshtein
+
+import (
+	"crypto/md5"
+	"encoding/json"
+	"fmt"
+	"math"
+)
+
+type ParametricState struct {
+	shapeID uint32
+	offset  uint32
+}
+
+func newParametricState() ParametricState {
+	return ParametricState{}
+}
+
+func (ps *ParametricState) isDeadEnd() bool {
+	return ps.shapeID == 0
+}
+
+type Transition struct {
+	destShapeID uint32
+	deltaOffset uint32
+}
+
+func (t *Transition) apply(state ParametricState) ParametricState {
+	ps := ParametricState{
+		shapeID: t.destShapeID}
+	// don't need any offset if we are in the dead state,
+	// this ensures we have only one dead state.
+	if t.destShapeID != 0 {
+		ps.offset = state.offset + t.deltaOffset
+	}
+
+	return ps
+}
+
+type ParametricStateIndex struct {
+	stateIndex []uint32
+	stateQueue []ParametricState
+	numOffsets uint32
+}
+
+func newParametricStateIndex(queryLen,
+	numParamState uint32) ParametricStateIndex {
+	numOffsets := queryLen + 1
+	if numParamState == 0 {
+		numParamState = numOffsets
+	}
+	maxNumStates := numParamState * numOffsets
+	psi := ParametricStateIndex{
+		stateIndex: make([]uint32, maxNumStates),
+		stateQueue: make([]ParametricState, 0, 150),
+		numOffsets: numOffsets,
+	}
+
+	for i := uint32(0); i < maxNumStates; i++ {
+		psi.stateIndex[i] = math.MaxUint32
+	}
+	return psi
+}
+
+func (psi *ParametricStateIndex) numStates() int {
+	return len(psi.stateQueue)
+}
+
+func (psi *ParametricStateIndex) maxNumStates() int {
+	return len(psi.stateIndex)
+}
+
+func (psi *ParametricStateIndex) get(stateID uint32) ParametricState {
+	return psi.stateQueue[stateID]
+}
+
+func (psi *ParametricStateIndex) getOrAllocate(ps ParametricState) uint32 {
+	bucket := ps.shapeID*psi.numOffsets + ps.offset
+	if bucket < uint32(len(psi.stateIndex)) &&
+		psi.stateIndex[bucket] != math.MaxUint32 {
+		return psi.stateIndex[bucket]
+	}
+	nState := uint32(len(psi.stateQueue))
+	psi.stateQueue = append(psi.stateQueue, ps)
+
+	psi.stateIndex[bucket] = nState
+	return nState
+}
+
+type ParametricDFA struct {
+	distance         []uint8
+	transitions      []Transition
+	maxDistance      uint8
+	transitionStride uint32
+	diameter         uint32
+}
+
+func (pdfa *ParametricDFA) initialState() ParametricState {
+	return ParametricState{shapeID: 1}
+}
+
+// Returns true iff whatever characters come afterward,
+// we will never reach a shorter distance
+func (pdfa *ParametricDFA) isPrefixSink(state ParametricState, queryLen uint32) bool {
+	if state.isDeadEnd() {
+		return true
+	}
+
+	remOffset := queryLen - state.offset
+	if remOffset < pdfa.diameter {
+		stateDistances := pdfa.distance[pdfa.diameter*state.shapeID:]
+		prefixDistance := stateDistances[remOffset]
+		if prefixDistance > pdfa.maxDistance {
+			return false
+		}
+
+		for _, d := range stateDistances {
+			if d < prefixDistance {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+func (pdfa *ParametricDFA) numStates() int {
+	return len(pdfa.transitions) / int(pdfa.transitionStride)
+}
+
+func min(x, y uint32) uint32 {
+	if x < y {
+		return x
+	}
+	return y
+}
+
+func (pdfa *ParametricDFA) transition(state ParametricState,
+	chi uint32) Transition {
+	return pdfa.transitions[pdfa.transitionStride*state.shapeID+chi]
+}
+
+func (pdfa *ParametricDFA) getDistance(state ParametricState,
+	qLen uint32) Distance {
+	remainingOffset := qLen - state.offset
+	if state.isDeadEnd() || remainingOffset >= pdfa.diameter {
+		return Atleast{d: pdfa.maxDistance + 1}
+	}
+	dist := pdfa.distance[int(pdfa.diameter*state.shapeID)+int(remainingOffset)]
+	if dist > pdfa.maxDistance {
+		return Atleast{d: dist}
+	}
+	return Exact{d: dist}
+}
+
+func (pdfa *ParametricDFA) computeDistance(left, right string) Distance {
+	state := pdfa.initialState()
+	leftChars := []rune(left)
+	for _, chr := range []rune(right) {
+		start := state.offset
+		stop := min(start+pdfa.diameter, uint32(len(leftChars)))
+		chi := characteristicVector(leftChars[start:stop], chr)
+		transition := pdfa.transition(state, uint32(chi))
+		state = transition.apply(state)
+		if state.isDeadEnd() {
+			return Atleast{d: pdfa.maxDistance + 1}
+		}
+	}
+	return pdfa.getDistance(state, uint32(len(left)))
+}
+
+func (pdfa *ParametricDFA) buildDfa(query string, distance uint8,
+	prefix bool) (*DFA, error) {
+	qLen := uint32(len([]rune(query)))
+	alphabet := queryChars(query)
+
+	psi := newParametricStateIndex(qLen, uint32(pdfa.numStates()))
+	maxNumStates := psi.maxNumStates()
+	deadEndStateID := psi.getOrAllocate(newParametricState())
+	if deadEndStateID != 0 {
+		return nil, fmt.Errorf("Invalid dead end state")
+	}
+
+	initialStateID := psi.getOrAllocate(pdfa.initialState())
+	dfaBuilder := withMaxStates(uint32(maxNumStates))
+	mask := uint32((1 << pdfa.diameter) - 1)
+
+	var stateID int
+	for stateID = 0; stateID < StateLimit; stateID++ {
+		if stateID == psi.numStates() {
+			break
+		}
+		state := psi.get(uint32(stateID))
+		if prefix && pdfa.isPrefixSink(state, qLen) {
+			distance := pdfa.getDistance(state, qLen)
+			dfaBuilder.addState(uint32(stateID), uint32(stateID), distance)
+		} else {
+			transition := pdfa.transition(state, 0)
+			defSuccessor := transition.apply(state)
+			defSuccessorID := psi.getOrAllocate(defSuccessor)
+			distance := pdfa.getDistance(state, qLen)
+			stateBuilder, err := dfaBuilder.addState(uint32(stateID), defSuccessorID, distance)
+
+			if err != nil {
+				return nil, fmt.Errorf("parametric_dfa: buildDfa, err: %v", err)
+			}
+
+			alphabet.resetNext()
+			chr, cv, err := alphabet.next()
+			for err == nil {
+				chi := cv.shiftAndMask(state.offset, mask)
+
+				transition := pdfa.transition(state, chi)
+
+				destState := transition.apply(state)
+
+				destStateID := psi.getOrAllocate(destState)
+
+				stateBuilder.addTransition(chr, destStateID)
+
+				chr, cv, err = alphabet.next()
+			}
+		}
+	}
+
+	if stateID == StateLimit {
+		return nil, ErrTooManyStates
+	}
+
+	dfaBuilder.setInitialState(initialStateID)
+	return dfaBuilder.build(distance), nil
+}
+
+func fromNfa(nfa *LevenshteinNFA) (*ParametricDFA, error) {
+	lookUp := newHash()
+	lookUp.getOrAllocate(*newMultiState())
+	initialState := nfa.initialStates()
+	lookUp.getOrAllocate(*initialState)
+
+	maxDistance := nfa.maxDistance()
+	msDiameter := nfa.msDiameter()
+
+	numChi := 1 << msDiameter
+	chiValues := make([]uint64, numChi)
+	for i := 0; i < numChi; i++ {
+		chiValues[i] = uint64(i)
+	}
+
+	transitions := make([]Transition, 0, numChi*int(msDiameter))
+	var stateID int
+	for stateID = 0; stateID < StateLimit; stateID++ {
+		if stateID == len(lookUp.items) {
+			break
+		}
+
+		for _, chi := range chiValues {
+			destMs := newMultiState()
+
+			ms := lookUp.getFromID(stateID)
+
+			nfa.transition(ms, destMs, chi)
+
+			translation := destMs.normalize()
+
+			destID := lookUp.getOrAllocate(*destMs)
+
+			transitions = append(transitions, Transition{
+				destShapeID: uint32(destID),
+				deltaOffset: translation,
+			})
+		}
+	}
+
+	if stateID == StateLimit {
+		return nil, ErrTooManyStates
+	}
+
+	ns := len(lookUp.items)
+	diameter := int(msDiameter)
+
+	distances := make([]uint8, 0, diameter*ns)
+	for stateID := 0; stateID < ns; stateID++ {
+		ms := lookUp.getFromID(stateID)
+		for offset := 0; offset < diameter; offset++ {
+			dist := nfa.multistateDistance(ms, uint32(offset))
+			distances = append(distances, dist.distance())
+		}
+	}
+
+	return &ParametricDFA{
+		diameter:         uint32(msDiameter),
+		transitions:      transitions,
+		maxDistance:      maxDistance,
+		transitionStride: uint32(numChi),
+		distance:         distances,
+	}, nil
+}
+
+type hash struct {
+	index map[[16]byte]int
+	items []MultiState
+}
+
+func newHash() *hash {
+	return &hash{
+		index: make(map[[16]byte]int, 100),
+		items: make([]MultiState, 0, 100),
+	}
+}
+
+func (h *hash) getOrAllocate(m MultiState) int {
+	size := len(h.items)
+	var exists bool
+	var pos int
+	md5 := getHash(&m)
+	if pos, exists = h.index[md5]; !exists {
+		h.index[md5] = size
+		pos = size
+		h.items = append(h.items, m)
+	}
+	return pos
+}
+
+func (h *hash) getFromID(id int) *MultiState {
+	return &h.items[id]
+}
+
+func getHash(ms *MultiState) [16]byte {
+	msBytes := []byte{}
+	for _, state := range ms.states {
+		jsonBytes, _ := json.Marshal(&state)
+		msBytes = append(msBytes, jsonBytes...)
+	}
+	return md5.Sum(msBytes)
+}