Remove camel case tokenization from repo indexer (#7733)

* Make repository indexer tokenize by camel case selectable

* Revert "Make repository indexer tokenize by camel case selectable"

This reverts commit a8d4e40af56ffeab9c040a13e01b9e0d5487bd3b.

* Remove app.ini setting, permanently remove camel case tokenizer

* Increase repoIndexerLatestVersion to force indexes to be rebuilt

* Changes introduced by the commmand make vendor

modules/indexer/repo.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/blevesearch/bleve"
 	"github.com/blevesearch/bleve/analysis/analyzer/custom"
-	"github.com/blevesearch/bleve/analysis/token/camelcase"
 	"github.com/blevesearch/bleve/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/analysis/token/unique"
 	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
@@ -24,7 +23,7 @@ const (
 	repoIndexerAnalyzer = "repoIndexerAnalyzer"
 	repoIndexerDocType  = "repoIndexerDocType"
 
-	repoIndexerLatestVersion = 1
+	repoIndexerLatestVersion = 2
 )
 
 // repoIndexer (thread-safe) index for repository contents
@@ -111,7 +110,7 @@ func createRepoIndexer(path string, latestVersion int) error {
 		"type":          custom.Name,
 		"char_filters":  []string{},
 		"tokenizer":     unicode.Name,
-		"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name, unique.Name},
+		"token_filters": []string{unicodeNormalizeName, lowercase.Name, unique.Name},
 	}); err != nil {
 		return err
 	}

vendor/github.com/blevesearch/bleve/analysis/token/camelcase/camelcase.go

@@ -1,78 +0,0 @@
-//  Copyright (c) 2016 Couchbase, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// 		http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package camelcase
-
-import (
-	"bytes"
-	"unicode/utf8"
-
-	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/registry"
-)
-
-const Name = "camelCase"
-
-// CamelCaseFilter splits a given token into a set of tokens where each resulting token
-// falls into one the following classes:
-// 1) Upper case followed by lower case letters.
-//		Terminated by a number, an upper case letter, and a non alpha-numeric symbol.
-// 2) Upper case followed by upper case letters.
-//		Terminated by a number, an upper case followed by a lower case letter, and a non alpha-numeric symbol.
-// 3) Lower case followed by lower case letters.
-//		Terminated by a number, an upper case letter, and a non alpha-numeric symbol.
-// 4) Number followed by numbers.
-//		Terminated by a letter, and a non alpha-numeric symbol.
-// 5) Non alpha-numeric symbol followed by non alpha-numeric symbols.
-//		Terminated by a number, and a letter.
-//
-// It does a one-time sequential pass over an input token, from left to right.
-// The scan is greedy and generates the longest substring that fits into one of the classes.
-//
-// See the test file for examples of classes and their parsings.
-type CamelCaseFilter struct{}
-
-func NewCamelCaseFilter() *CamelCaseFilter {
-	return &CamelCaseFilter{}
-}
-
-func (f *CamelCaseFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
-	rv := make(analysis.TokenStream, 0, len(input))
-
-	nextPosition := 1
-	for _, token := range input {
-		runeCount := utf8.RuneCount(token.Term)
-		runes := bytes.Runes(token.Term)
-
-		p := NewParser(runeCount, nextPosition, token.Start)
-		for i := 0; i < runeCount; i++ {
-			if i+1 >= runeCount {
-				p.Push(runes[i], nil)
-			} else {
-				p.Push(runes[i], &runes[i+1])
-			}
-		}
-		rv = append(rv, p.FlushTokens()...)
-		nextPosition = p.NextPosition()
-	}
-	return rv
-}
-
-func CamelCaseFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
-	return NewCamelCaseFilter(), nil
-}
-
-func init() {
-	registry.RegisterTokenFilter(Name, CamelCaseFilterConstructor)
-}

vendor/github.com/blevesearch/bleve/analysis/token/camelcase/parser.go

@@ -1,109 +0,0 @@
-//  Copyright (c) 2016 Couchbase, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// 		http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package camelcase
-
-import (
-	"github.com/blevesearch/bleve/analysis"
-)
-
-func (p *Parser) buildTokenFromTerm(buffer []rune) *analysis.Token {
-	term := analysis.BuildTermFromRunes(buffer)
-	token := &analysis.Token{
-		Term:     term,
-		Position: p.position,
-		Start:    p.index,
-		End:      p.index + len(term),
-	}
-	p.position++
-	p.index += len(term)
-	return token
-}
-
-// Parser accepts a symbol and passes it to the current state (representing a class).
-// The state can accept it (and accumulate it). Otherwise, the parser creates a new state that
-// starts with the pushed symbol.
-//
-// Parser accumulates a new resulting token every time it switches state.
-// Use FlushTokens() to get the results after the last symbol was pushed.
-type Parser struct {
-	bufferLen int
-	buffer    []rune
-	current   State
-	tokens    []*analysis.Token
-	position  int
-	index     int
-}
-
-func NewParser(length, position, index int) *Parser {
-	return &Parser{
-		bufferLen: length,
-		buffer:    make([]rune, 0, length),
-		tokens:    make([]*analysis.Token, 0, length),
-		position:  position,
-		index:     index,
-	}
-}
-
-func (p *Parser) Push(sym rune, peek *rune) {
-	if p.current == nil {
-		// the start of parsing
-		p.current = p.NewState(sym)
-		p.buffer = append(p.buffer, sym)
-
-	} else if p.current.Member(sym, peek) {
-		// same state, just accumulate
-		p.buffer = append(p.buffer, sym)
-
-	} else {
-		// the old state is no more, thus convert the buffer
-		p.tokens = append(p.tokens, p.buildTokenFromTerm(p.buffer))
-
-		// let the new state begin
-		p.current = p.NewState(sym)
-		p.buffer = make([]rune, 0, p.bufferLen)
-		p.buffer = append(p.buffer, sym)
-	}
-}
-
-// Note. States have to have different starting symbols.
-func (p *Parser) NewState(sym rune) State {
-	var found State
-
-	found = &LowerCaseState{}
-	if found.StartSym(sym) {
-		return found
-	}
-
-	found = &UpperCaseState{}
-	if found.StartSym(sym) {
-		return found
-	}
-
-	found = &NumberCaseState{}
-	if found.StartSym(sym) {
-		return found
-	}
-
-	return &NonAlphaNumericCaseState{}
-}
-
-func (p *Parser) FlushTokens() []*analysis.Token {
-	p.tokens = append(p.tokens, p.buildTokenFromTerm(p.buffer))
-	return p.tokens
-}
-
-func (p *Parser) NextPosition() int {
-	return p.position
-}

vendor/github.com/blevesearch/bleve/analysis/token/camelcase/states.go

@@ -1,87 +0,0 @@
-//  Copyright (c) 2016 Couchbase, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// 		http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package camelcase
-
-import (
-	"unicode"
-)
-
-// States codify the classes that the parser recognizes.
-type State interface {
-	// is _sym_ the start character
-	StartSym(sym rune) bool
-
-	// is _sym_ a member of a class.
-	// peek, the next sym on the tape, can also be used to determine a class.
-	Member(sym rune, peek *rune) bool
-}
-
-type LowerCaseState struct{}
-
-func (s *LowerCaseState) Member(sym rune, peek *rune) bool {
-	return unicode.IsLower(sym)
-}
-
-func (s *LowerCaseState) StartSym(sym rune) bool {
-	return s.Member(sym, nil)
-}
-
-type UpperCaseState struct {
-	startedCollecting bool // denotes that the start character has been read
-	collectingUpper   bool // denotes if this is a class of all upper case letters
-}
-
-func (s *UpperCaseState) Member(sym rune, peek *rune) bool {
-	if !(unicode.IsLower(sym) || unicode.IsUpper(sym)) {
-		return false
-	}
-
-	if peek != nil && unicode.IsUpper(sym) && unicode.IsLower(*peek) {
-		return false
-	}
-
-	if !s.startedCollecting {
-		// now we have to determine if upper-case letters are collected.
-		s.startedCollecting = true
-		s.collectingUpper = unicode.IsUpper(sym)
-		return true
-	}
-
-	return s.collectingUpper == unicode.IsUpper(sym)
-}
-
-func (s *UpperCaseState) StartSym(sym rune) bool {
-	return unicode.IsUpper(sym)
-}
-
-type NumberCaseState struct{}
-
-func (s *NumberCaseState) Member(sym rune, peek *rune) bool {
-	return unicode.IsNumber(sym)
-}
-
-func (s *NumberCaseState) StartSym(sym rune) bool {
-	return s.Member(sym, nil)
-}
-
-type NonAlphaNumericCaseState struct{}
-
-func (s *NonAlphaNumericCaseState) Member(sym rune, peek *rune) bool {
-	return !unicode.IsLower(sym) && !unicode.IsUpper(sym) && !unicode.IsNumber(sym)
-}
-
-func (s *NonAlphaNumericCaseState) StartSym(sym rune) bool {
-	return s.Member(sym, nil)
-}

vendor/modules.txt

@@ -24,7 +24,6 @@ github.com/beorn7/perks/quantile
 # github.com/blevesearch/bleve v0.0.0-20190214220507-05d86ea8f6e3
 github.com/blevesearch/bleve
 github.com/blevesearch/bleve/analysis/analyzer/custom
-github.com/blevesearch/bleve/analysis/token/camelcase
 github.com/blevesearch/bleve/analysis/token/lowercase
 github.com/blevesearch/bleve/analysis/token/unicodenorm
 github.com/blevesearch/bleve/analysis/token/unique