From 833f8b94c2cd88277eba32984594aad2b7b2b05d Mon Sep 17 00:00:00 2001 From: Ethan Koenig Date: Tue, 24 Jan 2017 21:43:02 -0500 Subject: [PATCH] Search bar for issues/pulls (#530) --- .gitignore | 1 + conf/app.ini | 4 + models/issue.go | 74 +- models/issue_comment.go | 18 +- models/issue_indexer.go | 183 + models/models.go | 4 + modules/indexer/indexer.go | 14 + modules/setting/setting.go | 6 + modules/util/util.go | 25 + public/css/index.css | 4 + routers/api/v1/repo/issue.go | 6 +- routers/api/v1/repo/issue_comment.go | 2 +- routers/init.go | 2 + routers/repo/issue.go | 34 +- routers/user/home.go | 5 +- templates/repo/issue/list.tmpl | 43 +- templates/repo/issue/navbar.tmpl | 2 +- templates/repo/issue/search.tmpl | 13 + .../blevesearch/bleve/CONTRIBUTING.md | 16 + vendor/github.com/blevesearch/bleve/LICENSE | 202 + vendor/github.com/blevesearch/bleve/README.md | 62 + .../bleve/analysis/analyzer/simple/simple.go | 46 + .../analysis/analyzer/standard/standard.go | 52 + .../analysis/datetime/flexible/flexible.go | 64 + .../analysis/datetime/optional/optional.go | 45 + .../blevesearch/bleve/analysis/freq.go | 111 + .../bleve/analysis/lang/en/analyzer_en.go | 70 + .../analysis/lang/en/possessive_filter_en.go | 67 + .../bleve/analysis/lang/en/stop_filter_en.go | 33 + .../bleve/analysis/lang/en/stop_words_en.go | 344 + .../blevesearch/bleve/analysis/test_words.txt | 7 + .../analysis/token/lowercase/lowercase.go | 105 + .../bleve/analysis/token/porter/porter.go | 53 + .../bleve/analysis/token/stop/stop.go | 70 + .../analysis/tokenizer/character/character.go | 76 + .../bleve/analysis/tokenizer/letter/letter.go | 33 + .../analysis/tokenizer/unicode/unicode.go | 131 + .../blevesearch/bleve/analysis/tokenmap.go | 76 + .../blevesearch/bleve/analysis/type.go | 103 + .../blevesearch/bleve/analysis/util.go | 92 + vendor/github.com/blevesearch/bleve/config.go | 88 + .../blevesearch/bleve/config_app.go | 23 + .../blevesearch/bleve/config_disk.go | 25 + vendor/github.com/blevesearch/bleve/doc.go | 38 + .../blevesearch/bleve/document/document.go | 75 + .../blevesearch/bleve/document/field.go | 39 + .../bleve/document/field_boolean.go | 107 + .../bleve/document/field_composite.go | 99 + .../bleve/document/field_datetime.go | 144 + .../bleve/document/field_numeric.go | 130 + .../blevesearch/bleve/document/field_text.go | 119 + .../bleve/document/indexing_options.go | 55 + vendor/github.com/blevesearch/bleve/error.go | 52 + vendor/github.com/blevesearch/bleve/index.go | 243 + .../blevesearch/bleve/index/analysis.go | 83 + .../blevesearch/bleve/index/field_cache.go | 88 + .../blevesearch/bleve/index/index.go | 239 + .../blevesearch/bleve/index/store/batch.go | 62 + .../bleve/index/store/boltdb/iterator.go | 85 + .../bleve/index/store/boltdb/reader.go | 73 + .../bleve/index/store/boltdb/stats.go | 26 + .../bleve/index/store/boltdb/store.go | 175 + .../bleve/index/store/boltdb/writer.go | 95 + .../bleve/index/store/gtreap/iterator.go | 152 + .../bleve/index/store/gtreap/reader.go | 66 + .../bleve/index/store/gtreap/store.go | 82 + .../bleve/index/store/gtreap/writer.go | 76 + .../blevesearch/bleve/index/store/kvstore.go | 174 + .../blevesearch/bleve/index/store/merge.go | 64 + .../blevesearch/bleve/index/store/multiget.go | 33 + .../bleve/index/upsidedown/analysis.go | 110 + .../bleve/index/upsidedown/benchmark_all.sh | 8 + .../bleve/index/upsidedown/dump.go | 172 + .../bleve/index/upsidedown/field_dict.go | 78 + .../bleve/index/upsidedown/index_reader.go | 189 + .../bleve/index/upsidedown/reader.go | 325 + .../blevesearch/bleve/index/upsidedown/row.go | 853 + .../bleve/index/upsidedown/row_merge.go | 76 + .../bleve/index/upsidedown/stats.go | 55 + .../bleve/index/upsidedown/upsidedown.go | 1037 + .../bleve/index/upsidedown/upsidedown.pb.go | 684 + .../bleve/index/upsidedown/upsidedown.proto | 14 + .../blevesearch/bleve/index_alias.go | 37 + .../blevesearch/bleve/index_alias_impl.go | 605 + .../blevesearch/bleve/index_impl.go | 729 + .../blevesearch/bleve/index_meta.go | 96 + .../blevesearch/bleve/index_stats.go | 75 + .../github.com/blevesearch/bleve/mapping.go | 61 + .../blevesearch/bleve/mapping/analysis.go | 99 + .../blevesearch/bleve/mapping/document.go | 490 + .../blevesearch/bleve/mapping/field.go | 296 + .../blevesearch/bleve/mapping/index.go | 430 + .../blevesearch/bleve/mapping/mapping.go | 49 + .../blevesearch/bleve/mapping/reflect.go | 89 + .../blevesearch/bleve/numeric/float.go | 34 + .../blevesearch/bleve/numeric/prefix_coded.go | 92 + vendor/github.com/blevesearch/bleve/query.go | 186 + .../blevesearch/bleve/registry/analyzer.go | 89 + .../blevesearch/bleve/registry/cache.go | 87 + .../blevesearch/bleve/registry/char_filter.go | 89 + .../bleve/registry/datetime_parser.go | 89 + .../bleve/registry/fragment_formatter.go | 89 + .../blevesearch/bleve/registry/fragmenter.go | 89 + .../blevesearch/bleve/registry/highlighter.go | 89 + .../blevesearch/bleve/registry/index_type.go | 45 + .../blevesearch/bleve/registry/registry.go | 184 + .../blevesearch/bleve/registry/store.go | 51 + .../bleve/registry/token_filter.go | 89 + .../blevesearch/bleve/registry/token_maps.go | 89 + .../blevesearch/bleve/registry/tokenizer.go | 89 + vendor/github.com/blevesearch/bleve/search.go | 451 + .../blevesearch/bleve/search/collector.go | 33 + .../bleve/search/collector/heap.go | 91 + .../bleve/search/collector/list.go | 78 + .../bleve/search/collector/slice.go | 68 + .../bleve/search/collector/topn.go | 269 + .../blevesearch/bleve/search/explanation.go | 34 + .../bleve/search/facet/benchmark_data.txt | 2909 + .../search/facet/facet_builder_datetime.go | 135 + .../search/facet/facet_builder_numeric.go | 129 + .../bleve/search/facet/facet_builder_terms.go | 95 + .../bleve/search/facets_builder.go | 296 + .../search/highlight/format/html/html.go | 89 + .../highlight/fragmenter/simple/simple.go | 142 + .../bleve/search/highlight/highlighter.go | 64 + .../search/highlight/highlighter/html/html.go | 50 + .../simple/fragment_scorer_simple.go | 49 + .../highlighter/simple/highlighter_simple.go | 221 + .../bleve/search/highlight/term_locations.go | 117 + .../blevesearch/bleve/search/levenshtein.go | 105 + .../blevesearch/bleve/search/pool.go | 76 + .../bleve/search/query/bool_field.go | 65 + .../blevesearch/bleve/search/query/boolean.go | 209 + .../blevesearch/bleve/search/query/boost.go | 33 + .../bleve/search/query/conjunction.go | 102 + .../bleve/search/query/date_range.go | 165 + .../bleve/search/query/disjunction.go | 114 + .../blevesearch/bleve/search/query/docid.go | 49 + .../blevesearch/bleve/search/query/fuzzy.go | 77 + .../blevesearch/bleve/search/query/match.go | 176 + .../bleve/search/query/match_all.go | 57 + .../bleve/search/query/match_none.go | 55 + .../bleve/search/query/match_phrase.go | 116 + .../bleve/search/query/numeric_range.go | 87 + .../blevesearch/bleve/search/query/phrase.go | 97 + .../blevesearch/bleve/search/query/prefix.go | 62 + .../blevesearch/bleve/search/query/query.go | 328 + .../bleve/search/query/query_string.go | 63 + .../bleve/search/query/query_string.y | 289 + .../bleve/search/query/query_string.y.go | 773 + .../bleve/search/query/query_string_lex.go | 322 + .../bleve/search/query/query_string_parser.go | 79 + .../blevesearch/bleve/search/query/regexp.go | 94 + .../blevesearch/bleve/search/query/term.go | 61 + .../bleve/search/query/wildcard.go | 106 + .../bleve/search/scorer/scorer_conjunction.go | 65 + .../bleve/search/scorer/scorer_constant.go | 108 + .../bleve/search/scorer/scorer_disjunction.go | 77 + .../bleve/search/scorer/scorer_term.go | 180 + .../bleve/search/scorer/sqrt_cache.go | 30 + .../blevesearch/bleve/search/search.go | 144 + .../search/searcher/ordered_searchers_list.go | 35 + .../bleve/search/searcher/search_boolean.go | 391 + .../search/searcher/search_conjunction.go | 232 + .../search/searcher/search_disjunction.go | 258 + .../bleve/search/searcher/search_docid.go | 93 + .../bleve/search/searcher/search_fuzzy.go | 143 + .../bleve/search/searcher/search_match_all.go | 105 + .../search/searcher/search_match_none.go | 62 + .../search/searcher/search_numeric_range.go | 235 + .../bleve/search/searcher/search_phrase.go | 197 + .../bleve/search/searcher/search_regexp.go | 92 + .../bleve/search/searcher/search_term.go | 110 + .../search/searcher/search_term_prefix.go | 111 + .../blevesearch/bleve/search/sort.go | 493 + .../blevesearch/bleve/search/util.go | 42 + .../blevesearch/go-porterstemmer/LICENSE | 19 + .../blevesearch/go-porterstemmer/README.md | 118 + .../go-porterstemmer/porterstemmer.go | 839 + vendor/github.com/blevesearch/segment/LICENSE | 202 + .../github.com/blevesearch/segment/README.md | 92 + vendor/github.com/blevesearch/segment/doc.go | 45 + .../blevesearch/segment/maketesttables.go | 219 + .../github.com/blevesearch/segment/segment.go | 284 + .../blevesearch/segment/segment_fuzz.go | 22 + .../blevesearch/segment/segment_words.go | 19542 ++ .../blevesearch/segment/segment_words.rl | 285 + .../blevesearch/segment/segment_words_prod.go | 173643 +++++++++++++++ vendor/github.com/steveyen/gtreap/LICENSE | 20 + vendor/github.com/steveyen/gtreap/README.md | 90 + vendor/github.com/steveyen/gtreap/treap.go | 188 + vendor/golang.org/x/net/context/context.go | 156 + vendor/golang.org/x/net/context/go17.go | 72 + vendor/golang.org/x/net/context/pre_go17.go | 300 + vendor/vendor.json | 226 + 195 files changed, 221830 insertions(+), 60 deletions(-) create mode 100644 models/issue_indexer.go create mode 100644 modules/indexer/indexer.go create mode 100644 modules/util/util.go create mode 100644 templates/repo/issue/search.tmpl create mode 100644 vendor/github.com/blevesearch/bleve/CONTRIBUTING.md create mode 100644 vendor/github.com/blevesearch/bleve/LICENSE create mode 100644 vendor/github.com/blevesearch/bleve/README.md create mode 100644 vendor/github.com/blevesearch/bleve/analysis/analyzer/simple/simple.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/analyzer/standard/standard.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/datetime/flexible/flexible.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/datetime/optional/optional.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/freq.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/lang/en/analyzer_en.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/lang/en/possessive_filter_en.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_filter_en.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_words_en.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/test_words.txt create mode 100644 vendor/github.com/blevesearch/bleve/analysis/token/lowercase/lowercase.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/token/porter/porter.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/token/stop/stop.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizer/character/character.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizer/letter/letter.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizer/unicode/unicode.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenmap.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/type.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/util.go create mode 100644 vendor/github.com/blevesearch/bleve/config.go create mode 100644 vendor/github.com/blevesearch/bleve/config_app.go create mode 100644 vendor/github.com/blevesearch/bleve/config_disk.go create mode 100644 vendor/github.com/blevesearch/bleve/doc.go create mode 100644 vendor/github.com/blevesearch/bleve/document/document.go create mode 100644 vendor/github.com/blevesearch/bleve/document/field.go create mode 100644 vendor/github.com/blevesearch/bleve/document/field_boolean.go create mode 100644 vendor/github.com/blevesearch/bleve/document/field_composite.go create mode 100644 vendor/github.com/blevesearch/bleve/document/field_datetime.go create mode 100644 vendor/github.com/blevesearch/bleve/document/field_numeric.go create mode 100644 vendor/github.com/blevesearch/bleve/document/field_text.go create mode 100644 vendor/github.com/blevesearch/bleve/document/indexing_options.go create mode 100644 vendor/github.com/blevesearch/bleve/error.go create mode 100644 vendor/github.com/blevesearch/bleve/index.go create mode 100644 vendor/github.com/blevesearch/bleve/index/analysis.go create mode 100644 vendor/github.com/blevesearch/bleve/index/field_cache.go create mode 100644 vendor/github.com/blevesearch/bleve/index/index.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/batch.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/boltdb/iterator.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/boltdb/stats.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/kvstore.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/merge.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/multiget.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go create mode 100755 vendor/github.com/blevesearch/bleve/index/upsidedown/benchmark_all.sh create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/field_dict.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/row.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/row_merge.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/stats.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.pb.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.proto create mode 100644 vendor/github.com/blevesearch/bleve/index_alias.go create mode 100644 vendor/github.com/blevesearch/bleve/index_alias_impl.go create mode 100644 vendor/github.com/blevesearch/bleve/index_impl.go create mode 100644 vendor/github.com/blevesearch/bleve/index_meta.go create mode 100644 vendor/github.com/blevesearch/bleve/index_stats.go create mode 100644 vendor/github.com/blevesearch/bleve/mapping.go create mode 100644 vendor/github.com/blevesearch/bleve/mapping/analysis.go create mode 100644 vendor/github.com/blevesearch/bleve/mapping/document.go create mode 100644 vendor/github.com/blevesearch/bleve/mapping/field.go create mode 100644 vendor/github.com/blevesearch/bleve/mapping/index.go create mode 100644 vendor/github.com/blevesearch/bleve/mapping/mapping.go create mode 100644 vendor/github.com/blevesearch/bleve/mapping/reflect.go create mode 100644 vendor/github.com/blevesearch/bleve/numeric/float.go create mode 100644 vendor/github.com/blevesearch/bleve/numeric/prefix_coded.go create mode 100644 vendor/github.com/blevesearch/bleve/query.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/analyzer.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/cache.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/char_filter.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/datetime_parser.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/fragment_formatter.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/fragmenter.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/highlighter.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/index_type.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/registry.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/store.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/token_filter.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/token_maps.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/tokenizer.go create mode 100644 vendor/github.com/blevesearch/bleve/search.go create mode 100644 vendor/github.com/blevesearch/bleve/search/collector.go create mode 100644 vendor/github.com/blevesearch/bleve/search/collector/heap.go create mode 100644 vendor/github.com/blevesearch/bleve/search/collector/list.go create mode 100644 vendor/github.com/blevesearch/bleve/search/collector/slice.go create mode 100644 vendor/github.com/blevesearch/bleve/search/collector/topn.go create mode 100644 vendor/github.com/blevesearch/bleve/search/explanation.go create mode 100644 vendor/github.com/blevesearch/bleve/search/facet/benchmark_data.txt create mode 100644 vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go create mode 100644 vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go create mode 100644 vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go create mode 100644 vendor/github.com/blevesearch/bleve/search/facets_builder.go create mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/format/html/html.go create mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/fragmenter/simple/simple.go create mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/highlighter.go create mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/highlighter/html/html.go create mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/fragment_scorer_simple.go create mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/highlighter_simple.go create mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go create mode 100644 vendor/github.com/blevesearch/bleve/search/levenshtein.go create mode 100644 vendor/github.com/blevesearch/bleve/search/pool.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/bool_field.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/boolean.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/boost.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/conjunction.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/date_range.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/disjunction.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/docid.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/fuzzy.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/match.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/match_all.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/match_none.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/match_phrase.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/numeric_range.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/phrase.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/prefix.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query_string.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query_string.y create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query_string.y.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query_string_parser.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/regexp.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/term.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/wildcard.go create mode 100644 vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go create mode 100644 vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go create mode 100644 vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go create mode 100644 vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go create mode 100644 vendor/github.com/blevesearch/bleve/search/scorer/sqrt_cache.go create mode 100644 vendor/github.com/blevesearch/bleve/search/search.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/ordered_searchers_list.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_term.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_term_prefix.go create mode 100644 vendor/github.com/blevesearch/bleve/search/sort.go create mode 100644 vendor/github.com/blevesearch/bleve/search/util.go create mode 100644 vendor/github.com/blevesearch/go-porterstemmer/LICENSE create mode 100644 vendor/github.com/blevesearch/go-porterstemmer/README.md create mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer.go create mode 100644 vendor/github.com/blevesearch/segment/LICENSE create mode 100644 vendor/github.com/blevesearch/segment/README.md create mode 100644 vendor/github.com/blevesearch/segment/doc.go create mode 100644 vendor/github.com/blevesearch/segment/maketesttables.go create mode 100644 vendor/github.com/blevesearch/segment/segment.go create mode 100644 vendor/github.com/blevesearch/segment/segment_fuzz.go create mode 100644 vendor/github.com/blevesearch/segment/segment_words.go create mode 100644 vendor/github.com/blevesearch/segment/segment_words.rl create mode 100644 vendor/github.com/blevesearch/segment/segment_words_prod.go create mode 100644 vendor/github.com/steveyen/gtreap/LICENSE create mode 100644 vendor/github.com/steveyen/gtreap/README.md create mode 100644 vendor/github.com/steveyen/gtreap/treap.go create mode 100644 vendor/golang.org/x/net/context/context.go create mode 100644 vendor/golang.org/x/net/context/go17.go create mode 100644 vendor/golang.org/x/net/context/pre_go17.go diff --git a/.gitignore b/.gitignore index 2cbc7ccfa..79198ffcf 100644 --- a/.gitignore +++ b/.gitignore @@ -41,5 +41,6 @@ coverage.out /dist /custom /data +/indexers /log /public/img/avatar diff --git a/conf/app.ini b/conf/app.ini index 303b006b2..254b1bc54 100644 --- a/conf/app.ini +++ b/conf/app.ini @@ -158,6 +158,10 @@ SSL_MODE = disable ; For "sqlite3" and "tidb", use absolute path when you start as service PATH = data/gitea.db +[indexer] +ISSUE_INDEXER_PATH = indexers/issues.bleve +UPDATE_BUFFER_LEN = 20 + [admin] [security] diff --git a/models/issue.go b/models/issue.go index ac50d2dfb..d92616138 100644 --- a/models/issue.go +++ b/models/issue.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) var ( @@ -451,8 +452,11 @@ func (issue *Issue) ReadBy(userID int64) error { } func updateIssueCols(e Engine, issue *Issue, cols ...string) error { - _, err := e.Id(issue.ID).Cols(cols...).Update(issue) - return err + if _, err := e.Id(issue.ID).Cols(cols...).Update(issue); err != nil { + return err + } + UpdateIssueIndexer(issue) + return nil } // UpdateIssueCols only updates values of specific columns for given issue. @@ -733,6 +737,8 @@ func newIssue(e *xorm.Session, opts NewIssueOptions) (err error) { return err } + UpdateIssueIndexer(opts.Issue) + if len(opts.Attachments) > 0 { attachments, err := getAttachmentsByUUIDs(e, opts.Attachments) if err != nil { @@ -865,10 +871,11 @@ type IssuesOptions struct { MilestoneID int64 RepoIDs []int64 Page int - IsClosed bool - IsPull bool + IsClosed util.OptionalBool + IsPull util.OptionalBool Labels string SortType string + IssueIDs []int64 } // sortIssuesSession sort an issues-related session based on the provided @@ -894,11 +901,23 @@ func sortIssuesSession(sess *xorm.Session, sortType string) { // Issues returns a list of issues by given conditions. func Issues(opts *IssuesOptions) ([]*Issue, error) { - if opts.Page <= 0 { - opts.Page = 1 + var sess *xorm.Session + if opts.Page >= 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * setting.UI.IssuePagingNum + } + sess = x.Limit(setting.UI.IssuePagingNum, start) + } else { + sess = x.NewSession() + defer sess.Close() } - sess := x.Limit(setting.UI.IssuePagingNum, (opts.Page-1)*setting.UI.IssuePagingNum) + if len(opts.IssueIDs) > 0 { + sess.In("issue.id", opts.IssueIDs) + } if opts.RepoID > 0 { sess.And("issue.repo_id=?", opts.RepoID) @@ -906,7 +925,13 @@ func Issues(opts *IssuesOptions) ([]*Issue, error) { // In case repository IDs are provided but actually no repository has issue. sess.In("issue.repo_id", opts.RepoIDs) } - sess.And("issue.is_closed=?", opts.IsClosed) + + switch opts.IsClosed { + case util.OptionalBoolTrue: + sess.And("issue.is_closed=true") + case util.OptionalBoolFalse: + sess.And("issue.is_closed=false") + } if opts.AssigneeID > 0 { sess.And("issue.assignee_id=?", opts.AssigneeID) @@ -926,7 +951,12 @@ func Issues(opts *IssuesOptions) ([]*Issue, error) { sess.And("issue.milestone_id=?", opts.MilestoneID) } - sess.And("issue.is_pull=?", opts.IsPull) + switch opts.IsPull { + case util.OptionalBoolTrue: + sess.And("issue.is_pull=true") + case util.OptionalBoolFalse: + sess.And("issue.is_pull=false") + } sortIssuesSession(sess, opts.SortType) @@ -1168,10 +1198,11 @@ type IssueStatsOptions struct { MentionedID int64 PosterID int64 IsPull bool + IssueIDs []int64 } // GetIssueStats returns issue statistic information by given conditions. -func GetIssueStats(opts *IssueStatsOptions) *IssueStats { +func GetIssueStats(opts *IssueStatsOptions) (*IssueStats, error) { stats := &IssueStats{} countSession := func(opts *IssueStatsOptions) *xorm.Session { @@ -1179,6 +1210,10 @@ func GetIssueStats(opts *IssueStatsOptions) *IssueStats { Where("issue.repo_id = ?", opts.RepoID). And("is_pull = ?", opts.IsPull) + if len(opts.IssueIDs) > 0 { + sess.In("issue.id", opts.IssueIDs) + } + if len(opts.Labels) > 0 && opts.Labels != "0" { labelIDs, err := base.StringsToInt64s(strings.Split(opts.Labels, ",")) if err != nil { @@ -1210,13 +1245,20 @@ func GetIssueStats(opts *IssueStatsOptions) *IssueStats { return sess } - stats.OpenCount, _ = countSession(opts). + var err error + stats.OpenCount, err = countSession(opts). And("is_closed = ?", false). Count(&Issue{}) - stats.ClosedCount, _ = countSession(opts). + if err != nil { + return nil, err + } + stats.ClosedCount, err = countSession(opts). And("is_closed = ?", true). Count(&Issue{}) - return stats + if err != nil { + return nil, err + } + return stats, nil } // GetUserIssueStats returns issue statistic information for dashboard by given conditions. @@ -1294,7 +1336,11 @@ func GetRepoIssueStats(repoID, uid int64, filterMode int, isPull bool) (numOpen func updateIssue(e Engine, issue *Issue) error { _, err := e.Id(issue.ID).AllCols().Update(issue) - return err + if err != nil { + return err + } + UpdateIssueIndexer(issue) + return nil } // UpdateIssue updates all fields of given issue. diff --git a/models/issue_comment.go b/models/issue_comment.go index e9a401b86..a17be97e7 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -454,28 +454,20 @@ func UpdateComment(c *Comment) error { return err } -// DeleteCommentByID deletes the comment by given ID. -func DeleteCommentByID(id int64) error { - comment, err := GetCommentByID(id) - if err != nil { - if IsErrCommentNotExist(err) { - return nil - } - return err - } - +// DeleteComment deletes the comment +func DeleteComment(comment *Comment) error { sess := x.NewSession() defer sessionRelease(sess) - if err = sess.Begin(); err != nil { + if err := sess.Begin(); err != nil { return err } - if _, err = sess.Id(comment.ID).Delete(new(Comment)); err != nil { + if _, err := sess.Id(comment.ID).Delete(new(Comment)); err != nil { return err } if comment.Type == CommentTypeComment { - if _, err = sess.Exec("UPDATE `issue` SET num_comments = num_comments - 1 WHERE id = ?", comment.IssueID); err != nil { + if _, err := sess.Exec("UPDATE `issue` SET num_comments = num_comments - 1 WHERE id = ?", comment.IssueID); err != nil { return err } } diff --git a/models/issue_indexer.go b/models/issue_indexer.go new file mode 100644 index 000000000..bbaf0e64b --- /dev/null +++ b/models/issue_indexer.go @@ -0,0 +1,183 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "fmt" + "os" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/analysis/analyzer/simple" + "github.com/blevesearch/bleve/search/query" +) + +// issueIndexerUpdateQueue queue of issues that need to be updated in the issues +// indexer +var issueIndexerUpdateQueue chan *Issue + +// issueIndexer (thread-safe) index for searching issues +var issueIndexer bleve.Index + +// issueIndexerData data stored in the issue indexer +type issueIndexerData struct { + ID int64 + RepoID int64 + + Title string + Content string +} + +// numericQuery an numeric-equality query for the given value and field +func numericQuery(value int64, field string) *query.NumericRangeQuery { + f := float64(value) + tru := true + q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru) + q.SetField(field) + return q +} + +// SearchIssuesByKeyword searches for issues by given conditions. +// Returns the matching issue IDs +func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) { + fields := strings.Fields(strings.ToLower(keyword)) + indexerQuery := bleve.NewConjunctionQuery( + numericQuery(repoID, "RepoID"), + bleve.NewDisjunctionQuery( + bleve.NewPhraseQuery(fields, "Title"), + bleve.NewPhraseQuery(fields, "Content"), + )) + search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false) + search.Fields = []string{"ID"} + + result, err := issueIndexer.Search(search) + if err != nil { + return nil, err + } + + issueIDs := make([]int64, len(result.Hits)) + for i, hit := range result.Hits { + issueIDs[i] = int64(hit.Fields["ID"].(float64)) + } + return issueIDs, nil +} + +// InitIssueIndexer initialize issue indexer +func InitIssueIndexer() { + _, err := os.Stat(setting.Indexer.IssuePath) + if err != nil { + if os.IsNotExist(err) { + if err = createIssueIndexer(); err != nil { + log.Fatal(4, "CreateIssuesIndexer: %v", err) + } + if err = populateIssueIndexer(); err != nil { + log.Fatal(4, "PopulateIssuesIndex: %v", err) + } + } else { + log.Fatal(4, "InitIssuesIndexer: %v", err) + } + } else { + issueIndexer, err = bleve.Open(setting.Indexer.IssuePath) + if err != nil { + log.Fatal(4, "InitIssuesIndexer, open index: %v", err) + } + } + issueIndexerUpdateQueue = make(chan *Issue, setting.Indexer.UpdateQueueLength) + go processIssueIndexerUpdateQueue() + // TODO close issueIndexer when Gitea closes +} + +// createIssueIndexer create an issue indexer if one does not already exist +func createIssueIndexer() error { + mapping := bleve.NewIndexMapping() + docMapping := bleve.NewDocumentMapping() + + docMapping.AddFieldMappingsAt("ID", bleve.NewNumericFieldMapping()) + docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping()) + + textFieldMapping := bleve.NewTextFieldMapping() + textFieldMapping.Analyzer = simple.Name + docMapping.AddFieldMappingsAt("Title", textFieldMapping) + docMapping.AddFieldMappingsAt("Content", textFieldMapping) + + mapping.AddDocumentMapping("issues", docMapping) + + var err error + issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping) + return err +} + +// populateIssueIndexer populate the issue indexer with issue data +func populateIssueIndexer() error { + for page := 1; ; page++ { + repos, err := Repositories(&SearchRepoOptions{ + Page: page, + PageSize: 10, + }) + if err != nil { + return fmt.Errorf("Repositories: %v", err) + } + if len(repos) == 0 { + return nil + } + batch := issueIndexer.NewBatch() + for _, repo := range repos { + issues, err := Issues(&IssuesOptions{ + RepoID: repo.ID, + IsClosed: util.OptionalBoolNone, + IsPull: util.OptionalBoolNone, + Page: -1, // do not page + }) + if err != nil { + return fmt.Errorf("Issues: %v", err) + } + for _, issue := range issues { + err = batch.Index(issue.indexUID(), issue.issueData()) + if err != nil { + return fmt.Errorf("batch.Index: %v", err) + } + } + } + if err = issueIndexer.Batch(batch); err != nil { + return fmt.Errorf("index.Batch: %v", err) + } + } +} + +func processIssueIndexerUpdateQueue() { + for { + select { + case issue := <-issueIndexerUpdateQueue: + if err := issueIndexer.Index(issue.indexUID(), issue.issueData()); err != nil { + log.Error(4, "issuesIndexer.Index: %v", err) + } + } + } +} + +// indexUID a unique identifier for an issue used in full-text indices +func (issue *Issue) indexUID() string { + return strconv.FormatInt(issue.ID, 36) +} + +func (issue *Issue) issueData() *issueIndexerData { + return &issueIndexerData{ + ID: issue.ID, + RepoID: issue.RepoID, + Title: issue.Title, + Content: issue.Content, + } +} + +// UpdateIssueIndexer add/update an issue to the issue indexer +func UpdateIssueIndexer(issue *Issue) { + go func() { + issueIndexerUpdateQueue <- issue + }() +} diff --git a/models/models.go b/models/models.go index d9716e79b..1ce704a9e 100644 --- a/models/models.go +++ b/models/models.go @@ -138,6 +138,10 @@ func LoadConfigs() { } DbCfg.SSLMode = sec.Key("SSL_MODE").String() DbCfg.Path = sec.Key("PATH").MustString("data/gitea.db") + + sec = setting.Cfg.Section("indexer") + setting.Indexer.IssuePath = sec.Key("ISSUE_INDEXER_PATH").MustString("indexers/issues.bleve") + setting.Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20) } // parsePostgreSQLHostPort parses given input in various forms defined in diff --git a/modules/indexer/indexer.go b/modules/indexer/indexer.go new file mode 100644 index 000000000..2b7b76f7f --- /dev/null +++ b/modules/indexer/indexer.go @@ -0,0 +1,14 @@ +// Copyright 2016 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package indexer + +import ( + "code.gitea.io/gitea/models" +) + +// NewContext start indexer service +func NewContext() { + models.InitIssueIndexer() +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 910ec9302..fd0f5085c 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -123,6 +123,12 @@ var ( UsePostgreSQL bool UseTiDB bool + // Indexer settings + Indexer struct { + IssuePath string + UpdateQueueLength int + } + // Webhook settings Webhook = struct { QueueLength int diff --git a/modules/util/util.go b/modules/util/util.go new file mode 100644 index 000000000..485996538 --- /dev/null +++ b/modules/util/util.go @@ -0,0 +1,25 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +// OptionalBool a boolean that can be "null" +type OptionalBool byte + +const ( + // OptionalBoolNone a "null" boolean value + OptionalBoolNone = iota + // OptionalBoolTrue a "true" boolean value + OptionalBoolTrue + // OptionalBoolFalse a "false" boolean value + OptionalBoolFalse +) + +// OptionalBoolOf get the corresponding OptionalBool of a bool +func OptionalBoolOf(b bool) OptionalBool { + if b { + return OptionalBoolTrue + } + return OptionalBoolFalse +} diff --git a/public/css/index.css b/public/css/index.css index 7c84cf851..d755e6127 100644 --- a/public/css/index.css +++ b/public/css/index.css @@ -2926,6 +2926,10 @@ footer .ui.language .menu { width: 16px; text-align: center; } +.navbar { + display: flex; + justify-content: space-between; +} .ui.repository.list .item { padding-bottom: 25px; } diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 908b5aeb9..ac0289c41 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -13,14 +13,16 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) // ListIssues list the issues of a repository func ListIssues(ctx *context.APIContext) { + isClosed := ctx.Query("state") == "closed" issueOpts := models.IssuesOptions{ RepoID: ctx.Repo.Repository.ID, Page: ctx.QueryInt("page"), - IsClosed: ctx.Query("state") == "closed", + IsClosed: util.OptionalBoolOf(isClosed), } issues, err := models.Issues(&issueOpts) @@ -29,7 +31,7 @@ func ListIssues(ctx *context.APIContext) { return } if ctx.Query("state") == "all" { - issueOpts.IsClosed = !issueOpts.IsClosed + issueOpts.IsClosed = util.OptionalBoolOf(!isClosed) tempIssues, err := models.Issues(&issueOpts) if err != nil { ctx.Error(500, "Issues", err) diff --git a/routers/api/v1/repo/issue_comment.go b/routers/api/v1/repo/issue_comment.go index 1afd3e9d7..13e3ec6ab 100644 --- a/routers/api/v1/repo/issue_comment.go +++ b/routers/api/v1/repo/issue_comment.go @@ -125,7 +125,7 @@ func DeleteIssueComment(ctx *context.APIContext) { return } - if err = models.DeleteCommentByID(comment.ID); err != nil { + if err = models.DeleteComment(comment); err != nil { ctx.Error(500, "DeleteCommentByID", err) return } diff --git a/routers/init.go b/routers/init.go index 697f33835..38a456639 100644 --- a/routers/init.go +++ b/routers/init.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" macaron "gopkg.in/macaron.v1" + "code.gitea.io/gitea/modules/indexer" ) func checkRunMode() { @@ -36,6 +37,7 @@ func checkRunMode() { func NewServices() { setting.NewServices() mailer.NewContext() + indexer.NewContext() } // GlobalInit is for global configuration reload-able. diff --git a/routers/repo/issue.go b/routers/repo/issue.go index 50995e70f..6ef34746e 100644 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -5,6 +5,7 @@ package repo import ( + "bytes" "errors" "fmt" "io" @@ -25,6 +26,7 @@ import ( "code.gitea.io/gitea/modules/markdown" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) const ( @@ -158,20 +160,39 @@ func Issues(ctx *context.Context) { milestoneID := ctx.QueryInt64("milestone") isShowClosed := ctx.Query("state") == "closed" + keyword := ctx.Query("q") + if bytes.Contains([]byte(keyword), []byte{0x00}) { + keyword = "" + } + + var issueIDs []int64 + var err error + if len(keyword) > 0 { + issueIDs, err = models.SearchIssuesByKeyword(repo.ID, keyword) + if len(issueIDs) == 0 { + forceEmpty = true + } + } + var issueStats *models.IssueStats if forceEmpty { issueStats = &models.IssueStats{} } else { - issueStats = models.GetIssueStats(&models.IssueStatsOptions{ + var err error + issueStats, err = models.GetIssueStats(&models.IssueStatsOptions{ RepoID: repo.ID, Labels: selectLabels, MilestoneID: milestoneID, AssigneeID: assigneeID, MentionedID: mentionedID, IsPull: isPullList, + IssueIDs: issueIDs, }) + if err != nil { + ctx.Error(500, "GetSearchIssueStats") + return + } } - page := ctx.QueryInt("page") if page <= 1 { page = 1 @@ -190,7 +211,6 @@ func Issues(ctx *context.Context) { if forceEmpty { issues = []*models.Issue{} } else { - var err error issues, err = models.Issues(&models.IssuesOptions{ AssigneeID: assigneeID, RepoID: repo.ID, @@ -198,10 +218,11 @@ func Issues(ctx *context.Context) { MentionedID: mentionedID, MilestoneID: milestoneID, Page: pager.Current(), - IsClosed: isShowClosed, - IsPull: isPullList, + IsClosed: util.OptionalBoolOf(isShowClosed), + IsPull: util.OptionalBoolOf(isPullList), Labels: selectLabels, SortType: sortType, + IssueIDs: issueIDs, }) if err != nil { ctx.Handle(500, "Issues", err) @@ -258,6 +279,7 @@ func Issues(ctx *context.Context) { ctx.Data["MilestoneID"] = milestoneID ctx.Data["AssigneeID"] = assigneeID ctx.Data["IsShowClosed"] = isShowClosed + ctx.Data["Keyword"] = keyword if isShowClosed { ctx.Data["State"] = "closed" } else { @@ -934,7 +956,7 @@ func DeleteComment(ctx *context.Context) { return } - if err = models.DeleteCommentByID(comment.ID); err != nil { + if err = models.DeleteComment(comment); err != nil { ctx.Handle(500, "DeleteCommentByID", err) return } diff --git a/routers/user/home.go b/routers/user/home.go index 571849df3..db2fe84f9 100644 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) const ( @@ -277,8 +278,8 @@ func Issues(ctx *context.Context) { PosterID: posterID, RepoIDs: repoIDs, Page: page, - IsClosed: isShowClosed, - IsPull: isPullList, + IsClosed: util.OptionalBoolOf(isShowClosed), + IsPull: util.OptionalBoolOf(isPullList), SortType: sortType, }) if err != nil { diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index d00c9aea2..bb7327ebf 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -4,6 +4,7 @@
@@ -105,7 +106,7 @@ {{.Title}} {{range .Labels}} - {{.Name}} + {{.Name}} {{end}} {{if .NumComments}} @@ -115,7 +116,7 @@

{{$.i18n.Tr "repo.issues.opened_by" $timeStr .Poster.HomeLink .Poster.Name | Safe}} {{if .Milestone}} - + {{.Milestone.Name}} {{end}} @@ -132,17 +133,17 @@ {{if gt .TotalPages 1}}

diff --git a/templates/repo/issue/navbar.tmpl b/templates/repo/issue/navbar.tmpl index a3e9d2660..1e864458e 100644 --- a/templates/repo/issue/navbar.tmpl +++ b/templates/repo/issue/navbar.tmpl @@ -1,4 +1,4 @@ -