Clean-up the settings hierarchy for issue_indexer queue (#16001)
There are a couple of settings in `[indexer]` relating to the `issue_indexer` queue which override settings in unpredictable ways. This PR adjusts this hierarchy and makes explicit that these settings are deprecated. Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
parent
5f4522cd1f
commit
ffbf35b7e9
4 changed files with 34 additions and 33 deletions
|
@ -1155,20 +1155,20 @@ PATH =
|
||||||
;STARTUP_TIMEOUT = 30s
|
;STARTUP_TIMEOUT = 30s
|
||||||
;;
|
;;
|
||||||
;; Issue indexer queue, currently support: channel, levelqueue or redis, default is levelqueue (deprecated - use [queue.issue_indexer])
|
;; Issue indexer queue, currently support: channel, levelqueue or redis, default is levelqueue (deprecated - use [queue.issue_indexer])
|
||||||
;ISSUE_INDEXER_QUEUE_TYPE = levelqueue
|
;ISSUE_INDEXER_QUEUE_TYPE = levelqueue; **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||||
;;
|
;;
|
||||||
;; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the path where the queue will be saved.
|
;; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the path where the queue will be saved.
|
||||||
;; This can be overridden by `ISSUE_INDEXER_QUEUE_CONN_STR`.
|
;; This can be overridden by `ISSUE_INDEXER_QUEUE_CONN_STR`.
|
||||||
;; default is queues/common
|
;; default is queues/common
|
||||||
;ISSUE_INDEXER_QUEUE_DIR = queues/common
|
;ISSUE_INDEXER_QUEUE_DIR = queues/common; **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||||
;;
|
;;
|
||||||
;; When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string.
|
;; When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string.
|
||||||
;; When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of
|
;; When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of
|
||||||
;; the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`.
|
;; the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`.
|
||||||
;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
|
;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"; **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||||
;;
|
;;
|
||||||
;; Batch queue number, default is 20
|
;; Batch queue number, default is 20
|
||||||
;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20
|
;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||||
|
|
||||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
;; Repository Indexer settings
|
;; Repository Indexer settings
|
||||||
|
@ -1197,7 +1197,7 @@ PATH =
|
||||||
;REPO_INDEXER_EXCLUDE =
|
;REPO_INDEXER_EXCLUDE =
|
||||||
;;
|
;;
|
||||||
;;
|
;;
|
||||||
;UPDATE_BUFFER_LEN = 20
|
;UPDATE_BUFFER_LEN = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||||
;MAX_FILE_SIZE = 1048576
|
;MAX_FILE_SIZE = 1048576
|
||||||
|
|
||||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
|
|
@ -356,10 +356,10 @@ relation to port exhaustion.
|
||||||
- `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch
|
- `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch
|
||||||
- `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch.
|
- `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch.
|
||||||
- The next 4 configuration values are deprecated and should be set in `queue.issue_indexer` however are kept for backwards compatibility:
|
- The next 4 configuration values are deprecated and should be set in `queue.issue_indexer` however are kept for backwards compatibility:
|
||||||
- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`.
|
- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`. **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||||
- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. (Previously this was `indexers/issues.queue`.)
|
- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||||
- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`.
|
- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||||
- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number.
|
- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number. **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||||
|
|
||||||
- `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
|
- `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
|
||||||
- `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`.
|
- `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`.
|
||||||
|
@ -370,7 +370,7 @@ relation to port exhaustion.
|
||||||
- `REPO_INDEXER_INCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **include** in the index. Use `**.txt` to match any files with .txt extension. An empty list means include all files.
|
- `REPO_INDEXER_INCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **include** in the index. Use `**.txt` to match any files with .txt extension. An empty list means include all files.
|
||||||
- `REPO_INDEXER_EXCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **exclude** from the index. Files that match this list will not be indexed, even if they match in `REPO_INDEXER_INCLUDE`.
|
- `REPO_INDEXER_EXCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **exclude** from the index. Files that match this list will not be indexed, even if they match in `REPO_INDEXER_INCLUDE`.
|
||||||
- `REPO_INDEXER_EXCLUDE_VENDORED`: **true**: Exclude vendored files from index.
|
- `REPO_INDEXER_EXCLUDE_VENDORED`: **true**: Exclude vendored files from index.
|
||||||
- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
|
- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||||
- `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
|
- `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
|
||||||
- `STARTUP_TIMEOUT`: **30s**: If the indexer takes longer than this timeout to start - fail. (This timeout will be added to the hammer time above for child processes - as bleve will not start until the previous parent is shutdown.) Set to zero to never timeout.
|
- `STARTUP_TIMEOUT`: **30s**: If the indexer takes longer than this timeout to start - fail. (This timeout will be added to the hammer time above for child processes - as bleve will not start until the previous parent is shutdown.) Set to zero to never timeout.
|
||||||
|
|
||||||
|
|
|
@ -28,10 +28,10 @@ var (
|
||||||
IssuePath string
|
IssuePath string
|
||||||
IssueConnStr string
|
IssueConnStr string
|
||||||
IssueIndexerName string
|
IssueIndexerName string
|
||||||
IssueQueueType string
|
IssueQueueType string // DEPRECATED - replaced by queue.issue_indexer
|
||||||
IssueQueueDir string
|
IssueQueueDir string // DEPRECATED - replaced by queue.issue_indexer
|
||||||
IssueQueueConnStr string
|
IssueQueueConnStr string // DEPRECATED - replaced by queue.issue_indexer
|
||||||
IssueQueueBatchNumber int
|
IssueQueueBatchNumber int // DEPRECATED - replaced by queue.issue_indexer
|
||||||
StartupTimeout time.Duration
|
StartupTimeout time.Duration
|
||||||
|
|
||||||
RepoIndexerEnabled bool
|
RepoIndexerEnabled bool
|
||||||
|
@ -39,20 +39,17 @@ var (
|
||||||
RepoPath string
|
RepoPath string
|
||||||
RepoConnStr string
|
RepoConnStr string
|
||||||
RepoIndexerName string
|
RepoIndexerName string
|
||||||
UpdateQueueLength int
|
UpdateQueueLength int // DEPRECATED - replaced by queue.issue_indexer
|
||||||
MaxIndexerFileSize int64
|
MaxIndexerFileSize int64
|
||||||
IncludePatterns []glob.Glob
|
IncludePatterns []glob.Glob
|
||||||
ExcludePatterns []glob.Glob
|
ExcludePatterns []glob.Glob
|
||||||
ExcludeVendored bool
|
ExcludeVendored bool
|
||||||
}{
|
}{
|
||||||
IssueType: "bleve",
|
IssueType: "bleve",
|
||||||
IssuePath: "indexers/issues.bleve",
|
IssuePath: "indexers/issues.bleve",
|
||||||
IssueConnStr: "",
|
IssueConnStr: "",
|
||||||
IssueIndexerName: "gitea_issues",
|
IssueIndexerName: "gitea_issues",
|
||||||
IssueQueueType: LevelQueueType,
|
IssueQueueType: LevelQueueType,
|
||||||
IssueQueueDir: "queues/common",
|
|
||||||
IssueQueueConnStr: "",
|
|
||||||
IssueQueueBatchNumber: 20,
|
|
||||||
|
|
||||||
RepoIndexerEnabled: false,
|
RepoIndexerEnabled: false,
|
||||||
RepoType: "bleve",
|
RepoType: "bleve",
|
||||||
|
@ -74,10 +71,12 @@ func newIndexerService() {
|
||||||
Indexer.IssueConnStr = sec.Key("ISSUE_INDEXER_CONN_STR").MustString(Indexer.IssueConnStr)
|
Indexer.IssueConnStr = sec.Key("ISSUE_INDEXER_CONN_STR").MustString(Indexer.IssueConnStr)
|
||||||
Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName)
|
Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName)
|
||||||
|
|
||||||
Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(LevelQueueType)
|
// The following settings are deprecated and can be overridden by settings in [queue] or [queue.issue_indexer]
|
||||||
Indexer.IssueQueueDir = filepath.ToSlash(sec.Key("ISSUE_INDEXER_QUEUE_DIR").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "queues/common"))))
|
Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString("")
|
||||||
|
Indexer.IssueQueueDir = filepath.ToSlash(sec.Key("ISSUE_INDEXER_QUEUE_DIR").MustString(""))
|
||||||
Indexer.IssueQueueConnStr = sec.Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString("")
|
Indexer.IssueQueueConnStr = sec.Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString("")
|
||||||
Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(20)
|
Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(0)
|
||||||
|
Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(0)
|
||||||
|
|
||||||
Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false)
|
Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false)
|
||||||
Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve")
|
Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve")
|
||||||
|
@ -91,7 +90,6 @@ func newIndexerService() {
|
||||||
Indexer.IncludePatterns = IndexerGlobFromString(sec.Key("REPO_INDEXER_INCLUDE").MustString(""))
|
Indexer.IncludePatterns = IndexerGlobFromString(sec.Key("REPO_INDEXER_INCLUDE").MustString(""))
|
||||||
Indexer.ExcludePatterns = IndexerGlobFromString(sec.Key("REPO_INDEXER_EXCLUDE").MustString(""))
|
Indexer.ExcludePatterns = IndexerGlobFromString(sec.Key("REPO_INDEXER_EXCLUDE").MustString(""))
|
||||||
Indexer.ExcludeVendored = sec.Key("REPO_INDEXER_EXCLUDE_VENDORED").MustBool(true)
|
Indexer.ExcludeVendored = sec.Key("REPO_INDEXER_EXCLUDE_VENDORED").MustBool(true)
|
||||||
Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20)
|
|
||||||
Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024)
|
Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024)
|
||||||
Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second)
|
Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second)
|
||||||
}
|
}
|
||||||
|
|
|
@ -65,7 +65,7 @@ func GetQueueSettings(name string) QueueSettings {
|
||||||
q.SetName = q.QueueName + Queue.SetName
|
q.SetName = q.QueueName + Queue.SetName
|
||||||
}
|
}
|
||||||
if !filepath.IsAbs(q.DataDir) {
|
if !filepath.IsAbs(q.DataDir) {
|
||||||
q.DataDir = filepath.Join(AppDataPath, q.DataDir)
|
q.DataDir = filepath.ToSlash(filepath.Join(AppDataPath, q.DataDir))
|
||||||
}
|
}
|
||||||
_, _ = sec.NewKey("DATADIR", q.DataDir)
|
_, _ = sec.NewKey("DATADIR", q.DataDir)
|
||||||
|
|
||||||
|
@ -98,6 +98,7 @@ func NewQueueService() {
|
||||||
Queue.QueueLength = sec.Key("LENGTH").MustInt(20)
|
Queue.QueueLength = sec.Key("LENGTH").MustInt(20)
|
||||||
Queue.BatchLength = sec.Key("BATCH_LENGTH").MustInt(20)
|
Queue.BatchLength = sec.Key("BATCH_LENGTH").MustInt(20)
|
||||||
Queue.ConnectionString = sec.Key("CONN_STR").MustString("")
|
Queue.ConnectionString = sec.Key("CONN_STR").MustString("")
|
||||||
|
defaultType := sec.Key("TYPE").String()
|
||||||
Queue.Type = sec.Key("TYPE").MustString("persistable-channel")
|
Queue.Type = sec.Key("TYPE").MustString("persistable-channel")
|
||||||
Queue.Network, Queue.Addresses, Queue.Password, Queue.DBIndex, _ = ParseQueueConnStr(Queue.ConnectionString)
|
Queue.Network, Queue.Addresses, Queue.Password, Queue.DBIndex, _ = ParseQueueConnStr(Queue.ConnectionString)
|
||||||
Queue.WrapIfNecessary = sec.Key("WRAP_IF_NECESSARY").MustBool(true)
|
Queue.WrapIfNecessary = sec.Key("WRAP_IF_NECESSARY").MustBool(true)
|
||||||
|
@ -117,7 +118,7 @@ func NewQueueService() {
|
||||||
for _, key := range section.Keys() {
|
for _, key := range section.Keys() {
|
||||||
sectionMap[key.Name()] = true
|
sectionMap[key.Name()] = true
|
||||||
}
|
}
|
||||||
if _, ok := sectionMap["TYPE"]; !ok {
|
if _, ok := sectionMap["TYPE"]; !ok && defaultType == "" {
|
||||||
switch Indexer.IssueQueueType {
|
switch Indexer.IssueQueueType {
|
||||||
case LevelQueueType:
|
case LevelQueueType:
|
||||||
_, _ = section.NewKey("TYPE", "level")
|
_, _ = section.NewKey("TYPE", "level")
|
||||||
|
@ -125,21 +126,23 @@ func NewQueueService() {
|
||||||
_, _ = section.NewKey("TYPE", "persistable-channel")
|
_, _ = section.NewKey("TYPE", "persistable-channel")
|
||||||
case RedisQueueType:
|
case RedisQueueType:
|
||||||
_, _ = section.NewKey("TYPE", "redis")
|
_, _ = section.NewKey("TYPE", "redis")
|
||||||
|
case "":
|
||||||
|
_, _ = section.NewKey("TYPE", "level")
|
||||||
default:
|
default:
|
||||||
log.Fatal("Unsupported indexer queue type: %v",
|
log.Fatal("Unsupported indexer queue type: %v",
|
||||||
Indexer.IssueQueueType)
|
Indexer.IssueQueueType)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, ok := sectionMap["LENGTH"]; !ok {
|
if _, ok := sectionMap["LENGTH"]; !ok && Indexer.UpdateQueueLength != 0 {
|
||||||
_, _ = section.NewKey("LENGTH", fmt.Sprintf("%d", Indexer.UpdateQueueLength))
|
_, _ = section.NewKey("LENGTH", fmt.Sprintf("%d", Indexer.UpdateQueueLength))
|
||||||
}
|
}
|
||||||
if _, ok := sectionMap["BATCH_LENGTH"]; !ok {
|
if _, ok := sectionMap["BATCH_LENGTH"]; !ok && Indexer.IssueQueueBatchNumber != 0 {
|
||||||
_, _ = section.NewKey("BATCH_LENGTH", fmt.Sprintf("%d", Indexer.IssueQueueBatchNumber))
|
_, _ = section.NewKey("BATCH_LENGTH", fmt.Sprintf("%d", Indexer.IssueQueueBatchNumber))
|
||||||
}
|
}
|
||||||
if _, ok := sectionMap["DATADIR"]; !ok {
|
if _, ok := sectionMap["DATADIR"]; !ok && Indexer.IssueQueueDir != "" {
|
||||||
_, _ = section.NewKey("DATADIR", Indexer.IssueQueueDir)
|
_, _ = section.NewKey("DATADIR", Indexer.IssueQueueDir)
|
||||||
}
|
}
|
||||||
if _, ok := sectionMap["CONN_STR"]; !ok {
|
if _, ok := sectionMap["CONN_STR"]; !ok && Indexer.IssueQueueConnStr != "" {
|
||||||
_, _ = section.NewKey("CONN_STR", Indexer.IssueQueueConnStr)
|
_, _ = section.NewKey("CONN_STR", Indexer.IssueQueueConnStr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue