62e6c9bc6c
* Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler <tystuyfzand@gmail.com>
349 lines
7.7 KiB
Go
Vendored
349 lines
7.7 KiB
Go
Vendored
package jsoniter
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
)
|
|
|
|
// ValueType the type for JSON element
|
|
type ValueType int
|
|
|
|
const (
|
|
// InvalidValue invalid JSON element
|
|
InvalidValue ValueType = iota
|
|
// StringValue JSON element "string"
|
|
StringValue
|
|
// NumberValue JSON element 100 or 0.10
|
|
NumberValue
|
|
// NilValue JSON element null
|
|
NilValue
|
|
// BoolValue JSON element true or false
|
|
BoolValue
|
|
// ArrayValue JSON element []
|
|
ArrayValue
|
|
// ObjectValue JSON element {}
|
|
ObjectValue
|
|
)
|
|
|
|
var hexDigits []byte
|
|
var valueTypes []ValueType
|
|
|
|
func init() {
|
|
hexDigits = make([]byte, 256)
|
|
for i := 0; i < len(hexDigits); i++ {
|
|
hexDigits[i] = 255
|
|
}
|
|
for i := '0'; i <= '9'; i++ {
|
|
hexDigits[i] = byte(i - '0')
|
|
}
|
|
for i := 'a'; i <= 'f'; i++ {
|
|
hexDigits[i] = byte((i - 'a') + 10)
|
|
}
|
|
for i := 'A'; i <= 'F'; i++ {
|
|
hexDigits[i] = byte((i - 'A') + 10)
|
|
}
|
|
valueTypes = make([]ValueType, 256)
|
|
for i := 0; i < len(valueTypes); i++ {
|
|
valueTypes[i] = InvalidValue
|
|
}
|
|
valueTypes['"'] = StringValue
|
|
valueTypes['-'] = NumberValue
|
|
valueTypes['0'] = NumberValue
|
|
valueTypes['1'] = NumberValue
|
|
valueTypes['2'] = NumberValue
|
|
valueTypes['3'] = NumberValue
|
|
valueTypes['4'] = NumberValue
|
|
valueTypes['5'] = NumberValue
|
|
valueTypes['6'] = NumberValue
|
|
valueTypes['7'] = NumberValue
|
|
valueTypes['8'] = NumberValue
|
|
valueTypes['9'] = NumberValue
|
|
valueTypes['t'] = BoolValue
|
|
valueTypes['f'] = BoolValue
|
|
valueTypes['n'] = NilValue
|
|
valueTypes['['] = ArrayValue
|
|
valueTypes['{'] = ObjectValue
|
|
}
|
|
|
|
// Iterator is a io.Reader like object, with JSON specific read functions.
|
|
// Error is not returned as return value, but stored as Error member on this iterator instance.
|
|
type Iterator struct {
|
|
cfg *frozenConfig
|
|
reader io.Reader
|
|
buf []byte
|
|
head int
|
|
tail int
|
|
depth int
|
|
captureStartedAt int
|
|
captured []byte
|
|
Error error
|
|
Attachment interface{} // open for customized decoder
|
|
}
|
|
|
|
// NewIterator creates an empty Iterator instance
|
|
func NewIterator(cfg API) *Iterator {
|
|
return &Iterator{
|
|
cfg: cfg.(*frozenConfig),
|
|
reader: nil,
|
|
buf: nil,
|
|
head: 0,
|
|
tail: 0,
|
|
depth: 0,
|
|
}
|
|
}
|
|
|
|
// Parse creates an Iterator instance from io.Reader
|
|
func Parse(cfg API, reader io.Reader, bufSize int) *Iterator {
|
|
return &Iterator{
|
|
cfg: cfg.(*frozenConfig),
|
|
reader: reader,
|
|
buf: make([]byte, bufSize),
|
|
head: 0,
|
|
tail: 0,
|
|
depth: 0,
|
|
}
|
|
}
|
|
|
|
// ParseBytes creates an Iterator instance from byte array
|
|
func ParseBytes(cfg API, input []byte) *Iterator {
|
|
return &Iterator{
|
|
cfg: cfg.(*frozenConfig),
|
|
reader: nil,
|
|
buf: input,
|
|
head: 0,
|
|
tail: len(input),
|
|
depth: 0,
|
|
}
|
|
}
|
|
|
|
// ParseString creates an Iterator instance from string
|
|
func ParseString(cfg API, input string) *Iterator {
|
|
return ParseBytes(cfg, []byte(input))
|
|
}
|
|
|
|
// Pool returns a pool can provide more iterator with same configuration
|
|
func (iter *Iterator) Pool() IteratorPool {
|
|
return iter.cfg
|
|
}
|
|
|
|
// Reset reuse iterator instance by specifying another reader
|
|
func (iter *Iterator) Reset(reader io.Reader) *Iterator {
|
|
iter.reader = reader
|
|
iter.head = 0
|
|
iter.tail = 0
|
|
iter.depth = 0
|
|
return iter
|
|
}
|
|
|
|
// ResetBytes reuse iterator instance by specifying another byte array as input
|
|
func (iter *Iterator) ResetBytes(input []byte) *Iterator {
|
|
iter.reader = nil
|
|
iter.buf = input
|
|
iter.head = 0
|
|
iter.tail = len(input)
|
|
iter.depth = 0
|
|
return iter
|
|
}
|
|
|
|
// WhatIsNext gets ValueType of relatively next json element
|
|
func (iter *Iterator) WhatIsNext() ValueType {
|
|
valueType := valueTypes[iter.nextToken()]
|
|
iter.unreadByte()
|
|
return valueType
|
|
}
|
|
|
|
func (iter *Iterator) skipWhitespacesWithoutLoadMore() bool {
|
|
for i := iter.head; i < iter.tail; i++ {
|
|
c := iter.buf[i]
|
|
switch c {
|
|
case ' ', '\n', '\t', '\r':
|
|
continue
|
|
}
|
|
iter.head = i
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (iter *Iterator) isObjectEnd() bool {
|
|
c := iter.nextToken()
|
|
if c == ',' {
|
|
return false
|
|
}
|
|
if c == '}' {
|
|
return true
|
|
}
|
|
iter.ReportError("isObjectEnd", "object ended prematurely, unexpected char "+string([]byte{c}))
|
|
return true
|
|
}
|
|
|
|
func (iter *Iterator) nextToken() byte {
|
|
// a variation of skip whitespaces, returning the next non-whitespace token
|
|
for {
|
|
for i := iter.head; i < iter.tail; i++ {
|
|
c := iter.buf[i]
|
|
switch c {
|
|
case ' ', '\n', '\t', '\r':
|
|
continue
|
|
}
|
|
iter.head = i + 1
|
|
return c
|
|
}
|
|
if !iter.loadMore() {
|
|
return 0
|
|
}
|
|
}
|
|
}
|
|
|
|
// ReportError record a error in iterator instance with current position.
|
|
func (iter *Iterator) ReportError(operation string, msg string) {
|
|
if iter.Error != nil {
|
|
if iter.Error != io.EOF {
|
|
return
|
|
}
|
|
}
|
|
peekStart := iter.head - 10
|
|
if peekStart < 0 {
|
|
peekStart = 0
|
|
}
|
|
peekEnd := iter.head + 10
|
|
if peekEnd > iter.tail {
|
|
peekEnd = iter.tail
|
|
}
|
|
parsing := string(iter.buf[peekStart:peekEnd])
|
|
contextStart := iter.head - 50
|
|
if contextStart < 0 {
|
|
contextStart = 0
|
|
}
|
|
contextEnd := iter.head + 50
|
|
if contextEnd > iter.tail {
|
|
contextEnd = iter.tail
|
|
}
|
|
context := string(iter.buf[contextStart:contextEnd])
|
|
iter.Error = fmt.Errorf("%s: %s, error found in #%v byte of ...|%s|..., bigger context ...|%s|...",
|
|
operation, msg, iter.head-peekStart, parsing, context)
|
|
}
|
|
|
|
// CurrentBuffer gets current buffer as string for debugging purpose
|
|
func (iter *Iterator) CurrentBuffer() string {
|
|
peekStart := iter.head - 10
|
|
if peekStart < 0 {
|
|
peekStart = 0
|
|
}
|
|
return fmt.Sprintf("parsing #%v byte, around ...|%s|..., whole buffer ...|%s|...", iter.head,
|
|
string(iter.buf[peekStart:iter.head]), string(iter.buf[0:iter.tail]))
|
|
}
|
|
|
|
func (iter *Iterator) readByte() (ret byte) {
|
|
if iter.head == iter.tail {
|
|
if iter.loadMore() {
|
|
ret = iter.buf[iter.head]
|
|
iter.head++
|
|
return ret
|
|
}
|
|
return 0
|
|
}
|
|
ret = iter.buf[iter.head]
|
|
iter.head++
|
|
return ret
|
|
}
|
|
|
|
func (iter *Iterator) loadMore() bool {
|
|
if iter.reader == nil {
|
|
if iter.Error == nil {
|
|
iter.head = iter.tail
|
|
iter.Error = io.EOF
|
|
}
|
|
return false
|
|
}
|
|
if iter.captured != nil {
|
|
iter.captured = append(iter.captured,
|
|
iter.buf[iter.captureStartedAt:iter.tail]...)
|
|
iter.captureStartedAt = 0
|
|
}
|
|
for {
|
|
n, err := iter.reader.Read(iter.buf)
|
|
if n == 0 {
|
|
if err != nil {
|
|
if iter.Error == nil {
|
|
iter.Error = err
|
|
}
|
|
return false
|
|
}
|
|
} else {
|
|
iter.head = 0
|
|
iter.tail = n
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
func (iter *Iterator) unreadByte() {
|
|
if iter.Error != nil {
|
|
return
|
|
}
|
|
iter.head--
|
|
return
|
|
}
|
|
|
|
// Read read the next JSON element as generic interface{}.
|
|
func (iter *Iterator) Read() interface{} {
|
|
valueType := iter.WhatIsNext()
|
|
switch valueType {
|
|
case StringValue:
|
|
return iter.ReadString()
|
|
case NumberValue:
|
|
if iter.cfg.configBeforeFrozen.UseNumber {
|
|
return json.Number(iter.readNumberAsString())
|
|
}
|
|
return iter.ReadFloat64()
|
|
case NilValue:
|
|
iter.skipFourBytes('n', 'u', 'l', 'l')
|
|
return nil
|
|
case BoolValue:
|
|
return iter.ReadBool()
|
|
case ArrayValue:
|
|
arr := []interface{}{}
|
|
iter.ReadArrayCB(func(iter *Iterator) bool {
|
|
var elem interface{}
|
|
iter.ReadVal(&elem)
|
|
arr = append(arr, elem)
|
|
return true
|
|
})
|
|
return arr
|
|
case ObjectValue:
|
|
obj := map[string]interface{}{}
|
|
iter.ReadMapCB(func(Iter *Iterator, field string) bool {
|
|
var elem interface{}
|
|
iter.ReadVal(&elem)
|
|
obj[field] = elem
|
|
return true
|
|
})
|
|
return obj
|
|
default:
|
|
iter.ReportError("Read", fmt.Sprintf("unexpected value type: %v", valueType))
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// limit maximum depth of nesting, as allowed by https://tools.ietf.org/html/rfc7159#section-9
|
|
const maxDepth = 10000
|
|
|
|
func (iter *Iterator) incrementDepth() (success bool) {
|
|
iter.depth++
|
|
if iter.depth <= maxDepth {
|
|
return true
|
|
}
|
|
iter.ReportError("incrementDepth", "exceeded max depth")
|
|
return false
|
|
}
|
|
|
|
func (iter *Iterator) decrementDepth() (success bool) {
|
|
iter.depth--
|
|
if iter.depth >= 0 {
|
|
return true
|
|
}
|
|
iter.ReportError("decrementDepth", "unexpected negative nesting")
|
|
return false
|
|
}
|