15a475b7db
* Some changes to fix recovery * Move Recovery to middlewares * Remove trace code * Fix lint * add session middleware and remove dependent on macaron for sso * Fix panic 500 page rendering * Fix bugs * Fix fmt * Fix vendor * recover unnecessary change * Fix lint and addd some comments about the copied codes. * Use util.StatDir instead of com.StatDir Co-authored-by: 6543 <6543@obermui.de>
403 lines
12 KiB
Go
Vendored
403 lines
12 KiB
Go
Vendored
package memcached
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"fmt"
|
|
"github.com/couchbase/gomemcached"
|
|
"math"
|
|
)
|
|
|
|
type SystemEventType int
|
|
|
|
const InvalidSysEvent SystemEventType = -1
|
|
|
|
const (
|
|
CollectionCreate SystemEventType = 0
|
|
CollectionDrop SystemEventType = iota
|
|
CollectionFlush SystemEventType = iota // KV did not implement
|
|
ScopeCreate SystemEventType = iota
|
|
ScopeDrop SystemEventType = iota
|
|
CollectionChanged SystemEventType = iota
|
|
)
|
|
|
|
type ScopeCreateEvent interface {
|
|
GetSystemEventName() (string, error)
|
|
GetScopeId() (uint32, error)
|
|
GetManifestId() (uint64, error)
|
|
}
|
|
|
|
type CollectionCreateEvent interface {
|
|
GetSystemEventName() (string, error)
|
|
GetScopeId() (uint32, error)
|
|
GetCollectionId() (uint32, error)
|
|
GetManifestId() (uint64, error)
|
|
GetMaxTTL() (uint32, error)
|
|
}
|
|
|
|
type CollectionDropEvent interface {
|
|
GetScopeId() (uint32, error)
|
|
GetCollectionId() (uint32, error)
|
|
GetManifestId() (uint64, error)
|
|
}
|
|
|
|
type ScopeDropEvent interface {
|
|
GetScopeId() (uint32, error)
|
|
GetManifestId() (uint64, error)
|
|
}
|
|
|
|
type CollectionChangedEvent interface {
|
|
GetCollectionId() (uint32, error)
|
|
GetManifestId() (uint64, error)
|
|
GetMaxTTL() (uint32, error)
|
|
}
|
|
|
|
var ErrorInvalidOp error = fmt.Errorf("Invalid Operation")
|
|
var ErrorInvalidVersion error = fmt.Errorf("Invalid version for parsing")
|
|
var ErrorValueTooShort error = fmt.Errorf("Value length is too short")
|
|
var ErrorNoMaxTTL error = fmt.Errorf("This event has no max TTL")
|
|
|
|
// UprEvent memcached events for UPR streams.
|
|
type UprEvent struct {
|
|
Opcode gomemcached.CommandCode // Type of event
|
|
Status gomemcached.Status // Response status
|
|
VBucket uint16 // VBucket this event applies to
|
|
DataType uint8 // data type
|
|
Opaque uint16 // 16 MSB of opaque
|
|
VBuuid uint64 // This field is set by downstream
|
|
Flags uint32 // Item flags
|
|
Expiry uint32 // Item expiration time
|
|
Key, Value []byte // Item key/value
|
|
OldValue []byte // TODO: TBD: old document value
|
|
Cas uint64 // CAS value of the item
|
|
Seqno uint64 // sequence number of the mutation
|
|
RevSeqno uint64 // rev sequence number : deletions
|
|
LockTime uint32 // Lock time
|
|
MetadataSize uint16 // Metadata size
|
|
SnapstartSeq uint64 // start sequence number of this snapshot
|
|
SnapendSeq uint64 // End sequence number of the snapshot
|
|
SnapshotType uint32 // 0: disk 1: memory
|
|
FailoverLog *FailoverLog // Failover log containing vvuid and sequnce number
|
|
Error error // Error value in case of a failure
|
|
ExtMeta []byte // Extended Metadata
|
|
AckSize uint32 // The number of bytes that can be Acked to DCP
|
|
SystemEvent SystemEventType // Only valid if IsSystemEvent() is true
|
|
SysEventVersion uint8 // Based on the version, the way Extra bytes is parsed is different
|
|
ValueLen int // Cache it to avoid len() calls for performance
|
|
CollectionId uint32 // Valid if Collection is in use
|
|
StreamId *uint16 // Nil if not in use
|
|
}
|
|
|
|
// FailoverLog containing vvuid and sequnce number
|
|
type FailoverLog [][2]uint64
|
|
|
|
// Containing a pair of vbno and the high seqno
|
|
type VBSeqnos [][2]uint64
|
|
|
|
func makeUprEvent(rq gomemcached.MCRequest, stream *UprStream, bytesReceivedFromDCP int) *UprEvent {
|
|
event := &UprEvent{
|
|
Opcode: rq.Opcode,
|
|
VBucket: stream.Vbucket,
|
|
VBuuid: stream.Vbuuid,
|
|
Value: rq.Body,
|
|
Cas: rq.Cas,
|
|
ExtMeta: rq.ExtMeta,
|
|
DataType: rq.DataType,
|
|
ValueLen: len(rq.Body),
|
|
SystemEvent: InvalidSysEvent,
|
|
CollectionId: math.MaxUint32,
|
|
}
|
|
|
|
event.PopulateFieldsBasedOnStreamType(rq, stream.StreamType)
|
|
|
|
// set AckSize for events that need to be acked to DCP,
|
|
// i.e., events with CommandCodes that need to be buffered in DCP
|
|
if _, ok := gomemcached.BufferedCommandCodeMap[rq.Opcode]; ok {
|
|
event.AckSize = uint32(bytesReceivedFromDCP)
|
|
}
|
|
|
|
// 16 LSBits are used by client library to encode vbucket number.
|
|
// 16 MSBits are left for application to multiplex on opaque value.
|
|
event.Opaque = appOpaque(rq.Opaque)
|
|
|
|
if len(rq.Extras) >= uprMutationExtraLen &&
|
|
event.Opcode == gomemcached.UPR_MUTATION {
|
|
|
|
event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
|
|
event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
|
|
event.Flags = binary.BigEndian.Uint32(rq.Extras[16:20])
|
|
event.Expiry = binary.BigEndian.Uint32(rq.Extras[20:24])
|
|
event.LockTime = binary.BigEndian.Uint32(rq.Extras[24:28])
|
|
event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[28:30])
|
|
|
|
} else if len(rq.Extras) >= uprDeletetionWithDeletionTimeExtraLen &&
|
|
event.Opcode == gomemcached.UPR_DELETION {
|
|
|
|
event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
|
|
event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
|
|
event.Expiry = binary.BigEndian.Uint32(rq.Extras[16:20])
|
|
|
|
} else if len(rq.Extras) >= uprDeletetionExtraLen &&
|
|
event.Opcode == gomemcached.UPR_DELETION ||
|
|
event.Opcode == gomemcached.UPR_EXPIRATION {
|
|
|
|
event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
|
|
event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
|
|
event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[16:18])
|
|
|
|
} else if len(rq.Extras) >= uprSnapshotExtraLen &&
|
|
event.Opcode == gomemcached.UPR_SNAPSHOT {
|
|
|
|
event.SnapstartSeq = binary.BigEndian.Uint64(rq.Extras[:8])
|
|
event.SnapendSeq = binary.BigEndian.Uint64(rq.Extras[8:16])
|
|
event.SnapshotType = binary.BigEndian.Uint32(rq.Extras[16:20])
|
|
} else if event.IsSystemEvent() {
|
|
event.PopulateEvent(rq.Extras)
|
|
} else if event.IsSeqnoAdv() {
|
|
event.PopulateSeqnoAdv(rq.Extras)
|
|
} else if event.IsOsoSnapshot() {
|
|
event.PopulateOso(rq.Extras)
|
|
}
|
|
|
|
return event
|
|
}
|
|
|
|
func (event *UprEvent) PopulateFieldsBasedOnStreamType(rq gomemcached.MCRequest, streamType DcpStreamType) {
|
|
switch streamType {
|
|
case CollectionsStreamId:
|
|
for _, extra := range rq.FramingExtras {
|
|
streamId, streamIdErr := extra.GetStreamId()
|
|
if streamIdErr == nil {
|
|
event.StreamId = &streamId
|
|
}
|
|
}
|
|
// After parsing streamID, still need to populate regular collectionID
|
|
fallthrough
|
|
case CollectionsNonStreamId:
|
|
switch rq.Opcode {
|
|
// Only these will have CID encoded within the key
|
|
case gomemcached.UPR_MUTATION,
|
|
gomemcached.UPR_DELETION,
|
|
gomemcached.UPR_EXPIRATION:
|
|
uleb128 := Uleb128(rq.Key)
|
|
result, bytesShifted := uleb128.ToUint32(rq.Keylen)
|
|
event.CollectionId = result
|
|
event.Key = rq.Key[bytesShifted:]
|
|
default:
|
|
event.Key = rq.Key
|
|
}
|
|
case NonCollectionStream:
|
|
// Let default behavior be legacy stream type
|
|
fallthrough
|
|
default:
|
|
event.Key = rq.Key
|
|
}
|
|
}
|
|
|
|
func (event *UprEvent) String() string {
|
|
name := gomemcached.CommandNames[event.Opcode]
|
|
if name == "" {
|
|
name = fmt.Sprintf("#%d", event.Opcode)
|
|
}
|
|
return name
|
|
}
|
|
|
|
func (event *UprEvent) IsSnappyDataType() bool {
|
|
return event.Opcode == gomemcached.UPR_MUTATION && (event.DataType&SnappyDataType > 0)
|
|
}
|
|
|
|
func (event *UprEvent) IsCollectionType() bool {
|
|
return event.IsSystemEvent() || event.CollectionId <= math.MaxUint32
|
|
}
|
|
|
|
func (event *UprEvent) IsSystemEvent() bool {
|
|
return event.Opcode == gomemcached.DCP_SYSTEM_EVENT
|
|
}
|
|
|
|
func (event *UprEvent) IsSeqnoAdv() bool {
|
|
return event.Opcode == gomemcached.DCP_SEQNO_ADV
|
|
}
|
|
|
|
func (event *UprEvent) IsOsoSnapshot() bool {
|
|
return event.Opcode == gomemcached.DCP_OSO_SNAPSHOT
|
|
}
|
|
|
|
func (event *UprEvent) PopulateEvent(extras []byte) {
|
|
if len(extras) < dcpSystemEventExtraLen {
|
|
// Wrong length, don't parse
|
|
return
|
|
}
|
|
|
|
event.Seqno = binary.BigEndian.Uint64(extras[:8])
|
|
event.SystemEvent = SystemEventType(binary.BigEndian.Uint32(extras[8:12]))
|
|
var versionTemp uint16 = binary.BigEndian.Uint16(extras[12:14])
|
|
event.SysEventVersion = uint8(versionTemp >> 8)
|
|
}
|
|
|
|
func (event *UprEvent) PopulateSeqnoAdv(extras []byte) {
|
|
if len(extras) < dcpSeqnoAdvExtraLen {
|
|
// Wrong length, don't parse
|
|
return
|
|
}
|
|
|
|
event.Seqno = binary.BigEndian.Uint64(extras[:8])
|
|
}
|
|
|
|
func (event *UprEvent) PopulateOso(extras []byte) {
|
|
if len(extras) < dcpOsoExtraLen {
|
|
// Wrong length, don't parse
|
|
return
|
|
}
|
|
event.Flags = binary.BigEndian.Uint32(extras[:4])
|
|
}
|
|
|
|
func (event *UprEvent) GetSystemEventName() (string, error) {
|
|
switch event.SystemEvent {
|
|
case CollectionCreate:
|
|
fallthrough
|
|
case ScopeCreate:
|
|
return string(event.Key), nil
|
|
default:
|
|
return "", ErrorInvalidOp
|
|
}
|
|
}
|
|
|
|
func (event *UprEvent) GetManifestId() (uint64, error) {
|
|
switch event.SystemEvent {
|
|
// Version 0 only checks
|
|
case CollectionChanged:
|
|
fallthrough
|
|
case ScopeDrop:
|
|
fallthrough
|
|
case ScopeCreate:
|
|
fallthrough
|
|
case CollectionDrop:
|
|
if event.SysEventVersion > 0 {
|
|
return 0, ErrorInvalidVersion
|
|
}
|
|
fallthrough
|
|
case CollectionCreate:
|
|
// CollectionCreate supports version 1
|
|
if event.SysEventVersion > 1 {
|
|
return 0, ErrorInvalidVersion
|
|
}
|
|
if event.ValueLen < 8 {
|
|
return 0, ErrorValueTooShort
|
|
}
|
|
return binary.BigEndian.Uint64(event.Value[0:8]), nil
|
|
default:
|
|
return 0, ErrorInvalidOp
|
|
}
|
|
}
|
|
|
|
func (event *UprEvent) GetCollectionId() (uint32, error) {
|
|
switch event.SystemEvent {
|
|
case CollectionDrop:
|
|
if event.SysEventVersion > 0 {
|
|
return 0, ErrorInvalidVersion
|
|
}
|
|
fallthrough
|
|
case CollectionCreate:
|
|
if event.SysEventVersion > 1 {
|
|
return 0, ErrorInvalidVersion
|
|
}
|
|
if event.ValueLen < 16 {
|
|
return 0, ErrorValueTooShort
|
|
}
|
|
return binary.BigEndian.Uint32(event.Value[12:16]), nil
|
|
case CollectionChanged:
|
|
if event.SysEventVersion > 0 {
|
|
return 0, ErrorInvalidVersion
|
|
}
|
|
if event.ValueLen < 12 {
|
|
return 0, ErrorValueTooShort
|
|
}
|
|
return binary.BigEndian.Uint32(event.Value[8:12]), nil
|
|
default:
|
|
return 0, ErrorInvalidOp
|
|
}
|
|
}
|
|
|
|
func (event *UprEvent) GetScopeId() (uint32, error) {
|
|
switch event.SystemEvent {
|
|
// version 0 checks
|
|
case ScopeCreate:
|
|
fallthrough
|
|
case ScopeDrop:
|
|
fallthrough
|
|
case CollectionDrop:
|
|
if event.SysEventVersion > 0 {
|
|
return 0, ErrorInvalidVersion
|
|
}
|
|
fallthrough
|
|
case CollectionCreate:
|
|
// CollectionCreate could be either 0 or 1
|
|
if event.SysEventVersion > 1 {
|
|
return 0, ErrorInvalidVersion
|
|
}
|
|
if event.ValueLen < 12 {
|
|
return 0, ErrorValueTooShort
|
|
}
|
|
return binary.BigEndian.Uint32(event.Value[8:12]), nil
|
|
default:
|
|
return 0, ErrorInvalidOp
|
|
}
|
|
}
|
|
|
|
func (event *UprEvent) GetMaxTTL() (uint32, error) {
|
|
switch event.SystemEvent {
|
|
case CollectionCreate:
|
|
if event.SysEventVersion < 1 {
|
|
return 0, ErrorNoMaxTTL
|
|
}
|
|
if event.ValueLen < 20 {
|
|
return 0, ErrorValueTooShort
|
|
}
|
|
return binary.BigEndian.Uint32(event.Value[16:20]), nil
|
|
case CollectionChanged:
|
|
if event.SysEventVersion > 0 {
|
|
return 0, ErrorInvalidVersion
|
|
}
|
|
if event.ValueLen < 16 {
|
|
return 0, ErrorValueTooShort
|
|
}
|
|
return binary.BigEndian.Uint32(event.Value[12:16]), nil
|
|
default:
|
|
return 0, ErrorInvalidOp
|
|
}
|
|
}
|
|
|
|
// Only if error is nil:
|
|
// Returns true if event states oso begins
|
|
// Return false if event states oso ends
|
|
func (event *UprEvent) GetOsoBegin() (bool, error) {
|
|
if !event.IsOsoSnapshot() {
|
|
return false, ErrorInvalidOp
|
|
}
|
|
|
|
if event.Flags == 1 {
|
|
return true, nil
|
|
} else if event.Flags == 2 {
|
|
return false, nil
|
|
} else {
|
|
return false, ErrorInvalidOp
|
|
}
|
|
}
|
|
|
|
type Uleb128 []byte
|
|
|
|
func (u Uleb128) ToUint32(cachedLen int) (result uint32, bytesShifted int) {
|
|
var shift uint = 0
|
|
|
|
for curByte := 0; curByte < cachedLen; curByte++ {
|
|
oneByte := u[curByte]
|
|
last7Bits := 0x7f & oneByte
|
|
result |= uint32(last7Bits) << shift
|
|
bytesShifted++
|
|
if oneByte&0x80 == 0 {
|
|
break
|
|
}
|
|
shift += 7
|
|
}
|
|
|
|
return
|
|
}
|