package nodb import ( "bufio" "encoding/binary" "fmt" "io" "io/ioutil" "os" "path" "strconv" "strings" "sync" "time" "gitea.com/lunny/log" "gitea.com/lunny/nodb/config" ) type BinLogHead struct { CreateTime uint32 BatchId uint32 PayloadLen uint32 } func (h *BinLogHead) Len() int { return 12 } func (h *BinLogHead) Write(w io.Writer) error { if err := binary.Write(w, binary.BigEndian, h.CreateTime); err != nil { return err } if err := binary.Write(w, binary.BigEndian, h.BatchId); err != nil { return err } if err := binary.Write(w, binary.BigEndian, h.PayloadLen); err != nil { return err } return nil } func (h *BinLogHead) handleReadError(err error) error { if err == io.EOF { return io.ErrUnexpectedEOF } else { return err } } func (h *BinLogHead) Read(r io.Reader) error { var err error if err = binary.Read(r, binary.BigEndian, &h.CreateTime); err != nil { return err } if err = binary.Read(r, binary.BigEndian, &h.BatchId); err != nil { return h.handleReadError(err) } if err = binary.Read(r, binary.BigEndian, &h.PayloadLen); err != nil { return h.handleReadError(err) } return nil } func (h *BinLogHead) InSameBatch(ho *BinLogHead) bool { if h.CreateTime == ho.CreateTime && h.BatchId == ho.BatchId { return true } else { return false } } /* index file format: ledis-bin.00001 ledis-bin.00002 ledis-bin.00003 log file format Log: Head|PayloadData Head: createTime|batchId|payloadData */ type BinLog struct { sync.Mutex path string cfg *config.BinLogConfig logFile *os.File logWb *bufio.Writer indexName string logNames []string lastLogIndex int64 batchId uint32 ch chan struct{} } func NewBinLog(cfg *config.Config) (*BinLog, error) { l := new(BinLog) l.cfg = &cfg.BinLog l.cfg.Adjust() l.path = path.Join(cfg.DataDir, "binlog") if err := os.MkdirAll(l.path, os.ModePerm); err != nil { return nil, err } l.logNames = make([]string, 0, 16) l.ch = make(chan struct{}) if err := l.loadIndex(); err != nil { return nil, err } return l, nil } func (l *BinLog) flushIndex() error { data := strings.Join(l.logNames, "\n") bakName := fmt.Sprintf("%s.bak", l.indexName) f, err := os.OpenFile(bakName, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { log.Errorf("create binlog bak index error %s", err.Error()) return err } if _, err := f.WriteString(data); err != nil { log.Errorf("write binlog index error %s", err.Error()) f.Close() return err } f.Close() if err := os.Rename(bakName, l.indexName); err != nil { log.Errorf("rename binlog bak index error %s", err.Error()) return err } return nil } func (l *BinLog) loadIndex() error { l.indexName = path.Join(l.path, fmt.Sprintf("ledis-bin.index")) if _, err := os.Stat(l.indexName); os.IsNotExist(err) { //no index file, nothing to do } else { indexData, err := ioutil.ReadFile(l.indexName) if err != nil { return err } lines := strings.Split(string(indexData), "\n") for _, line := range lines { line = strings.Trim(line, "\r\n ") if len(line) == 0 { continue } if _, err := os.Stat(path.Join(l.path, line)); err != nil { log.Errorf("load index line %s error %s", line, err.Error()) return err } else { l.logNames = append(l.logNames, line) } } } if l.cfg.MaxFileNum > 0 && len(l.logNames) > l.cfg.MaxFileNum { //remove oldest logfile if err := l.Purge(len(l.logNames) - l.cfg.MaxFileNum); err != nil { return err } } var err error if len(l.logNames) == 0 { l.lastLogIndex = 1 } else { lastName := l.logNames[len(l.logNames)-1] if l.lastLogIndex, err = strconv.ParseInt(path.Ext(lastName)[1:], 10, 64); err != nil { log.Errorf("invalid logfile name %s", err.Error()) return err } //like mysql, if server restart, a new binlog will create l.lastLogIndex++ } return nil } func (l *BinLog) getLogFile() string { return l.FormatLogFileName(l.lastLogIndex) } func (l *BinLog) openNewLogFile() error { var err error lastName := l.getLogFile() logPath := path.Join(l.path, lastName) if l.logFile, err = os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0666); err != nil { log.Errorf("open new logfile error %s", err.Error()) return err } if l.cfg.MaxFileNum > 0 && len(l.logNames) == l.cfg.MaxFileNum { l.purge(1) } l.logNames = append(l.logNames, lastName) if l.logWb == nil { l.logWb = bufio.NewWriterSize(l.logFile, 1024) } else { l.logWb.Reset(l.logFile) } if err = l.flushIndex(); err != nil { return err } return nil } func (l *BinLog) checkLogFileSize() bool { if l.logFile == nil { return false } st, _ := l.logFile.Stat() if st.Size() >= int64(l.cfg.MaxFileSize) { l.closeLog() return true } return false } func (l *BinLog) closeLog() { l.lastLogIndex++ l.logFile.Close() l.logFile = nil } func (l *BinLog) purge(n int) { for i := 0; i < n; i++ { logPath := path.Join(l.path, l.logNames[i]) os.Remove(logPath) } copy(l.logNames[0:], l.logNames[n:]) l.logNames = l.logNames[0 : len(l.logNames)-n] } func (l *BinLog) Close() { if l.logFile != nil { l.logFile.Close() l.logFile = nil } } func (l *BinLog) LogNames() []string { return l.logNames } func (l *BinLog) LogFileName() string { return l.getLogFile() } func (l *BinLog) LogFilePos() int64 { if l.logFile == nil { return 0 } else { st, _ := l.logFile.Stat() return st.Size() } } func (l *BinLog) LogFileIndex() int64 { return l.lastLogIndex } func (l *BinLog) FormatLogFileName(index int64) string { return fmt.Sprintf("ledis-bin.%07d", index) } func (l *BinLog) FormatLogFilePath(index int64) string { return path.Join(l.path, l.FormatLogFileName(index)) } func (l *BinLog) LogPath() string { return l.path } func (l *BinLog) Purge(n int) error { l.Lock() defer l.Unlock() if len(l.logNames) == 0 { return nil } if n >= len(l.logNames) { n = len(l.logNames) //can not purge current log file if l.logNames[n-1] == l.getLogFile() { n = n - 1 } } l.purge(n) return l.flushIndex() } func (l *BinLog) PurgeAll() error { l.Lock() defer l.Unlock() l.closeLog() return l.openNewLogFile() } func (l *BinLog) Log(args ...[]byte) error { l.Lock() defer l.Unlock() var err error if l.logFile == nil { if err = l.openNewLogFile(); err != nil { return err } } head := &BinLogHead{} head.CreateTime = uint32(time.Now().Unix()) head.BatchId = l.batchId l.batchId++ for _, data := range args { head.PayloadLen = uint32(len(data)) if err := head.Write(l.logWb); err != nil { return err } if _, err := l.logWb.Write(data); err != nil { return err } } if err = l.logWb.Flush(); err != nil { log.Errorf("write log error %s", err.Error()) return err } l.checkLogFileSize() close(l.ch) l.ch = make(chan struct{}) return nil } func (l *BinLog) Wait() <-chan struct{} { return l.ch }