200 lines
3.9 KiB
Go
200 lines
3.9 KiB
Go
package remote
|
|
|
|
import (
|
|
"bufio"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
)
|
|
|
|
// EventSpool is a bounded, non-blocking durable queue for remote events.
|
|
// Logging never waits for disk or network I/O. If the in-memory handoff is full,
|
|
// the event is dropped so the seckill path keeps priority.
|
|
type EventSpool struct {
|
|
mu sync.Mutex
|
|
path string
|
|
offsetPath string
|
|
file *os.File
|
|
queue chan LogEvent
|
|
wake chan struct{}
|
|
done chan struct{}
|
|
wg sync.WaitGroup
|
|
writeEnd int64
|
|
sentEnd int64
|
|
dropped atomic.Uint64
|
|
queued atomic.Uint64
|
|
closed atomic.Bool
|
|
}
|
|
|
|
func NewEventSpool(dir string) (*EventSpool, error) {
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return nil, err
|
|
}
|
|
path := filepath.Join(dir, "events.ndjson")
|
|
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
info, err := file.Stat()
|
|
if err != nil {
|
|
_ = file.Close()
|
|
return nil, err
|
|
}
|
|
s := &EventSpool{
|
|
path: path,
|
|
offsetPath: filepath.Join(dir, "events.offset"),
|
|
file: file,
|
|
queue: make(chan LogEvent, 8192),
|
|
wake: make(chan struct{}, 1),
|
|
done: make(chan struct{}),
|
|
writeEnd: info.Size(),
|
|
}
|
|
s.sentEnd = loadSpoolOffset(s.offsetPath, info.Size())
|
|
s.wg.Add(1)
|
|
go s.writer()
|
|
return s, nil
|
|
}
|
|
|
|
func (s *EventSpool) Enqueue(event LogEvent) bool {
|
|
if s == nil || s.closed.Load() {
|
|
return false
|
|
}
|
|
select {
|
|
case s.queue <- event:
|
|
s.queued.Add(1)
|
|
return true
|
|
default:
|
|
s.dropped.Add(1)
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (s *EventSpool) Wake() <-chan struct{} {
|
|
if s == nil {
|
|
return nil
|
|
}
|
|
return s.wake
|
|
}
|
|
|
|
func (s *EventSpool) ReadPending(limit int) ([]LogEvent, int64, error) {
|
|
if s == nil {
|
|
return nil, 0, nil
|
|
}
|
|
if limit <= 0 || limit > 500 {
|
|
limit = 200
|
|
}
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.sentEnd >= s.writeEnd {
|
|
return nil, s.sentEnd, nil
|
|
}
|
|
length := s.writeEnd - s.sentEnd
|
|
buf := make([]byte, length)
|
|
n, err := s.file.ReadAt(buf, s.sentEnd)
|
|
if err != nil && !errors.Is(err, io.EOF) {
|
|
return nil, s.sentEnd, err
|
|
}
|
|
buf = buf[:n]
|
|
reader := bufio.NewReader(strings.NewReader(string(buf)))
|
|
events := make([]LogEvent, 0, limit)
|
|
consumed := int64(0)
|
|
for len(events) < limit {
|
|
line, err := reader.ReadBytes('\n')
|
|
if err != nil {
|
|
break
|
|
}
|
|
consumed += int64(len(line))
|
|
var event LogEvent
|
|
if json.Unmarshal(line, &event) == nil {
|
|
events = append(events, event)
|
|
}
|
|
}
|
|
return events, s.sentEnd + consumed, nil
|
|
}
|
|
|
|
func (s *EventSpool) Ack(offset int64) {
|
|
if s == nil || offset < 0 {
|
|
return
|
|
}
|
|
s.mu.Lock()
|
|
if offset > s.sentEnd {
|
|
s.sentEnd = offset
|
|
}
|
|
current := s.sentEnd
|
|
s.mu.Unlock()
|
|
_ = os.WriteFile(s.offsetPath, []byte(strconv.FormatInt(current, 10)), 0o600)
|
|
}
|
|
|
|
func (s *EventSpool) Dropped() uint64 {
|
|
if s == nil {
|
|
return 0
|
|
}
|
|
return s.dropped.Load()
|
|
}
|
|
|
|
func (s *EventSpool) Queued() uint64 {
|
|
if s == nil {
|
|
return 0
|
|
}
|
|
return s.queued.Load()
|
|
}
|
|
|
|
func (s *EventSpool) Close() error {
|
|
if s == nil || !s.closed.CompareAndSwap(false, true) {
|
|
return nil
|
|
}
|
|
close(s.done)
|
|
s.wg.Wait()
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.file.Close()
|
|
}
|
|
|
|
func (s *EventSpool) writer() {
|
|
defer s.wg.Done()
|
|
for {
|
|
select {
|
|
case <-s.done:
|
|
return
|
|
case event := <-s.queue:
|
|
s.queued.Add(^uint64(0))
|
|
data, err := json.Marshal(event)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
data = append(data, '\n')
|
|
s.mu.Lock()
|
|
_, err = s.file.Write(data)
|
|
if err == nil {
|
|
s.writeEnd += int64(len(data))
|
|
}
|
|
s.mu.Unlock()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
select {
|
|
case s.wake <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func loadSpoolOffset(path string, fileSize int64) int64 {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
value, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
|
|
if err != nil || value < 0 || value > fileSize {
|
|
return 0
|
|
}
|
|
return value
|
|
}
|