Files
dygou_go/internal/remote/spool.go
T

200 lines
3.9 KiB
Go

package remote
import (
"bufio"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
)
// EventSpool is a bounded, non-blocking durable queue for remote events.
// Logging never waits for disk or network I/O. If the in-memory handoff is full,
// the event is dropped so the seckill path keeps priority.
type EventSpool struct {
mu sync.Mutex
path string
offsetPath string
file *os.File
queue chan LogEvent
wake chan struct{}
done chan struct{}
wg sync.WaitGroup
writeEnd int64
sentEnd int64
dropped atomic.Uint64
queued atomic.Uint64
closed atomic.Bool
}
func NewEventSpool(dir string) (*EventSpool, error) {
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, err
}
path := filepath.Join(dir, "events.ndjson")
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600)
if err != nil {
return nil, err
}
info, err := file.Stat()
if err != nil {
_ = file.Close()
return nil, err
}
s := &EventSpool{
path: path,
offsetPath: filepath.Join(dir, "events.offset"),
file: file,
queue: make(chan LogEvent, 8192),
wake: make(chan struct{}, 1),
done: make(chan struct{}),
writeEnd: info.Size(),
}
s.sentEnd = loadSpoolOffset(s.offsetPath, info.Size())
s.wg.Add(1)
go s.writer()
return s, nil
}
func (s *EventSpool) Enqueue(event LogEvent) bool {
if s == nil || s.closed.Load() {
return false
}
select {
case s.queue <- event:
s.queued.Add(1)
return true
default:
s.dropped.Add(1)
return false
}
}
func (s *EventSpool) Wake() <-chan struct{} {
if s == nil {
return nil
}
return s.wake
}
func (s *EventSpool) ReadPending(limit int) ([]LogEvent, int64, error) {
if s == nil {
return nil, 0, nil
}
if limit <= 0 || limit > 500 {
limit = 200
}
s.mu.Lock()
defer s.mu.Unlock()
if s.sentEnd >= s.writeEnd {
return nil, s.sentEnd, nil
}
length := s.writeEnd - s.sentEnd
buf := make([]byte, length)
n, err := s.file.ReadAt(buf, s.sentEnd)
if err != nil && !errors.Is(err, io.EOF) {
return nil, s.sentEnd, err
}
buf = buf[:n]
reader := bufio.NewReader(strings.NewReader(string(buf)))
events := make([]LogEvent, 0, limit)
consumed := int64(0)
for len(events) < limit {
line, err := reader.ReadBytes('\n')
if err != nil {
break
}
consumed += int64(len(line))
var event LogEvent
if json.Unmarshal(line, &event) == nil {
events = append(events, event)
}
}
return events, s.sentEnd + consumed, nil
}
func (s *EventSpool) Ack(offset int64) {
if s == nil || offset < 0 {
return
}
s.mu.Lock()
if offset > s.sentEnd {
s.sentEnd = offset
}
current := s.sentEnd
s.mu.Unlock()
_ = os.WriteFile(s.offsetPath, []byte(strconv.FormatInt(current, 10)), 0o600)
}
func (s *EventSpool) Dropped() uint64 {
if s == nil {
return 0
}
return s.dropped.Load()
}
func (s *EventSpool) Queued() uint64 {
if s == nil {
return 0
}
return s.queued.Load()
}
func (s *EventSpool) Close() error {
if s == nil || !s.closed.CompareAndSwap(false, true) {
return nil
}
close(s.done)
s.wg.Wait()
s.mu.Lock()
defer s.mu.Unlock()
return s.file.Close()
}
func (s *EventSpool) writer() {
defer s.wg.Done()
for {
select {
case <-s.done:
return
case event := <-s.queue:
s.queued.Add(^uint64(0))
data, err := json.Marshal(event)
if err != nil {
continue
}
data = append(data, '\n')
s.mu.Lock()
_, err = s.file.Write(data)
if err == nil {
s.writeEnd += int64(len(data))
}
s.mu.Unlock()
if err != nil {
continue
}
select {
case s.wake <- struct{}{}:
default:
}
}
}
}
func loadSpoolOffset(path string, fileSize int64) int64 {
data, err := os.ReadFile(path)
if err != nil {
return 0
}
value, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
if err != nil || value < 0 || value > fileSize {
return 0
}
return value
}