package remote import ( "bufio" "encoding/json" "errors" "io" "os" "path/filepath" "strconv" "strings" "sync" "sync/atomic" ) // EventSpool is a bounded, non-blocking durable queue for remote events. // Logging never waits for disk or network I/O. If the in-memory handoff is full, // the event is dropped so the seckill path keeps priority. type EventSpool struct { mu sync.Mutex path string offsetPath string file *os.File queue chan LogEvent wake chan struct{} done chan struct{} wg sync.WaitGroup writeEnd int64 sentEnd int64 dropped atomic.Uint64 queued atomic.Uint64 closed atomic.Bool } func NewEventSpool(dir string) (*EventSpool, error) { if err := os.MkdirAll(dir, 0o700); err != nil { return nil, err } path := filepath.Join(dir, "events.ndjson") file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600) if err != nil { return nil, err } info, err := file.Stat() if err != nil { _ = file.Close() return nil, err } s := &EventSpool{ path: path, offsetPath: filepath.Join(dir, "events.offset"), file: file, queue: make(chan LogEvent, 8192), wake: make(chan struct{}, 1), done: make(chan struct{}), writeEnd: info.Size(), } s.sentEnd = loadSpoolOffset(s.offsetPath, info.Size()) s.wg.Add(1) go s.writer() return s, nil } func (s *EventSpool) Enqueue(event LogEvent) bool { if s == nil || s.closed.Load() { return false } select { case s.queue <- event: s.queued.Add(1) return true default: s.dropped.Add(1) return false } } func (s *EventSpool) Wake() <-chan struct{} { if s == nil { return nil } return s.wake } func (s *EventSpool) ReadPending(limit int) ([]LogEvent, int64, error) { if s == nil { return nil, 0, nil } if limit <= 0 || limit > 500 { limit = 200 } s.mu.Lock() defer s.mu.Unlock() if s.sentEnd >= s.writeEnd { return nil, s.sentEnd, nil } length := s.writeEnd - s.sentEnd buf := make([]byte, length) n, err := s.file.ReadAt(buf, s.sentEnd) if err != nil && !errors.Is(err, io.EOF) { return nil, s.sentEnd, err } buf = buf[:n] reader := bufio.NewReader(strings.NewReader(string(buf))) events := make([]LogEvent, 0, limit) consumed := int64(0) for len(events) < limit { line, err := reader.ReadBytes('\n') if err != nil { break } consumed += int64(len(line)) var event LogEvent if json.Unmarshal(line, &event) == nil { events = append(events, event) } } return events, s.sentEnd + consumed, nil } func (s *EventSpool) Ack(offset int64) { if s == nil || offset < 0 { return } s.mu.Lock() if offset > s.sentEnd { s.sentEnd = offset } current := s.sentEnd s.mu.Unlock() _ = os.WriteFile(s.offsetPath, []byte(strconv.FormatInt(current, 10)), 0o600) } func (s *EventSpool) Dropped() uint64 { if s == nil { return 0 } return s.dropped.Load() } func (s *EventSpool) Queued() uint64 { if s == nil { return 0 } return s.queued.Load() } func (s *EventSpool) Close() error { if s == nil || !s.closed.CompareAndSwap(false, true) { return nil } close(s.done) s.wg.Wait() s.mu.Lock() defer s.mu.Unlock() return s.file.Close() } func (s *EventSpool) writer() { defer s.wg.Done() for { select { case <-s.done: return case event := <-s.queue: s.queued.Add(^uint64(0)) data, err := json.Marshal(event) if err != nil { continue } data = append(data, '\n') s.mu.Lock() _, err = s.file.Write(data) if err == nil { s.writeEnd += int64(len(data)) } s.mu.Unlock() if err != nil { continue } select { case s.wake <- struct{}{}: default: } } } } func loadSpoolOffset(path string, fileSize int64) int64 { data, err := os.ReadFile(path) if err != nil { return 0 } value, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) if err != nil || value < 0 || value > fileSize { return 0 } return value }