476 lines
11 KiB
Go
476 lines
11 KiB
Go
package remote
|
|
|
|
import (
|
|
"bufio"
|
|
"encoding/csv"
|
|
"encoding/json"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type EventExportFilter struct {
|
|
MachineID string
|
|
InstanceID string
|
|
RunID string
|
|
Query string
|
|
From time.Time
|
|
To time.Time
|
|
}
|
|
|
|
const (
|
|
maxServerEvents = 20000
|
|
maxRunRecords = 500
|
|
)
|
|
|
|
type Hub struct {
|
|
mu sync.RWMutex
|
|
dataDir string
|
|
events []ServerEvent
|
|
nextEvent uint64
|
|
instances map[string]InstanceState
|
|
runs map[string]RunRecord
|
|
commands map[string]CommandAudit
|
|
eventFile *os.File
|
|
runFile *os.File
|
|
}
|
|
|
|
func NewHub(dataDir string) (*Hub, error) {
|
|
if strings.TrimSpace(dataDir) == "" {
|
|
dataDir = "remote-data"
|
|
}
|
|
if err := os.MkdirAll(dataDir, 0o700); err != nil {
|
|
return nil, err
|
|
}
|
|
hub := &Hub{
|
|
dataDir: dataDir,
|
|
instances: make(map[string]InstanceState),
|
|
runs: make(map[string]RunRecord),
|
|
commands: make(map[string]CommandAudit),
|
|
}
|
|
if err := hub.loadEvents(); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := hub.loadRuns(); err != nil {
|
|
return nil, err
|
|
}
|
|
eventFile, err := os.OpenFile(filepath.Join(dataDir, "events.ndjson"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
runFile, err := os.OpenFile(filepath.Join(dataDir, "runs.ndjson"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
|
if err != nil {
|
|
_ = eventFile.Close()
|
|
return nil, err
|
|
}
|
|
hub.eventFile = eventFile
|
|
hub.runFile = runFile
|
|
return hub, nil
|
|
}
|
|
|
|
func (h *Hub) Close() error {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if h.eventFile != nil {
|
|
_ = h.eventFile.Close()
|
|
h.eventFile = nil
|
|
}
|
|
if h.runFile != nil {
|
|
_ = h.runFile.Close()
|
|
h.runFile = nil
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (h *Hub) ApplySnapshot(snapshot Snapshot, remoteAddr string) {
|
|
if snapshot.MachineID == "" || snapshot.InstanceID == "" {
|
|
return
|
|
}
|
|
now := time.Now()
|
|
if snapshot.UpdatedAt.IsZero() {
|
|
snapshot.UpdatedAt = now
|
|
}
|
|
key := instanceKey(snapshot.MachineID, snapshot.InstanceID)
|
|
h.mu.Lock()
|
|
state := InstanceState{
|
|
Snapshot: snapshot,
|
|
Online: true,
|
|
LastSeen: now,
|
|
RemoteAddr: remoteAddr,
|
|
}
|
|
h.instances[key] = state
|
|
h.updateRunLocked(snapshot, now)
|
|
h.mu.Unlock()
|
|
}
|
|
|
|
func (h *Hub) MarkOffline(machineID, instanceID string) {
|
|
key := instanceKey(machineID, instanceID)
|
|
h.mu.Lock()
|
|
if state, ok := h.instances[key]; ok {
|
|
state.Online = false
|
|
state.LastSeen = time.Now()
|
|
h.instances[key] = state
|
|
}
|
|
h.mu.Unlock()
|
|
}
|
|
|
|
func (h *Hub) AppendEvents(events []LogEvent) {
|
|
if len(events) == 0 {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
for _, event := range events {
|
|
h.nextEvent++
|
|
serverEvent := ServerEvent{Seq: h.nextEvent, LogEvent: event}
|
|
h.events = append(h.events, serverEvent)
|
|
if len(h.events) > maxServerEvents {
|
|
copy(h.events, h.events[len(h.events)-maxServerEvents:])
|
|
h.events = h.events[:maxServerEvents]
|
|
}
|
|
if h.eventFile != nil {
|
|
if data, err := json.Marshal(serverEvent); err == nil {
|
|
_, _ = h.eventFile.Write(append(data, '\n'))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (h *Hub) AddCommand(audit CommandAudit) {
|
|
h.mu.Lock()
|
|
h.commands[audit.ID] = audit
|
|
h.mu.Unlock()
|
|
}
|
|
|
|
func (h *Hub) MarkCommandDelivered(id string) {
|
|
h.mu.Lock()
|
|
audit, ok := h.commands[id]
|
|
if !ok {
|
|
h.mu.Unlock()
|
|
return
|
|
}
|
|
audit.State = "delivered"
|
|
audit.DeliveredAt = time.Now()
|
|
h.commands[id] = audit
|
|
h.mu.Unlock()
|
|
}
|
|
|
|
func (h *Hub) ApplyCommandResult(result CommandResult) {
|
|
h.mu.Lock()
|
|
audit, ok := h.commands[result.ID]
|
|
if !ok {
|
|
h.mu.Unlock()
|
|
return
|
|
}
|
|
audit.State = "failed"
|
|
if result.Success {
|
|
audit.State = "success"
|
|
}
|
|
audit.Message = result.Message
|
|
audit.FinishedAt = result.FinishedAt
|
|
h.commands[result.ID] = audit
|
|
h.mu.Unlock()
|
|
}
|
|
|
|
func (h *Hub) Dashboard() DashboardState {
|
|
h.mu.RLock()
|
|
defer h.mu.RUnlock()
|
|
instances := make([]InstanceState, 0, len(h.instances))
|
|
for _, state := range h.instances {
|
|
if time.Since(state.LastSeen) > 45*time.Second {
|
|
state.Online = false
|
|
}
|
|
instances = append(instances, state)
|
|
}
|
|
sort.Slice(instances, func(i, j int) bool {
|
|
if instances[i].MachineName == instances[j].MachineName {
|
|
return instances[i].InstanceName < instances[j].InstanceName
|
|
}
|
|
return instances[i].MachineName < instances[j].MachineName
|
|
})
|
|
runs := make([]RunRecord, 0, len(h.runs))
|
|
for _, run := range h.runs {
|
|
runs = append(runs, run)
|
|
}
|
|
sort.Slice(runs, func(i, j int) bool {
|
|
return runs[i].UpdatedAt.After(runs[j].UpdatedAt)
|
|
})
|
|
if len(runs) > maxRunRecords {
|
|
runs = runs[:maxRunRecords]
|
|
}
|
|
commands := make([]CommandAudit, 0, len(h.commands))
|
|
for _, command := range h.commands {
|
|
commands = append(commands, command)
|
|
}
|
|
sort.Slice(commands, func(i, j int) bool {
|
|
return commands[i].CreatedAt.After(commands[j].CreatedAt)
|
|
})
|
|
if len(commands) > 100 {
|
|
commands = commands[:100]
|
|
}
|
|
return DashboardState{
|
|
ServerTime: time.Now(),
|
|
Instances: instances,
|
|
Runs: runs,
|
|
Commands: commands,
|
|
}
|
|
}
|
|
|
|
func (h *Hub) EventsSince(since uint64, limit int) []ServerEvent {
|
|
return h.EventsSinceFiltered(since, limit, "", "", "")
|
|
}
|
|
|
|
func (h *Hub) EventsSinceFiltered(since uint64, limit int, machineID, instanceID, runID string) []ServerEvent {
|
|
if limit <= 0 || limit > 1000 {
|
|
limit = 300
|
|
}
|
|
h.mu.RLock()
|
|
defer h.mu.RUnlock()
|
|
out := make([]ServerEvent, 0, limit)
|
|
for _, event := range h.events {
|
|
if event.Seq <= since {
|
|
continue
|
|
}
|
|
if machineID != "" && event.MachineID != machineID {
|
|
continue
|
|
}
|
|
if instanceID != "" && event.InstanceID != instanceID {
|
|
continue
|
|
}
|
|
if runID != "" && event.RunID != runID {
|
|
continue
|
|
}
|
|
out = append(out, event)
|
|
if len(out) >= limit {
|
|
break
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (h *Hub) ExportEvents(w io.Writer, format string, filter EventExportFilter) (int, error) {
|
|
file, err := os.Open(filepath.Join(h.dataDir, "events.ndjson"))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer file.Close()
|
|
|
|
format = strings.ToLower(strings.TrimSpace(format))
|
|
var jsonFirst = true
|
|
var csvWriter *csv.Writer
|
|
if format == "json" {
|
|
if _, err := io.WriteString(w, "["); err != nil {
|
|
return 0, err
|
|
}
|
|
} else {
|
|
if _, err := w.Write([]byte{0xEF, 0xBB, 0xBF}); err != nil {
|
|
return 0, err
|
|
}
|
|
csvWriter = csv.NewWriter(w)
|
|
if err := csvWriter.Write([]string{
|
|
"seq", "time", "machine_id", "machine_name", "instance_id", "instance_name",
|
|
"run_id", "category", "level", "account", "message",
|
|
}); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
|
|
query := strings.ToLower(strings.TrimSpace(filter.Query))
|
|
scanner := bufio.NewScanner(file)
|
|
scanner.Buffer(make([]byte, 0, 64*1024), 2*1024*1024)
|
|
count := 0
|
|
for scanner.Scan() {
|
|
var event ServerEvent
|
|
if json.Unmarshal(scanner.Bytes(), &event) != nil || !matchesExport(event.LogEvent, filter, query) {
|
|
continue
|
|
}
|
|
count++
|
|
if format == "json" {
|
|
data, err := json.Marshal(event)
|
|
if err != nil {
|
|
return count, err
|
|
}
|
|
if !jsonFirst {
|
|
if _, err := io.WriteString(w, ","); err != nil {
|
|
return count, err
|
|
}
|
|
}
|
|
jsonFirst = false
|
|
if _, err := w.Write(data); err != nil {
|
|
return count, err
|
|
}
|
|
continue
|
|
}
|
|
if err := csvWriter.Write([]string{
|
|
strconv.FormatUint(event.Seq, 10),
|
|
event.Time.Format(time.RFC3339Nano),
|
|
event.MachineID,
|
|
event.MachineName,
|
|
event.InstanceID,
|
|
event.InstanceName,
|
|
event.RunID,
|
|
event.Category,
|
|
event.Level,
|
|
event.Account,
|
|
event.Message,
|
|
}); err != nil {
|
|
return count, err
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return count, err
|
|
}
|
|
if format == "json" {
|
|
if _, err := io.WriteString(w, "]"); err != nil {
|
|
return count, err
|
|
}
|
|
return count, nil
|
|
}
|
|
if csvWriter != nil {
|
|
csvWriter.Flush()
|
|
if err := csvWriter.Error(); err != nil {
|
|
return count, err
|
|
}
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
func matchesExport(event LogEvent, filter EventExportFilter, query string) bool {
|
|
if filter.MachineID != "" && event.MachineID != filter.MachineID {
|
|
return false
|
|
}
|
|
if filter.InstanceID != "" && event.InstanceID != filter.InstanceID {
|
|
return false
|
|
}
|
|
if filter.RunID != "" && event.RunID != filter.RunID {
|
|
return false
|
|
}
|
|
if !filter.From.IsZero() && event.Time.Before(filter.From) {
|
|
return false
|
|
}
|
|
if !filter.To.IsZero() && event.Time.After(filter.To) {
|
|
return false
|
|
}
|
|
if query != "" {
|
|
haystack := strings.ToLower(strings.Join([]string{
|
|
event.MachineName, event.InstanceName, event.Account, event.Message,
|
|
}, " "))
|
|
if !strings.Contains(haystack, query) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (h *Hub) updateRunLocked(snapshot Snapshot, now time.Time) {
|
|
if snapshot.RunID == "" {
|
|
return
|
|
}
|
|
run, ok := h.runs[snapshot.RunID]
|
|
if !ok {
|
|
run = RunRecord{
|
|
RunID: snapshot.RunID,
|
|
MachineID: snapshot.MachineID,
|
|
MachineName: snapshot.MachineName,
|
|
InstanceID: snapshot.InstanceID,
|
|
InstanceName: snapshot.InstanceName,
|
|
StartedAt: now,
|
|
}
|
|
}
|
|
run.State = snapshot.State
|
|
run.Running = snapshot.Running
|
|
run.Results = cloneResults(snapshot.Results)
|
|
run.UpdatedAt = now
|
|
if !snapshot.Running && run.FinishedAt.IsZero() {
|
|
run.FinishedAt = now
|
|
}
|
|
h.runs[snapshot.RunID] = run
|
|
if len(h.runs) > maxRunRecords {
|
|
h.pruneRunsLocked()
|
|
}
|
|
if h.runFile != nil {
|
|
if data, err := json.Marshal(run); err == nil {
|
|
_, _ = h.runFile.Write(append(data, '\n'))
|
|
}
|
|
}
|
|
}
|
|
|
|
func (h *Hub) pruneRunsLocked() {
|
|
type entry struct {
|
|
id string
|
|
at time.Time
|
|
}
|
|
items := make([]entry, 0, len(h.runs))
|
|
for id, run := range h.runs {
|
|
items = append(items, entry{id: id, at: run.UpdatedAt})
|
|
}
|
|
sort.Slice(items, func(i, j int) bool {
|
|
return items[i].at.Before(items[j].at)
|
|
})
|
|
for len(h.runs) > maxRunRecords && len(items) > 0 {
|
|
delete(h.runs, items[0].id)
|
|
items = items[1:]
|
|
}
|
|
}
|
|
|
|
func (h *Hub) loadEvents() error {
|
|
file, err := os.Open(filepath.Join(h.dataDir, "events.ndjson"))
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
scanner := bufio.NewScanner(file)
|
|
buf := make([]byte, 0, 64*1024)
|
|
scanner.Buffer(buf, 2*1024*1024)
|
|
for scanner.Scan() {
|
|
var event ServerEvent
|
|
if json.Unmarshal(scanner.Bytes(), &event) != nil {
|
|
continue
|
|
}
|
|
if event.Seq > h.nextEvent {
|
|
h.nextEvent = event.Seq
|
|
}
|
|
h.events = append(h.events, event)
|
|
if len(h.events) > maxServerEvents {
|
|
copy(h.events, h.events[len(h.events)-maxServerEvents:])
|
|
h.events = h.events[:maxServerEvents]
|
|
}
|
|
}
|
|
return scanner.Err()
|
|
}
|
|
|
|
func (h *Hub) loadRuns() error {
|
|
file, err := os.Open(filepath.Join(h.dataDir, "runs.ndjson"))
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
scanner := bufio.NewScanner(file)
|
|
for scanner.Scan() {
|
|
var run RunRecord
|
|
if json.Unmarshal(scanner.Bytes(), &run) != nil || run.RunID == "" {
|
|
continue
|
|
}
|
|
h.runs[run.RunID] = run
|
|
}
|
|
if len(h.runs) > maxRunRecords {
|
|
h.pruneRunsLocked()
|
|
}
|
|
return scanner.Err()
|
|
}
|
|
|
|
func instanceKey(machineID, instanceID string) string {
|
|
return machineID + "\x00" + instanceID
|
|
}
|