518 lines
12 KiB
Go
518 lines
12 KiB
Go
package remote
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"math/rand"
|
|
"net/http"
|
|
"net/url"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/gorilla/websocket"
|
|
|
|
"douyu_go/internal/constants"
|
|
"douyu_go/internal/types"
|
|
)
|
|
|
|
type AgentOptions struct {
|
|
DataDir string
|
|
ServerURL string
|
|
Token string
|
|
MachineID string
|
|
MachineName string
|
|
InstanceID string
|
|
Instance string
|
|
Version string
|
|
EnrollmentToken string
|
|
Snapshot func() Snapshot
|
|
Stop func() error
|
|
Cancel func() error
|
|
}
|
|
|
|
type Agent struct {
|
|
opts AgentOptions
|
|
spool *EventSpool
|
|
streamID string
|
|
|
|
ctx context.Context
|
|
cancel context.CancelFunc
|
|
wg sync.WaitGroup
|
|
|
|
mu sync.RWMutex
|
|
snapshot Snapshot
|
|
runCounter atomic.Uint64
|
|
seq atomic.Uint64
|
|
stateWake chan struct{}
|
|
writeMu sync.Mutex
|
|
connected atomic.Bool
|
|
lastError string
|
|
lastConnect time.Time
|
|
}
|
|
|
|
func NewAgent(opts AgentOptions) (*Agent, error) {
|
|
opts.ServerURL = strings.TrimSpace(opts.ServerURL)
|
|
opts.Token = strings.TrimSpace(opts.Token)
|
|
opts.EnrollmentToken = strings.TrimSpace(opts.EnrollmentToken)
|
|
opts.MachineName = strings.TrimSpace(opts.MachineName)
|
|
opts.Instance = strings.TrimSpace(opts.Instance)
|
|
if opts.ServerURL == "" {
|
|
return nil, errors.New("远端地址为空")
|
|
}
|
|
if opts.MachineName == "" {
|
|
opts.MachineName = opts.MachineID
|
|
}
|
|
if opts.Instance == "" {
|
|
opts.Instance = opts.InstanceID
|
|
}
|
|
if opts.MachineID == "" {
|
|
id, err := LoadOrCreateMachineID(opts.DataDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
opts.MachineID = id
|
|
}
|
|
if opts.InstanceID == "" {
|
|
opts.InstanceID = opts.Instance
|
|
}
|
|
spool, err := NewEventSpool(filepath.Join(strings.TrimSpace(opts.DataDir), "remote"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
streamID, err := randomID()
|
|
if err != nil {
|
|
_ = spool.Close()
|
|
return nil, err
|
|
}
|
|
now := time.Now()
|
|
agent := &Agent{
|
|
opts: opts,
|
|
spool: spool,
|
|
streamID: streamID,
|
|
stateWake: make(chan struct{}, 1),
|
|
snapshot: Snapshot{
|
|
MachineID: opts.MachineID,
|
|
MachineName: opts.MachineName,
|
|
InstanceID: opts.InstanceID,
|
|
InstanceName: opts.Instance,
|
|
Version: opts.Version,
|
|
State: "idle",
|
|
UpdatedAt: now,
|
|
},
|
|
}
|
|
agent.refreshSnapshot()
|
|
return agent, nil
|
|
}
|
|
|
|
func (a *Agent) Start(ctx context.Context) {
|
|
if a == nil {
|
|
return
|
|
}
|
|
a.ctx, a.cancel = context.WithCancel(ctx)
|
|
a.wg.Add(1)
|
|
go a.run()
|
|
}
|
|
|
|
func (a *Agent) Close() error {
|
|
if a == nil {
|
|
return nil
|
|
}
|
|
if a.cancel != nil {
|
|
a.cancel()
|
|
}
|
|
a.wg.Wait()
|
|
return a.spool.Close()
|
|
}
|
|
|
|
func (a *Agent) Observe(entry types.LogEntry) {
|
|
if a == nil || a.spool == nil {
|
|
return
|
|
}
|
|
occurred := time.Now()
|
|
if parsed, err := time.ParseInLocation("2006-01-02 15:04:05.000", entry.Time, constants.CST); err == nil {
|
|
occurred = parsed
|
|
}
|
|
a.mu.RLock()
|
|
runID := ""
|
|
if a.snapshot.Running {
|
|
runID = a.snapshot.RunID
|
|
}
|
|
a.mu.RUnlock()
|
|
a.spool.Enqueue(LogEvent{
|
|
MachineID: a.opts.MachineID,
|
|
MachineName: a.opts.MachineName,
|
|
InstanceID: a.opts.InstanceID,
|
|
InstanceName: a.opts.Instance,
|
|
StreamID: a.streamID,
|
|
Sequence: a.seq.Add(1),
|
|
Time: occurred,
|
|
RunID: runID,
|
|
Category: entry.Category,
|
|
Level: entry.Level,
|
|
Account: entry.Account,
|
|
Message: stripLogPrefix(entry.Message),
|
|
})
|
|
}
|
|
|
|
func (a *Agent) SetState(state string) {
|
|
if a == nil {
|
|
return
|
|
}
|
|
state = strings.TrimSpace(state)
|
|
if state == "" {
|
|
return
|
|
}
|
|
a.mu.Lock()
|
|
running := state == "running" || state == "stopping"
|
|
if running && !a.snapshot.Running {
|
|
a.snapshot.RunID = fmt.Sprintf("%s-%s-%d", a.opts.InstanceID, time.Now().Format("20060102_150405"), a.runCounter.Add(1))
|
|
}
|
|
a.snapshot.State = state
|
|
a.snapshot.Running = running
|
|
if state == "idle" {
|
|
a.snapshot.Running = false
|
|
}
|
|
a.mu.Unlock()
|
|
a.refreshSnapshot()
|
|
select {
|
|
case a.stateWake <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
func (a *Agent) Connected() bool {
|
|
return a != nil && a.connected.Load()
|
|
}
|
|
|
|
func (a *Agent) Status() (bool, string, time.Time) {
|
|
if a == nil {
|
|
return false, "", time.Time{}
|
|
}
|
|
a.mu.RLock()
|
|
defer a.mu.RUnlock()
|
|
return a.connected.Load(), a.lastError, a.lastConnect
|
|
}
|
|
|
|
func (a *Agent) run() {
|
|
defer a.wg.Done()
|
|
backoff := 2 * time.Second
|
|
for {
|
|
if a.ctx.Err() != nil {
|
|
return
|
|
}
|
|
startedAt := time.Now()
|
|
if err := a.ensureToken(); err != nil {
|
|
a.mu.Lock()
|
|
a.connected.Store(false)
|
|
a.lastError = err.Error()
|
|
a.mu.Unlock()
|
|
wait := jitter(backoff, 20)
|
|
timer := time.NewTimer(wait)
|
|
select {
|
|
case <-a.ctx.Done():
|
|
timer.Stop()
|
|
return
|
|
case <-timer.C:
|
|
}
|
|
continue
|
|
}
|
|
err := a.runConnection()
|
|
if a.ctx.Err() != nil {
|
|
return
|
|
}
|
|
if time.Since(startedAt) > 30*time.Second {
|
|
backoff = 2 * time.Second
|
|
}
|
|
a.mu.Lock()
|
|
a.connected.Store(false)
|
|
if err != nil {
|
|
a.lastError = err.Error()
|
|
}
|
|
a.mu.Unlock()
|
|
wait := jitter(backoff, 20)
|
|
timer := time.NewTimer(wait)
|
|
select {
|
|
case <-a.ctx.Done():
|
|
timer.Stop()
|
|
return
|
|
case <-timer.C:
|
|
}
|
|
if backoff < 30*time.Second {
|
|
backoff *= 2
|
|
if backoff > 30*time.Second {
|
|
backoff = 30 * time.Second
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (a *Agent) runConnection() error {
|
|
wsURL, err := agentWebSocketURL(a.opts.ServerURL)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
header := http.Header{}
|
|
header.Set("Authorization", "Bearer "+a.opts.Token)
|
|
dialer := websocket.Dialer{HandshakeTimeout: 8 * time.Second}
|
|
conn, response, err := dialer.DialContext(a.ctx, wsURL, header)
|
|
if err != nil {
|
|
if response != nil && response.StatusCode == http.StatusUnauthorized {
|
|
a.opts.Token = ""
|
|
clearAgentCredential(a.opts.DataDir)
|
|
return errors.New("远端拒绝了实例令牌,将重新自动注册")
|
|
}
|
|
return fmt.Errorf("连接远端失败: %w", err)
|
|
}
|
|
defer conn.Close()
|
|
conn.SetReadLimit(1 << 20)
|
|
_ = conn.SetReadDeadline(time.Now().Add(70 * time.Second))
|
|
conn.SetPongHandler(func(string) error {
|
|
return conn.SetReadDeadline(time.Now().Add(70 * time.Second))
|
|
})
|
|
|
|
a.mu.Lock()
|
|
a.connected.Store(true)
|
|
a.lastError = ""
|
|
a.lastConnect = time.Now()
|
|
a.mu.Unlock()
|
|
defer a.connected.Store(false)
|
|
|
|
if err := a.writeEnvelope(conn, Envelope{Type: MessageHello, Hello: a.currentSnapshot()}); err != nil {
|
|
return err
|
|
}
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
a.readLoop(conn)
|
|
}()
|
|
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
heartbeat := time.NewTicker(5 * time.Second)
|
|
defer heartbeat.Stop()
|
|
pingTicker := time.NewTicker(25 * time.Second)
|
|
defer pingTicker.Stop()
|
|
for {
|
|
if err := a.flushEvents(conn); err != nil {
|
|
return err
|
|
}
|
|
select {
|
|
case <-a.ctx.Done():
|
|
return a.ctx.Err()
|
|
case <-done:
|
|
return errors.New("远端连接已断开")
|
|
case <-a.stateWake:
|
|
if err := a.writeEnvelope(conn, Envelope{Type: MessageSnapshot, Snapshot: a.currentSnapshot()}); err != nil {
|
|
return err
|
|
}
|
|
case <-a.spool.Wake():
|
|
case <-ticker.C:
|
|
if err := a.flushEvents(conn); err != nil {
|
|
return err
|
|
}
|
|
case <-heartbeat.C:
|
|
if err := a.writeEnvelope(conn, Envelope{Type: MessageHeartbeat, Snapshot: a.currentSnapshot()}); err != nil {
|
|
return err
|
|
}
|
|
case <-pingTicker.C:
|
|
if err := conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(5*time.Second)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (a *Agent) readLoop(conn *websocket.Conn) {
|
|
for {
|
|
var envelope Envelope
|
|
if err := conn.ReadJSON(&envelope); err != nil {
|
|
return
|
|
}
|
|
if envelope.Type != MessageCommand || envelope.Command == nil {
|
|
continue
|
|
}
|
|
cmd := *envelope.Command
|
|
result := a.runCommand(cmd)
|
|
result.ID = cmd.ID
|
|
result.Type = cmd.Type
|
|
result.FinishedAt = time.Now()
|
|
_ = a.writeEnvelope(conn, Envelope{Type: MessageCommandResult, Result: &result})
|
|
}
|
|
}
|
|
|
|
func (a *Agent) runCommand(cmd Command) CommandResult {
|
|
if !cmd.ExpiresAt.IsZero() && time.Now().After(cmd.ExpiresAt) {
|
|
return CommandResult{Success: false, Message: "命令已过期"}
|
|
}
|
|
switch strings.TrimSpace(cmd.Type) {
|
|
case "ping":
|
|
return CommandResult{Success: true, Message: "pong"}
|
|
case "status":
|
|
a.refreshSnapshot()
|
|
return CommandResult{Success: true, Message: "状态已刷新"}
|
|
case "stop":
|
|
if a.opts.Stop == nil {
|
|
return CommandResult{Success: false, Message: "当前实例不支持停止命令"}
|
|
}
|
|
if err := a.opts.Stop(); err != nil {
|
|
return CommandResult{Success: false, Message: err.Error()}
|
|
}
|
|
return CommandResult{Success: true, Message: "停止命令已提交"}
|
|
case "cancel_schedule":
|
|
if a.opts.Cancel == nil {
|
|
return CommandResult{Success: false, Message: "当前实例不支持取消定时"}
|
|
}
|
|
if err := a.opts.Cancel(); err != nil {
|
|
return CommandResult{Success: false, Message: err.Error()}
|
|
}
|
|
return CommandResult{Success: true, Message: "取消定时命令已提交"}
|
|
default:
|
|
return CommandResult{Success: false, Message: "不支持的命令"}
|
|
}
|
|
}
|
|
|
|
func (a *Agent) flushEvents(conn *websocket.Conn) error {
|
|
events, next, err := a.spool.ReadPending(200)
|
|
if err != nil || len(events) == 0 {
|
|
return err
|
|
}
|
|
if err := a.writeEnvelope(conn, Envelope{Type: MessageEvents, Events: events}); err != nil {
|
|
return err
|
|
}
|
|
a.spool.Ack(next)
|
|
return nil
|
|
}
|
|
|
|
func (a *Agent) writeEnvelope(conn *websocket.Conn, envelope Envelope) error {
|
|
a.writeMu.Lock()
|
|
defer a.writeMu.Unlock()
|
|
_ = conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
|
|
return conn.WriteJSON(envelope)
|
|
}
|
|
|
|
func (a *Agent) refreshSnapshot() {
|
|
a.mu.Lock()
|
|
next := a.snapshot
|
|
a.mu.Unlock()
|
|
if a.opts.Snapshot != nil {
|
|
partial := a.opts.Snapshot()
|
|
if partial.ActivityID != "" {
|
|
next.ActivityID = partial.ActivityID
|
|
}
|
|
next.Results = cloneResults(partial.Results)
|
|
next.AccountCount = partial.AccountCount
|
|
}
|
|
a.mu.Lock()
|
|
next.MachineID = a.opts.MachineID
|
|
next.MachineName = a.opts.MachineName
|
|
next.InstanceID = a.opts.InstanceID
|
|
next.InstanceName = a.opts.Instance
|
|
next.Version = a.opts.Version
|
|
next.State = a.snapshot.State
|
|
next.Running = a.snapshot.Running
|
|
next.RunID = a.snapshot.RunID
|
|
next.QueuedEvents = a.spool.Queued()
|
|
next.DroppedEvents = a.spool.Dropped()
|
|
next.UpdatedAt = time.Now()
|
|
a.snapshot = next
|
|
a.mu.Unlock()
|
|
}
|
|
|
|
func (a *Agent) currentSnapshot() *Snapshot {
|
|
a.refreshSnapshot()
|
|
a.mu.RLock()
|
|
defer a.mu.RUnlock()
|
|
copy := a.snapshot
|
|
copy.Results = cloneResults(a.snapshot.Results)
|
|
return ©
|
|
}
|
|
|
|
func cloneResults(source map[string]string) map[string]string {
|
|
if len(source) == 0 {
|
|
return nil
|
|
}
|
|
out := make(map[string]string, len(source))
|
|
for key, value := range source {
|
|
out[key] = value
|
|
}
|
|
return out
|
|
}
|
|
|
|
func agentWebSocketURL(raw string) (string, error) {
|
|
raw = strings.TrimSpace(raw)
|
|
if raw == "" {
|
|
return "", errors.New("远端地址为空")
|
|
}
|
|
u, err := url.Parse(raw)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
switch strings.ToLower(u.Scheme) {
|
|
case "http":
|
|
u.Scheme = "ws"
|
|
case "https":
|
|
u.Scheme = "wss"
|
|
case "ws", "wss":
|
|
default:
|
|
return "", errors.New("远端地址必须以 http://、https://、ws:// 或 wss:// 开头")
|
|
}
|
|
if strings.TrimSpace(u.Path) == "" || u.Path == "/" {
|
|
u.Path = "/ws/agent"
|
|
}
|
|
return u.String(), nil
|
|
}
|
|
|
|
// ValidateServerURL checks the configured control-center address.
|
|
func ValidateServerURL(raw string) error {
|
|
_, err := agentWebSocketURL(raw)
|
|
return err
|
|
}
|
|
|
|
func stripLogPrefix(message string) string {
|
|
message = strings.TrimSpace(message)
|
|
if index := strings.LastIndex(message, "] "); index >= 0 {
|
|
rest := message[index+2:]
|
|
if strings.HasPrefix(message, "20") {
|
|
return rest
|
|
}
|
|
}
|
|
return message
|
|
}
|
|
|
|
func jitter(d time.Duration, percent int64) time.Duration {
|
|
if d <= 0 {
|
|
return 0
|
|
}
|
|
span := int64(d) * percent / 100
|
|
if span <= 0 {
|
|
return d
|
|
}
|
|
return d + time.Duration(rand.Int63n(span))
|
|
}
|
|
|
|
func (a *Agent) ensureToken() error {
|
|
if strings.TrimSpace(a.opts.Token) != "" {
|
|
return nil
|
|
}
|
|
token, err := loadAgentCredential(a.opts.DataDir, a.opts.ServerURL, a.opts.MachineID, a.opts.InstanceID)
|
|
if err != nil {
|
|
return fmt.Errorf("读取实例凭据失败: %w", err)
|
|
}
|
|
if token == "" {
|
|
token, err = enrollAgent(a.opts.ServerURL, a.opts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := saveAgentCredential(a.opts.DataDir, a.opts.ServerURL, a.opts.MachineID, a.opts.InstanceID, token); err != nil {
|
|
return fmt.Errorf("保存实例凭据失败: %w", err)
|
|
}
|
|
}
|
|
a.opts.Token = token
|
|
return nil
|
|
}
|