Files
live-hub-py/web/backend/services/integration/outbox.py
T
yml bba3d9dac9 feat(fulfillment): 增加签名 Webhook 事务 Outbox
- 新增直播履约状态事务 Outbox 与签名投递 worker
- 增加 order_site 状态回传、重试和低频对账兜底
- 补充 Outbox、签名校验与集成接口测试
2026-11-18 10:59:20 +08:00

323 lines
9.8 KiB
Python

"""数据库事务 Outbox 与签名 Webhook 投递 worker。"""
from __future__ import annotations
import json
import os
import threading
import time
import uuid
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from loguru import logger
from sqlalchemy.orm import Session
from ...database import SessionLocal
from ...models import IntegrationOutbox
from .security import build_webhook_signature
DEFAULT_INTERVAL_SECONDS = 1.0
DEFAULT_BATCH_SIZE = 20
DEFAULT_MAX_ATTEMPTS = 12
REQUEST_TIMEOUT_SECONDS = 5
PROCESSING_STALE_SECONDS = 120
_stop_event = threading.Event()
_worker_thread: threading.Thread | None = None
_lock = threading.Lock()
@dataclass(frozen=True)
class _ClaimedEvent:
event_id: str
payload_json: str
attempts: int
def enqueue_integration_event(
db: Session,
*,
destination: str,
event_type: str,
payload: dict[str, Any],
dedupe_key: str = "",
) -> bool:
"""把事件加入当前事务;调用方必须在业务状态同一事务内 commit。"""
event_id = str(uuid.uuid4())
body = {
"eventId": event_id,
"eventType": event_type,
"occurredAt": datetime.now(UTC).isoformat(),
**payload,
}
normalized_dedupe = dedupe_key.strip() or None
if normalized_dedupe:
# SessionLocal 关闭了 autoflush;先落当前事务内的业务变更和先前事件,
# 避免同一事务重复调用时看不到刚 add 的 outbox 行。
db.flush()
existing = (
db.query(IntegrationOutbox)
.filter(IntegrationOutbox.dedupe_key == normalized_dedupe)
.first()
)
if existing is not None and existing.status == "dead":
existing.status = "pending"
existing.attempts = 0
existing.next_attempt_at = datetime.now(UTC)
existing.locked_at = None
existing.last_error = ""
return True
if existing is not None:
return False
db.add(
IntegrationOutbox(
event_id=event_id,
destination=destination,
event_type=event_type,
dedupe_key=normalized_dedupe,
payload_json=body,
status="pending",
attempts=0,
next_attempt_at=datetime.now(UTC),
created_at=datetime.now(UTC),
)
)
return True
def outbox_backoff_seconds(attempts: int) -> int:
normalized = max(1, int(attempts or 1))
return min(3600, 5 * 2 ** min(normalized - 1, 10))
def _configured() -> tuple[str, str]:
base_url = os.getenv("ORDER_SITE_PLATFORM_FULFILLMENT_URL", "").strip().rstrip("/")
integration_key = os.getenv("PLATFORM_FULFILLMENT_INTEGRATION_KEY", "").strip()
return base_url, integration_key
def _env_float(name: str, default: float, minimum: float, maximum: float) -> float:
try:
value = float(os.getenv(name, str(default)))
except ValueError:
return default
return max(minimum, min(maximum, value))
def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return max(minimum, min(maximum, value))
def _claim_due_events(limit: int) -> list[_ClaimedEvent]:
db = SessionLocal()
now = datetime.now(UTC)
try:
(
db.query(IntegrationOutbox)
.filter(
IntegrationOutbox.status == "processing",
IntegrationOutbox.locked_at
< now - timedelta(seconds=PROCESSING_STALE_SECONDS),
)
.update(
{
IntegrationOutbox.status: "pending",
IntegrationOutbox.locked_at: None,
IntegrationOutbox.next_attempt_at: now,
},
synchronize_session=False,
)
)
rows = (
db.query(IntegrationOutbox)
.filter(
IntegrationOutbox.status == "pending",
IntegrationOutbox.next_attempt_at <= now,
)
.order_by(IntegrationOutbox.id.asc())
.with_for_update(skip_locked=True)
.limit(limit)
.all()
)
claimed: list[_ClaimedEvent] = []
for row in rows:
row.status = "processing"
row.locked_at = now
claimed.append(
_ClaimedEvent(
event_id=row.event_id,
payload_json=json.dumps(
row.payload_json or {},
ensure_ascii=False,
separators=(",", ":"),
),
attempts=int(row.attempts or 0),
)
)
db.commit()
return claimed
except Exception:
db.rollback()
raise
finally:
db.close()
def _deliver(event: _ClaimedEvent, base_url: str, integration_key: str) -> None:
timestamp = str(int(time.time()))
signature = build_webhook_signature(
integration_key,
timestamp,
event.payload_json.encode("utf-8"),
)
request = Request(
f"{base_url}/api/integration/platform-fulfillment/binding-state",
data=event.payload_json.encode("utf-8"),
method="POST",
headers={
"Content-Type": "application/json",
"X-Integration-Key": integration_key,
"X-Integration-Event-Id": event.event_id,
"X-Integration-Timestamp": timestamp,
"X-Integration-Signature": signature,
},
)
try:
with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
if not 200 <= response.status < 300:
raise RuntimeError(f"order_site 返回 HTTP {response.status}")
except HTTPError as exc:
raise RuntimeError(f"order_site 返回 HTTP {exc.code}") from exc
except (URLError, OSError) as exc:
raise RuntimeError(str(exc)) from exc
def _mark_sent(event_id: str) -> None:
db = SessionLocal()
try:
(
db.query(IntegrationOutbox)
.filter(IntegrationOutbox.event_id == event_id)
.update(
{
IntegrationOutbox.status: "sent",
IntegrationOutbox.sent_at: datetime.now(UTC),
IntegrationOutbox.locked_at: None,
IntegrationOutbox.last_error: "",
},
synchronize_session=False,
)
)
db.commit()
finally:
db.close()
def _mark_failed(event: _ClaimedEvent, error: Exception) -> None:
attempts = event.attempts + 1
max_attempts = _env_int(
"INTEGRATION_OUTBOX_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS, 1, 100
)
dead = attempts >= max_attempts
message = str(error)[:2000]
db = SessionLocal()
try:
(
db.query(IntegrationOutbox)
.filter(IntegrationOutbox.event_id == event.event_id)
.update(
{
IntegrationOutbox.status: "dead" if dead else "pending",
IntegrationOutbox.attempts: attempts,
IntegrationOutbox.locked_at: None,
IntegrationOutbox.next_attempt_at: datetime.now(UTC)
+ (
timedelta(seconds=0)
if dead
else timedelta(seconds=outbox_backoff_seconds(attempts))
),
IntegrationOutbox.last_error: message,
},
synchronize_session=False,
)
)
db.commit()
finally:
db.close()
if dead:
logger.error(
"集成 Outbox 事件超过最大重试次数 event_id={} attempts={} error={}",
event.event_id,
attempts,
message,
)
def _publish_due_events() -> None:
base_url, integration_key = _configured()
if not base_url or not integration_key:
return
batch_size = _env_int(
"INTEGRATION_OUTBOX_BATCH_SIZE", DEFAULT_BATCH_SIZE, 1, 100
)
for event in _claim_due_events(batch_size):
try:
_deliver(event, base_url, integration_key)
_mark_sent(event.event_id)
except Exception as exc: # noqa: BLE001 - 投递失败必须回落数据库重试
_mark_failed(event, exc)
def _run() -> None:
interval = _env_float(
"INTEGRATION_OUTBOX_INTERVAL_SECONDS",
DEFAULT_INTERVAL_SECONDS,
0.25,
60,
)
while not _stop_event.is_set():
try:
_publish_due_events()
except Exception as exc: # noqa: BLE001 - worker 不能因单轮异常退出
logger.warning("集成 Outbox 投递轮次失败: {}", exc)
_stop_event.wait(interval)
def start_integration_outbox_worker() -> bool:
global _worker_thread
with _lock:
if _worker_thread is not None and _worker_thread.is_alive():
return False
_stop_event.clear()
_worker_thread = threading.Thread(
target=_run,
name="integration-outbox-worker",
daemon=True,
)
_worker_thread.start()
base_url, integration_key = _configured()
if base_url and integration_key:
logger.info("事务 Outbox worker 已启动")
else:
logger.warning("未配置 order_site 地址或集成密钥,Outbox worker 仅保持待命")
return True
def stop_integration_outbox_worker() -> None:
global _worker_thread
_stop_event.set()
with _lock:
thread = _worker_thread
_worker_thread = None
if thread is not None and thread.is_alive():
thread.join(timeout=3)